A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix]: phi.py get rope_theta from config file (#9503)
Co-authored-by: Isotr0py <2037008807@qq.com>
Falko1 committed
Oct 22, 2024 at 04:53 UTC
74692421f7d5013c313790559f7fc2a338ae5272
1 file changed
+3
-2
vllm/model_executor/models/phi.py
+3
-2
@@ -102,8 +102,9 @@ class PhiAttention(nn.Module):
102
# pylint: disable=C0301
103
# Refer to:
104
# https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
105
- rope_theta = 10000
106
- max_position_embeddings = getattr(config, "n_positions", 2048)
105
+ rope_theta = getattr(config, "rope_theta", 10000.0)
106
+ max_position_embeddings = getattr(config, "max_position_embeddings",
107
+ 2048)
108
self.rotary_emb = get_rope(
109
self.head_size,
110
rotary_dim=rotary_dim,