@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74692421f7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix]: phi.py get rope_theta from config file (#9503)

Co-authored-by: Isotr0py <2037008807@qq.com>

Falko1 committed Oct 22, 2024 at 04:53 UTC 74692421f7d5013c313790559f7fc2a338ae5272
1 file changed +3 -2
vllm/model_executor/models/phi.py
+3 -2
@@ -102,8 +102,9 @@ class PhiAttention(nn.Module):
102 # pylint: disable=C0301
103 # Refer to:
104 # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
105 - rope_theta = 10000
106 - max_position_embeddings = getattr(config, "n_positions", 2048)
105 + rope_theta = getattr(config, "rope_theta", 10000.0)
106 + max_position_embeddings = getattr(config, "max_position_embeddings",
107 + 2048)
108 self.rotary_emb = get_rope(
109 self.head_size,
110 rotary_dim=rotary_dim,