A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Bugfix] Fix dynamic rotary embedding (#20343)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jul 2, 2025 at 14:31 UTC
1a03dd496b6daff8d58b41c8feaba4b520adfe1c
1 file changed
+7
-4
vllm/model_executor/layers/rotary_embedding.py
+7
-4
@@ -1963,16 +1963,19 @@ def get_rope(
1963
scaling_factor, dtype,
1964
mixed_b)
1965
elif scaling_type == "dynamic":
1966
- scaling_factor = rope_scaling["factor"]
1967
- scaling_alpha = rope_scaling["alpha"]
1968
- if scaling_alpha:
1966
+ if "alpha" in rope_scaling:
1967
+ scaling_alpha = rope_scaling["alpha"]
1968
rotary_emb = DynamicNTKAlphaRotaryEmbedding(
1969
head_size, rotary_dim, max_position, base, is_neox_style,
1970
scaling_alpha, dtype)
1972
- else:
1971
+ elif "factor" in rope_scaling:
1972
+ scaling_factor = rope_scaling["factor"]
1973
rotary_emb = DynamicNTKScalingRotaryEmbedding(
1974
head_size, rotary_dim, max_position, base, is_neox_style,
1975
scaling_factor, dtype)
1976
+ else:
1977
+ raise ValueError("Dynamic rope scaling must contain either "
1978
+ "'alpha' or 'factor' field")
1979
elif scaling_type == "yarn":
1980
scaling_factor = rope_scaling["factor"]
1981
original_max_position = rope_scaling[