A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix rope cache key error (#1867)
Roy committed
Dec 1, 2023 at 00:29 UTC
d27f4bae393214b4e7715fc3cb5754d4bf801bce
1 file changed
+2
-1
vllm/model_executor/layers/rotary_embedding.py
+2
-1
@@ -284,9 +284,10 @@ def get_rope(
284
rope_scaling: Optional[Dict[str, Any]] = None,
285
) -> RotaryEmbedding:
286
key = (head_size, rotary_dim, max_position, base, is_neox_style,
287
- rope_scaling)
287
+ tuple(rope_scaling.items()) if rope_scaling is not None else None)
288
if key in _ROPE_DICT:
289
return _ROPE_DICT[key]
290
+
291
if rope_scaling is None:
292
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
293
is_neox_style)