@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d27f4bae39
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix rope cache key error (#1867)

Roy committed Dec 1, 2023 at 00:29 UTC d27f4bae393214b4e7715fc3cb5754d4bf801bce
1 file changed +2 -1
vllm/model_executor/layers/rotary_embedding.py
+2 -1
@@ -284,9 +284,10 @@ def get_rope(
284 rope_scaling: Optional[Dict[str, Any]] = None,
285 ) -> RotaryEmbedding:
286 key = (head_size, rotary_dim, max_position, base, is_neox_style,
287 - rope_scaling)
287 + tuple(rope_scaling.items()) if rope_scaling is not None else None)
288 if key in _ROPE_DICT:
289 return _ROPE_DICT[key]
290 +
291 if rope_scaling is None:
292 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
293 is_neox_style)