A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Use RoPE cache for MRoPE (#8396)
Woosuk Kwon committed
Sep 11, 2024 at 23:13 UTC
42ffba11ad4597289b5ae609900a74a153fbd067
1 file changed
+1
-1
vllm/model_executor/layers/rotary_embedding.py
+1
-1
@@ -990,7 +990,7 @@ def get_rope(
990
base, is_neox_style, dtype, short_factor, long_factor,
991
**extra_kwargs)
992
elif scaling_type == "mrope":
993
- return MRotaryEmbedding(
993
+ rotary_emb = MRotaryEmbedding(
994
head_size,
995
rotary_dim,
996
max_position,