A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Bugfix] DeepSeek Accuracy (#14476)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Mar 8, 2025 at 11:47 UTC
db84f5eb3bd22ea76fc35f62bbc48081aa7365f8
1 file changed
+7
-5
vllm/v1/attention/backends/mla/common.py
+7
-5
@@ -222,8 +222,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
222
Fp8LinearGenericOp, current_platform_fp8_dtype, is_fp8)
223
from vllm.model_executor.layers.quantization.utils.quant_utils import (
224
scaled_quantize)
225
-from vllm.model_executor.layers.rotary_embedding import (
226
- DeepseekScalingRotaryEmbedding, RotaryEmbedding)
225
+from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
226
from vllm.utils import cdiv, round_down
227
228
try:
@@ -626,9 +625,12 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
625
self.qk_head_dim = qk_head_dim
626
self.v_head_dim = v_head_dim
627
629
- self.rotary_emb = rotary_emb
630
- self.use_yarn_rope = isinstance(rotary_emb,
631
- DeepseekScalingRotaryEmbedding)
628
+ # Hack for V1 for now to avoid torch library overhead (since we are
629
+ # already inside an attention custom op), pull out the forward
630
+ # method from the rotary embedding and call it directly
631
+ # TODO(lucas): we should probably find a cleaner way to do this
632
+ self.rotary_emb = rotary_emb._forward_method
633
+
634
self.q_proj = q_proj
635
self.kv_b_proj = kv_b_proj
636
self.o_proj = o_proj