A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] Add MLA with pure RoPE support for deepseek-vl2 models (#12729)
Isotr0py committed
Feb 5, 2025 at 12:44 UTC
98fd089fc974313ac13370f79f4c02a3839baf4a
3 files changed
+30
-6
vllm/attention/backends/mla/utils.py
+26
-4
@@ -26,7 +26,8 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
26
apply_fp8_linear_generic, current_platform_fp8_dtype, is_fp8)
27
from vllm.model_executor.layers.quantization.utils.quant_utils import (
28
scaled_dequantize, scaled_quantize)
29
-from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
29
+from vllm.model_executor.layers.rotary_embedding import (
30
+ DeepseekScalingRotaryEmbedding, RotaryEmbedding)
31
32
try:
33
from vllm.vllm_flash_attn import flash_attn_varlen_func
@@ -174,6 +175,8 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
175
self.v_head_dim = v_head_dim
176
177
self.rotary_emb = rotary_emb
178
+ self.use_yarn_rope = isinstance(rotary_emb,
179
+ DeepseekScalingRotaryEmbedding)
180
self.q_proj = q_proj
181
self.kv_b_proj = kv_b_proj
182
self.o_proj = o_proj
@@ -420,6 +423,24 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
423
) -> torch.Tensor:
424
raise NotImplementedError
425
426
+ def apply_pure_rope(
427
+ self,
428
+ input_positions: torch.Tensor,
429
+ q_pe: torch.Tensor,
430
+ k_pe: torch.Tensor,
431
+ ) -> tuple[torch.Tensor, torch.Tensor]:
432
+ seq_len = input_positions.size(0)
433
+ ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
434
+
435
+ q_pe, k_pe = self.rotary_emb(
436
+ input_positions,
437
+ q_pe.reshape(seq_len, -1),
438
+ k_pe.reshape(seq_len, -1),
439
+ )
440
+ q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
441
+
442
+ return q_pe, k_pe
443
+
444
def forward(
445
self,
446
layer: AttentionLayer,
@@ -444,13 +465,14 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
465
# Restore head dim (for rotary embedding)
466
k_pe = k_pe.unsqueeze(1)
467
assert hasattr(attn_metadata, "input_positions")
468
+ rope_fn = (self.rotary_emb
469
+ if self.use_yarn_rope else self.apply_pure_rope)
470
471
if is_decode:
472
q_nope = self._q_proj_and_k_up_proj(hidden_states_or_q_c)
473
q_pe = torch.matmul(hidden_states_or_q_c, self.W_QR)\
474
.view(-1, self.num_heads, self.qk_rope_head_dim)
452
- q_pe, k_pe = \
453
- self.rotary_emb(attn_metadata.input_positions, q_pe, k_pe)
475
+ q_pe, k_pe = rope_fn(attn_metadata.input_positions, q_pe, k_pe)
476
else:
477
assert is_prefill
478
q = self.q_proj(hidden_states_or_q_c)[0]\
@@ -458,7 +480,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
480
481
# TODO(lucas): there must be a nicer way to write this line
482
q[..., self.qk_nope_head_dim:], k_pe = \
461
- self.rotary_emb(
483
+ rope_fn(
484
attn_metadata.input_positions,
485
q[..., self.qk_nope_head_dim:], k_pe)
486
vllm/model_executor/models/deepseek_v2.py
+2
-1
@@ -414,7 +414,8 @@ class DeepseekV2MLAAttention(nn.Module):
414
quant_config=quant_config,
415
prefix=f"{prefix}.o_proj")
416
417
- rope_scaling["rope_type"] = 'deepseek_yarn'
417
+ if rope_scaling:
418
+ rope_scaling["rope_type"] = 'deepseek_yarn'
419
self.rotary_emb = get_rope(qk_rope_head_dim,
420
rotary_dim=qk_rope_head_dim,
421
max_position=max_position_embeddings,
vllm/model_executor/models/deepseek_v3.py
+2
-1
@@ -422,7 +422,8 @@ class DeepseekV3MLAAttention(nn.Module):
422
quant_config=quant_config,
423
prefix=f"{prefix}.o_proj")
424
425
- rope_scaling["rope_type"] = 'deepseek_yarn'
425
+ if rope_scaling:
426
+ rope_scaling["rope_type"] = 'deepseek_yarn'
427
self.rotary_emb = get_rope(qk_rope_head_dim,
428
rotary_dim=qk_rope_head_dim,
429
max_position=max_position_embeddings,