@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98fd089fc9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] Add MLA with pure RoPE support for deepseek-vl2 models (#12729)

Isotr0py committed Feb 5, 2025 at 12:44 UTC 98fd089fc974313ac13370f79f4c02a3839baf4a
3 files changed +30 -6
vllm/attention/backends/mla/utils.py
+26 -4
@@ -26,7 +26,8 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
26 apply_fp8_linear_generic, current_platform_fp8_dtype, is_fp8)
27 from vllm.model_executor.layers.quantization.utils.quant_utils import (
28 scaled_dequantize, scaled_quantize)
29 -from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
29 +from vllm.model_executor.layers.rotary_embedding import (
30 + DeepseekScalingRotaryEmbedding, RotaryEmbedding)
31
32 try:
33 from vllm.vllm_flash_attn import flash_attn_varlen_func
@@ -174,6 +175,8 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
175 self.v_head_dim = v_head_dim
176
177 self.rotary_emb = rotary_emb
178 + self.use_yarn_rope = isinstance(rotary_emb,
179 + DeepseekScalingRotaryEmbedding)
180 self.q_proj = q_proj
181 self.kv_b_proj = kv_b_proj
182 self.o_proj = o_proj
@@ -420,6 +423,24 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
423 ) -> torch.Tensor:
424 raise NotImplementedError
425
426 + def apply_pure_rope(
427 + self,
428 + input_positions: torch.Tensor,
429 + q_pe: torch.Tensor,
430 + k_pe: torch.Tensor,
431 + ) -> tuple[torch.Tensor, torch.Tensor]:
432 + seq_len = input_positions.size(0)
433 + ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
434 +
435 + q_pe, k_pe = self.rotary_emb(
436 + input_positions,
437 + q_pe.reshape(seq_len, -1),
438 + k_pe.reshape(seq_len, -1),
439 + )
440 + q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
441 +
442 + return q_pe, k_pe
443 +
444 def forward(
445 self,
446 layer: AttentionLayer,
@@ -444,13 +465,14 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
465 # Restore head dim (for rotary embedding)
466 k_pe = k_pe.unsqueeze(1)
467 assert hasattr(attn_metadata, "input_positions")
468 + rope_fn = (self.rotary_emb
469 + if self.use_yarn_rope else self.apply_pure_rope)
470
471 if is_decode:
472 q_nope = self._q_proj_and_k_up_proj(hidden_states_or_q_c)
473 q_pe = torch.matmul(hidden_states_or_q_c, self.W_QR)\
474 .view(-1, self.num_heads, self.qk_rope_head_dim)
452 - q_pe, k_pe = \
453 - self.rotary_emb(attn_metadata.input_positions, q_pe, k_pe)
475 + q_pe, k_pe = rope_fn(attn_metadata.input_positions, q_pe, k_pe)
476 else:
477 assert is_prefill
478 q = self.q_proj(hidden_states_or_q_c)[0]\
@@ -458,7 +480,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
480
481 # TODO(lucas): there must be a nicer way to write this line
482 q[..., self.qk_nope_head_dim:], k_pe = \
461 - self.rotary_emb(
483 + rope_fn(
484 attn_metadata.input_positions,
485 q[..., self.qk_nope_head_dim:], k_pe)
486
vllm/model_executor/models/deepseek_v2.py
+2 -1
@@ -414,7 +414,8 @@ class DeepseekV2MLAAttention(nn.Module):
414 quant_config=quant_config,
415 prefix=f"{prefix}.o_proj")
416
417 - rope_scaling["rope_type"] = 'deepseek_yarn'
417 + if rope_scaling:
418 + rope_scaling["rope_type"] = 'deepseek_yarn'
419 self.rotary_emb = get_rope(qk_rope_head_dim,
420 rotary_dim=qk_rope_head_dim,
421 max_position=max_position_embeddings,
vllm/model_executor/models/deepseek_v3.py
+2 -1
@@ -422,7 +422,8 @@ class DeepseekV3MLAAttention(nn.Module):
422 quant_config=quant_config,
423 prefix=f"{prefix}.o_proj")
424
425 - rope_scaling["rope_type"] = 'deepseek_yarn'
425 + if rope_scaling:
426 + rope_scaling["rope_type"] = 'deepseek_yarn'
427 self.rotary_emb = get_rope(qk_rope_head_dim,
428 rotary_dim=qk_rope_head_dim,
429 max_position=max_position_embeddings,