A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Add missing `packed_modules_mapping` to `DeepseekV2ForCausalLM` (#22352)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
fxmarty-amd committed
Aug 7, 2025 at 15:30 UTC
7e0b121812a30975365497e39608898312c18984
1 file changed
+16
vllm/model_executor/models/deepseek_v2.py
+16
@@ -726,6 +726,9 @@ class DeepseekV2Model(nn.Module):
726
727
728
class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
729
+ packed_modules_mapping = {
730
+ "gate_up_proj": ["gate_proj", "up_proj"],
731
+ }
732
733
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
734
super().__init__()
@@ -733,6 +736,19 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
736
quant_config = vllm_config.quant_config
737
self.config = config
738
self.quant_config = quant_config
739
+
740
+ # `packed_modules_mapping` needs to be modified before
741
+ # initializing DeepseekV2Model, as it is passed inplace to
742
+ # quantization config init and may be used to select the
743
+ # quant_method for relevant layers during initialization.
744
+ self.fuse_qkv_a_proj = hasattr(
745
+ config, "q_lora_rank") and config.q_lora_rank is not None
746
+ if self.fuse_qkv_a_proj:
747
+ self.packed_modules_mapping["fused_qkv_a_proj"] = [
748
+ "q_a_proj",
749
+ "kv_a_proj_with_mqa",
750
+ ]
751
+
752
self.model = DeepseekV2Model(vllm_config=vllm_config,
753
prefix=maybe_prefix(prefix, "model"))
754
if get_pp_group().is_last_rank: