A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Allow shared_experts skip quantization for DeepSeekV2/V3 (#14100)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Mar 3, 2025 at 16:20 UTC
2b04c209ee98174f29f1fc98f0dc3222d652a7bd
1 file changed
+1
vllm/model_executor/models/deepseek_v2.py
+1
@@ -145,6 +145,7 @@ class DeepseekV2MoE(nn.Module):
145
hidden_act=config.hidden_act,
146
quant_config=quant_config,
147
reduce_results=False,
148
+ prefix=f"{prefix}.shared_experts",
149
)
150
151
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: