A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
Correct PPMissingLayer handling in Deepseek-V2-Lite PP deployment (#20665)
Signed-off-by: Seiji Eicher <seiji@anyscale.com>
Seiji Eicher committed
Jul 9, 2025 at 20:34 UTC
ad6c2e1a0b56c29065c7d70ff2e736e4f2fb03af
1 file changed
+9
-3
vllm/model_executor/models/deepseek_v2.py
+9
-3
@@ -739,14 +739,20 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
739
self.num_expert_groups = config.n_group
740
741
self.moe_layers: list[FusedMoE] = []
742
+ example_moe = None
743
for layer in self.model.layers:
744
+ if isinstance(layer, PPMissingLayer):
745
+ continue
746
+
747
assert isinstance(layer, DeepseekV2DecoderLayer)
748
if isinstance(layer.mlp, DeepseekV2MoE):
749
+ # Pick last one layer since the first ones may be dense layers.
750
+ example_moe = layer.mlp
751
self.moe_layers.append(layer.mlp.experts)
752
747
- # Pick last one layer since the first ones may be dense layers.
748
- example_moe = typing.cast(
749
- DeepseekV2MoE, self.model.layers[config.num_hidden_layers - 1].mlp)
753
+ if example_moe is None:
754
+ raise RuntimeError("No DeepseekV2MoE layer found in model.layers.")
755
+
756
self.num_logical_experts = example_moe.n_logical_experts
757
self.num_physical_experts = example_moe.n_physical_experts
758
self.num_local_physical_experts = example_moe.n_local_physical_experts