A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[MTP][GLM][Bugfix] Fixed .weight_scale loading logic that dropped MTP prediction accuracy with fp8+mtp (#32101)
Signed-off-by: Andy Liu <andyliu@roblox.com>
Andy Liu committed
Jan 10, 2026 at 23:14 UTC
0dd63639be24c67a0bccc9e3b0f0fd6ab039455d
1 file changed
+6
-5
vllm/model_executor/models/glm4_moe_mtp.py
+6
-5
@@ -268,11 +268,6 @@ class Glm4MoeMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
268
if spec_layer is None:
269
continue
270
name = self._rewrite_spec_layer_name(spec_layer, name)
271
- # Some checkpoints include weight scale tensors for the LM head even
272
- # when the quantized head isn't built. Skip them if the model does
273
- # not expose a matching parameter to avoid KeyError during load.
274
- if name.endswith(".weight_scale") and name not in params_dict:
275
- continue
271
for param_name, weight_name, shard_id in stacked_params_mapping:
272
# Skip non-stacked layers and experts (experts handled below).
273
if weight_name not in name:
@@ -315,6 +310,12 @@ class Glm4MoeMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
310
# Skip loading extra bias for GPTQ models.
311
if name.endswith(".bias") and name not in params_dict:
312
continue
313
+ # Some checkpoints include weight scale tensors for the
314
+ # LM head even when the quantized head isn't built. Skip
315
+ # them if the model does not expose a matching parameter
316
+ # to avoid KeyError during load.
317
+ if name.endswith(".weight_scale") and name not in params_dict:
318
+ continue
319
320
# According to DeepSeek-V3 Technical Report, MTP modules
321
# shares embedding layer. We only load the first weights.