[Bugfix] Fix humming lm_head crash and FusedMoE weight_shape coercion (#46420)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Michael Goin committed
Jun 22, 2026 at 19:19 UTC
8207ce085069c1b6cf448b3172653eecfa8478ae
5 files changed
+27
-7
tests/evals/gsm8k/configs/gemma-4-E4B-it-qat-mobile-ct.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "google/gemma-4-E4B-it-qat-mobile-ct"
2
+accuracy_threshold: 0.50
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/models-small.txt
+2
-1
@@ -4,4 +4,5 @@ Llama-3-8B-Instruct-nonuniform-CT.yaml
4
Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
5
Qwen1.5-MoE-W4A16-CT.yaml
6
DeepSeek-V2-Lite-Instruct-FP8.yaml
7
-Qwen3-30B-A3B-MXFP4A16.yaml
\ No newline at end of file
7
+Qwen3-30B-A3B-MXFP4A16.yaml
8
+gemma-4-E4B-it-qat-mobile-ct.yaml
\ No newline at end of file
tests/evals/gsm8k/test_gsm8k_correctness.py
+9
@@ -70,6 +70,15 @@ def test_gsm8k_correctness(config_filename):
70
"Marlin kernels are not supported."
71
)
72
73
+ if (
74
+ not current_platform.is_cuda()
75
+ and "gemma-4-E4B-it-qat-mobile-ct" in eval_config["model_name"]
76
+ ):
77
+ pytest.skip(
78
+ "Skipping gemma-4-E4B-it-qat-mobile-ct on non-CUDA platforms. "
79
+ "Its W2A16 (uint2b2) scheme has no kernel outside CUDA."
80
+ )
81
+
82
# TODO(akaratza): Enable DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms
83
if current_platform.is_rocm() and (
84
"deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"]
vllm/model_executor/layers/fused_moe/routed_experts.py
+4
-2
@@ -530,8 +530,10 @@ class RoutedExperts(PluggableLayer):
530
):
531
param_data = param.data
532
533
- # Input scales can be loaded directly and should be equal.
534
- param_data[expert_id] = self._to_scalar(loaded_weight)
533
+ # Used for both scalar input_scale and the size-2 `weight_shape`
534
+ # param (compressed-tensors). Assign directly so both shapes load;
535
+ # _to_scalar's reshape(()) would reject the size-2 weight_shape.
536
+ param_data[expert_id] = loaded_weight
537
538
def _load_g_idx(
539
self,
vllm/model_executor/layers/quantization/utils/humming_utils.py
+7
-4
@@ -83,10 +83,13 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict):
83
input_schema = HummingInputSchema()
84
85
# ReplicatedLinear has no TP partitioning and so does not set
86
- # input_size_per_partition; for it that is just input_size.
87
- input_size_per_partition = getattr(
88
- layer, "input_size_per_partition", layer.input_size
89
- )
86
+ # input_size_per_partition; for it that is just input_size. Use hasattr
87
+ # rather than getattr's default arg, which is evaluated eagerly and would
88
+ # raise on layers lacking input_size (e.g. ParallelLMHead).
89
+ if hasattr(layer, "input_size_per_partition"):
90
+ input_size_per_partition = layer.input_size_per_partition
91
+ else:
92
+ input_size_per_partition = layer.input_size
93
shape_k_stacks = [input_size_per_partition]
94
shape_n_stacks = layer.output_partition_sizes
95