[Quantization][CI] add humming lm-eval test (#43752)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jinzhen Lin committed
Jun 23, 2026 at 02:23 UTC
e4b3da3feb20c1854a4b23e431cfb787ee268f72
9 files changed
+91
-2
.buildkite/test_areas/lm_eval.yaml
+43
@@ -136,6 +136,49 @@ steps:
136
commands:
137
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
138
139
+- label: LM Eval Humming (A100 - TEMPORARY)
140
+ key: lm-eval-humming-a100
141
+ timeout_in_minutes: 30
142
+ device: a100
143
+ optional: true
144
+ num_devices: 1
145
+ source_file_dependencies:
146
+ - vllm/model_executor/layers/quantization/humming.py
147
+ - vllm/model_executor/layers/quantization/utils/humming_utils.py
148
+ - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
149
+ - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
150
+ commands:
151
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
152
+
153
+- label: LM Eval Humming (H100 - TEMPORARY)
154
+ key: lm-eval-humming-h100
155
+ timeout_in_minutes: 30
156
+ device: h100
157
+ optional: true
158
+ num_devices: 1
159
+ source_file_dependencies:
160
+ - vllm/model_executor/layers/quantization/humming.py
161
+ - vllm/model_executor/layers/quantization/utils/humming_utils.py
162
+ - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
163
+ - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
164
+ commands:
165
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
166
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
167
+
168
+- label: LM Eval Humming (B200 - TEMPORARY)
169
+ key: lm-eval-humming-b200
170
+ timeout_in_minutes: 30
171
+ device: b200-k8s
172
+ optional: true
173
+ num_devices: 1
174
+ source_file_dependencies:
175
+ - vllm/model_executor/layers/quantization/humming.py
176
+ - vllm/model_executor/layers/quantization/utils/humming_utils.py
177
+ - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
178
+ - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
179
+ commands:
180
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
181
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
182
183
- label: LM Eval TurboQuant KV Cache
184
key: lm-eval-turboquant-kv-cache
requirements/cuda.txt
+1
-1
@@ -26,4 +26,4 @@ quack-kernels>=0.3.3
26
tokenspeed-mla==0.1.2
27
28
# Humming kernels for quantization gemm
29
-humming-kernels[cu13]==0.1.4
29
+humming-kernels[cu13]==0.1.6
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
new
+12
@@ -0,0 +1,12 @@
1
+model_name: "nm-testing/Qwen3-30B-A3B-MXFP4A16"
2
+accuracy_threshold: 0.86
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 8192
8
+ --tensor-parallel-size 1
9
+ --quantization humming
10
+ --kernel-config.enable_flashinfer_autotune=False
11
+env:
12
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml
new
+10
@@ -0,0 +1,10 @@
1
+model_name: "nm-testing/Qwen3-30B-A3B-MXFP4A16"
2
+accuracy_threshold: 0.86
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 8192
8
+ --tensor-parallel-size 1
9
+ --quantization humming
10
+ --kernel-config.enable_flashinfer_autotune=False
tests/evals/gsm8k/configs/humming/config-act-fp8.txt
new
+2
@@ -0,0 +1,2 @@
1
+gpt-oss-20b-humming-act-fp8.yaml
2
+Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
tests/evals/gsm8k/configs/humming/config.txt
new
+2
@@ -0,0 +1,2 @@
1
+gpt-oss-20b-humming.yaml
2
+Qwen3-30B-A3B-MXFP4A16-humming.yaml
tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml
new
+11
@@ -0,0 +1,11 @@
1
+model_name: "openai/gpt-oss-20b"
2
+accuracy_threshold: 0.30
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 8192
8
+ --tensor-parallel-size 1
9
+ --moe-backend humming
10
+env:
11
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml
new
+9
@@ -0,0 +1,9 @@
1
+model_name: "openai/gpt-oss-20b"
2
+accuracy_threshold: 0.30
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 8192
8
+ --tensor-parallel-size 1
9
+ --moe-backend humming
vllm/model_executor/layers/quantization/humming.py
+1
-1
@@ -852,7 +852,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
852
853
# use moe modular
854
experts: HummingIndexedExperts | HummingGroupedExperts
855
- layer.ensure_moe_quant_config_init()
855
+ layer._ensure_moe_quant_config_init()
856
assert self.moe_quant_config is not None
857
if get_humming_moe_gemm_type() == "indexed":
858
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)