[CI] Shard Humming H100 eval (#52326)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Kevin H. Luu committed
Aug 14, 2026 at 15:12 UTC
bb4b448f9896cf3f5607bd5ed4b35b29cd866a6b
4 files changed
+17
-2
.buildkite/test_areas/lm_eval.yaml
+3
-2
@@ -225,12 +225,13 @@ steps:
225
commands:
226
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
227
228
-- label: LM Eval Humming f16 (H100 - TEMPORARY)
228
+- label: LM Eval Humming f16 (H100 - TEMPORARY) %N
229
key: lm-eval-humming-f16-h100
230
timeout_in_minutes: 70
231
device: h100
232
optional: true
233
num_devices: 1
234
+ parallelism: 3
235
source_file_dependencies:
236
- vllm/model_executor/layers/quantization/humming.py
237
- vllm/model_executor/layers/quantization/utils/humming_utils.py
@@ -238,7 +239,7 @@ steps:
239
- vllm/model_executor/layers/fused_moe/oracle/
240
- vllm/model_executor/kernels/linear/
241
commands:
241
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
242
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-h100-shard-$$BUILDKITE_PARALLEL_JOB.txt
243
244
- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
245
key: lm-eval-humming-act-h100
tests/evals/gsm8k/configs/humming/config-h100-shard-0.txt
new
+5
@@ -0,0 +1,5 @@
1
+Qwen3.5-35B-A3B-experts-int8-humming.yaml
2
+Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
3
+Qwen3-30B-A3B-Fp8-v1-humming.yaml
4
+Qwen3-30B-A3B-NVFP4-humming.yaml
5
+Qwen2-1.5B-Instruct-FP8W8-humming.yaml
tests/evals/gsm8k/configs/humming/config-h100-shard-1.txt
new
+5
@@ -0,0 +1,5 @@
1
+Qwen3.6-35B-A3B-NVFP4-humming.yaml
2
+Qwen3-30B-A3B-AWQ-humming.yaml
3
+Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
4
+Qwen3-30B-A3B-FP8-block-humming.yaml
5
+Qwen3-0.6B-MXFP8-humming.yaml
tests/evals/gsm8k/configs/humming/config-h100-shard-2.txt
new
+4
@@ -0,0 +1,4 @@
1
+NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml
2
+Qwen3.5-35B-A3B-FP8-humming.yaml
3
+Qwen3-30B-A3B-MXFP4A16-humming.yaml
4
+gpt-oss-20b-humming.yaml