[CI] Shard Humming A100 eval (#52325)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Kevin H. Luu committed
Aug 17, 2026 at 02:03 UTC
bb233626caa31602728f7ee4625f3d2a4d1a3ad5
4 files changed
+17
-2
.buildkite/test_areas/lm_eval.yaml
+3
-2
@@ -195,12 +195,13 @@ steps:
195
commands:
196
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
197
198
-- label: LM Eval Humming f16 (A100 - TEMPORARY)
198
+- label: LM Eval Humming f16 (A100 - TEMPORARY) %N
199
key: lm-eval-humming-f16-a100
200
timeout_in_minutes: 75
201
device: a100
202
optional: true
203
num_devices: 1
204
+ parallelism: 3
205
source_file_dependencies:
206
- vllm/model_executor/layers/quantization/humming.py
207
- vllm/model_executor/layers/quantization/utils/humming_utils.py
@@ -208,7 +209,7 @@ steps:
209
- vllm/model_executor/layers/fused_moe/oracle/
210
- vllm/model_executor/kernels/linear/
211
commands:
211
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
212
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-a100-shard-$$BUILDKITE_PARALLEL_JOB.txt
213
214
- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
215
key: lm-eval-humming-act-a100
tests/evals/gsm8k/configs/humming/config-a100-shard-0.txt
new
+4
@@ -0,0 +1,4 @@
1
+Qwen3.5-35B-A3B-experts-int8-humming.yaml
2
+gpt-oss-20b-humming.yaml
3
+Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
4
+Qwen3-30B-A3B-FP8-block-humming.yaml
tests/evals/gsm8k/configs/humming/config-a100-shard-1.txt
new
+5
@@ -0,0 +1,5 @@
1
+Qwen3.6-35B-A3B-NVFP4-humming.yaml
2
+Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
3
+Qwen3-30B-A3B-Fp8-v1-humming.yaml
4
+Qwen3-30B-A3B-AWQ-humming.yaml
5
+Qwen3-0.6B-MXFP8-humming.yaml
tests/evals/gsm8k/configs/humming/config-a100-shard-2.txt
new
+5
@@ -0,0 +1,5 @@
1
+Qwen3.5-35B-A3B-FP8-humming.yaml
2
+NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml
3
+Qwen3-30B-A3B-MXFP4A16-humming.yaml
4
+Qwen3-30B-A3B-NVFP4-humming.yaml
5
+Qwen2-1.5B-Instruct-FP8W8-humming.yaml