[CI] Split (H200) LM Eval Large Models into per-model jobs (#57965)
Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com> Co-authored-by: Kimi Code <noreply@moonshot.cn> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Thang Nguyen committed
Sep 21, 2026 at 20:19 UTC
78d78365254dfa20d3a265b6da695ab8151ec8b1
7 files changed
+62
-25
.buildkite/test_areas/lm_eval.yaml
+56
-19
@@ -343,29 +343,66 @@ steps:
343
commands:
344
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
345
346
-- label: ":nvidia: (H200) LM Eval Large Models"
347
- key: lm-eval-large-models-8xh200
348
- timeout_in_minutes: 120
346
+- label: ":nvidia: (H200) LM Eval DeepSeek R1 TP"
347
+ key: lm-eval-deepseek-r1-tp-8xh200
348
+ timeout_in_minutes: 35
349
device: h200
350
optional: true
351
num_devices: 8
352
commands:
353
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt
354
- mirror:
355
- amd:
356
- label: ":amd: (MI300) LM Eval Large Models"
357
- dind: false
358
- device: mi300_8
359
- timeout_in_minutes: 65
360
- depends_on:
361
- - image-build-amd
362
- commands:
363
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
364
- - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
365
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
366
- source_file_dependencies:
367
- - vllm/_aiter_ops.py
368
- - vllm/platforms/rocm.py
353
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-tp.txt
354
+
355
+- label: ":nvidia: (H200) LM Eval DeepSeek R1 DP"
356
+ key: lm-eval-deepseek-r1-dp-8xh200
357
+ timeout_in_minutes: 35
358
+ device: h200
359
+ optional: true
360
+ num_devices: 8
361
+ commands:
362
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-dp.txt
363
+
364
+- label: ":nvidia: (H200) LM Eval DeepSeek V3.2 TP"
365
+ key: lm-eval-deepseek-v32-tp-8xh200
366
+ timeout_in_minutes: 35
367
+ device: h200
368
+ optional: true
369
+ num_devices: 8
370
+ commands:
371
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-tp.txt
372
+
373
+- label: ":nvidia: (H200) LM Eval DeepSeek V3.2 DP"
374
+ key: lm-eval-deepseek-v32-dp-8xh200
375
+ timeout_in_minutes: 35
376
+ device: h200
377
+ optional: true
378
+ num_devices: 8
379
+ commands:
380
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-dp.txt
381
+
382
+- label: ":nvidia: (H200) LM Eval Nemotron 3 Super"
383
+ key: lm-eval-nemotron-3-super-8xh200
384
+ timeout_in_minutes: 35
385
+ device: h200
386
+ optional: true
387
+ num_devices: 8
388
+ commands:
389
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-nemotron-3-super.txt
390
+
391
+- label: ":amd: (MI300) LM Eval Large Models"
392
+ key: amd-lm-eval-large-models
393
+ dind: false
394
+ device: mi300_8
395
+ optional: true
396
+ timeout_in_minutes: 65
397
+ depends_on:
398
+ - image-build-amd
399
+ commands:
400
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
401
+ - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
402
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
403
+ source_file_dependencies:
404
+ - vllm/_aiter_ops.py
405
+ - vllm/platforms/rocm.py
406
407
- label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY"
408
key: moe-refactor-integration-test-h100-temporary
tests/evals/gsm8k/configs/models-h200-deepseek-r1-dp.txt
new
+1
@@ -0,0 +1 @@
1
+DeepSeek-R1-DP.yaml
tests/evals/gsm8k/configs/models-h200-deepseek-r1-tp.txt
new
+1
@@ -0,0 +1 @@
1
+DeepSeek-R1-TP.yaml
tests/evals/gsm8k/configs/models-h200-deepseek-v32-dp.txt
new
+1
@@ -0,0 +1 @@
1
+DeepSeek-V3.2-DP.yaml
tests/evals/gsm8k/configs/models-h200-deepseek-v32-tp.txt
new
+1
@@ -0,0 +1 @@
1
+DeepSeek-V3.2-TP.yaml
tests/evals/gsm8k/configs/models-h200-nemotron-3-super.txt
new
+2
@@ -0,0 +1,2 @@
1
+Nemotron-3-Super-120B-A12B-BF16.yaml
2
+Nemotron-3-Super-120B-A12B-FP8.yaml
tests/evals/gsm8k/configs/models-h200.txt
deleted
-6
@@ -1,6 +0,0 @@
1
-DeepSeek-R1-TP.yaml
2
-DeepSeek-R1-DP.yaml
3
-DeepSeek-V3.2-TP.yaml
4
-DeepSeek-V3.2-DP.yaml
5
-Nemotron-3-Super-120B-A12B-BF16.yaml
6
-Nemotron-3-Super-120B-A12B-FP8.yaml