[CI] Raise GSM8K startup max wait to 2400s for flaky 30B+ MoE eval configs (#57361)
Signed-off-by: khluu <khluu000@gmail.com>
Kevin H. Luu committed
Sep 18, 2026 at 03:40 UTC
04a3a00eb79118dda521577caaaba2ba6b65bb33
5 files changed
+10
tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
+2
@@ -3,6 +3,8 @@ accuracy_threshold: 0.90
3
num_questions: 1319
4
num_fewshot: 5
5
max_concurrency: 100
6
+# 30B+ MoE with EP/PCP: download + kernel autotuning can exceed the 1200s default.
7
+startup_max_wait_seconds: 2400
8
# Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation.
9
server_args: >-
10
--compilation-config '{"cudagraph_mode":"NONE"}'
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml
+2
@@ -2,6 +2,8 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
2
accuracy_threshold: 0.88
3
num_questions: 1319
4
num_fewshot: 5
5
+# 30B MoE + DeepGEMM/DeepEP JIT compile can exceed the 1200s default startup wait.
6
+startup_max_wait_seconds: 2400
7
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log"
8
env:
9
VLLM_USE_DEEP_GEMM: "1"
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml
+2
@@ -2,6 +2,8 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
2
accuracy_threshold: 0.85
3
num_questions: 1319
4
num_fewshot: 5
5
+# 30B MoE + DeepGEMM/DeepEP JIT compile can exceed the 1200s default startup wait.
6
+startup_max_wait_seconds: 2400
7
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log"
8
env:
9
VLLM_USE_DEEP_GEMM: "1"
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml
+2
@@ -2,4 +2,6 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
accuracy_threshold: 0.88
3
num_questions: 1319
4
num_fewshot: 5
5
+# 30B MoE + CuTeDSL/DeepEP JIT compile can exceed the 1200s default startup wait.
6
+startup_max_wait_seconds: 2400
7
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl"
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml
+2
@@ -2,4 +2,6 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
accuracy_threshold: 0.88
3
num_questions: 1319
4
num_fewshot: 5
5
+# 30B MoE + CuTeDSL/DeepEP JIT compile can exceed the 1200s default startup wait.
6
+startup_max_wait_seconds: 2400
7
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl"