[CI] Add GSM8K accuracy test for amd/DeepSeek-V4-Flash-MXFP4 (#50632)
Signed-off-by: Colin Zeng <Colin.Zeng@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Colin Z committed
Aug 25, 2026 at 09:06 UTC
bc11ecaf4e6aeaa0d6a18567d59d75a90207444e
4 files changed
+32
-2
.buildkite/lm-eval-harness/configs/DeepSeek-V4-Flash-MXFP4.yaml
new
+25
@@ -0,0 +1,25 @@
1
+# DeepSeek-V4-Flash MXFP4 (AMD Quark) accuracy test.
2
+# Runs on ROCm (gfx950) with the AITER MoE + fused shared-experts path.
3
+# Reference GSM8K (8-shot, lm-eval): strict 0.9522 / flexible 0.9515 measured
4
+# on gfx950 TP=4 (AITER_MXFP4_BF16 backend); default rtol=0.08.
5
+model_name: "amd/DeepSeek-V4-Flash-MXFP4"
6
+tasks:
7
+- name: "gsm8k"
8
+ metrics:
9
+ - name: "exact_match,strict-match"
10
+ value: 0.95
11
+ - name: "exact_match,flexible-extract"
12
+ value: 0.95
13
+limit: 1319
14
+num_fewshot: 8
15
+max_model_len: 8192
16
+kv_cache_dtype: fp8
17
+moe_backend: aiter
18
+enforce_eager: false
19
+trust_remote_code: true
20
+tokenizer_mode: deepseek_v4
21
+required_gpu_arch:
22
+- gfx950
23
+env_vars:
24
+ VLLM_ROCM_USE_AITER: "1"
25
+ VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "1"
.buildkite/lm-eval-harness/configs/models-large-rocm-tp4.txt
renamed
+1
@@ -1 +1,2 @@
1
Qwen3-235B-A22B-Instruct-2507-FP8.yaml
2
+DeepSeek-V4-Flash-MXFP4.yaml
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+4
@@ -77,6 +77,10 @@ def launch_lm_eval(eval_config, tp_size):
77
if rocm_load_strategy is not None:
78
model_args += f"safetensors_load_strategy={rocm_load_strategy},"
79
80
+ tokenizer_mode = eval_config.get("tokenizer_mode", None)
81
+ if tokenizer_mode is not None:
82
+ model_args += f"tokenizer_mode={tokenizer_mode},"
83
+
84
env_vars = eval_config.get("env_vars", None)
85
with scoped_env_vars(env_vars):
86
results = lm_eval.simple_evaluate(
.buildkite/test-amd.yaml
+2
-2
@@ -1488,7 +1488,7 @@ steps:
1488
- vllm/platforms/rocm.py
1489
commands:
1490
- export VLLM_USE_DEEP_GEMM=0
1491
- - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
1491
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
1492
1493
#--------------------------------------------------------- mi300 · examples ----------------------------------------------------------#
1494
@@ -3374,7 +3374,7 @@ steps:
3374
- vllm/platforms/rocm.py
3375
commands:
3376
- export VLLM_USE_DEEP_GEMM=0
3377
- - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
3377
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
3378
3379
- label: ":amd: (MI355) LM Eval Large Models" # TBD
3380
timeout_in_minutes: 180