[CI][ROCm] Add DSpark evals (#54852)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Sep 2, 2026 at 13:14 UTC
bf7a14d307bb495afc8c2587343bdc7d88c0f135
4 files changed
+98
.buildkite/test_areas/lm_eval.yaml
+39
@@ -123,6 +123,45 @@ steps:
123
- vllm/v1/worker/gpu/spec_decode/speculator.py
124
commands:
125
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt
126
+ mirror:
127
+ amd:
128
+ label: ":amd: (MI355) LM Eval Spec Decode"
129
+ dind: false
130
+ device: mi355_4
131
+ timeout_in_minutes: 120
132
+ depends_on:
133
+ - image-build-amd
134
+ source_file_dependencies:
135
+ - csrc/
136
+ - tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-AITER-TP4.yaml
137
+ - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml
138
+ - tests/evals/gsm8k/configs/models-spec-decode-rocm.txt
139
+ - tests/evals/gsm8k/test_gsm8k_correctness.py
140
+ - tests/evals/gsm8k/gsm8k_eval.py
141
+ - tests/evals/gsm8k/conftest.py
142
+ - vllm/config/
143
+ - vllm/envs.py
144
+ - vllm/models/kimi_k3/
145
+ - vllm/models/deepseek_v4/
146
+ - vllm/model_executor/kernels/mhc/
147
+ - vllm/model_executor/layers/mhc.py
148
+ - vllm/model_executor/layers/quantization/
149
+ - vllm/model_executor/layers/sparse_attn_indexer.py
150
+ - vllm/model_executor/models/config.py
151
+ - vllm/model_executor/models/registry.py
152
+ - vllm/model_executor/models/qwen3_dspark.py
153
+ - vllm/model_executor/model_loader/
154
+ - vllm/model_executor/warmup/
155
+ - vllm/model_executor/layers/fused_moe/
156
+ - vllm/tokenizers/
157
+ - vllm/v1/attention/
158
+ - vllm/v1/core/kv_cache_utils.py
159
+ - vllm/v1/spec_decode/metrics.py
160
+ - vllm/v1/worker/
161
+ - vllm/_aiter_ops.py
162
+ - vllm/platforms/rocm.py
163
+ commands:
164
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode-rocm.txt
165
166
- label: ":nvidia: (B200) LM Eval Large Models EP"
167
key: lm-eval-large-models-ep-2xb200
tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml
new
+29
@@ -0,0 +1,29 @@
1
+model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark"
2
+accuracy_threshold: 0.92
3
+# Loose lower bound for smoke testing fixed-length DSpark verification.
4
+min_acceptance_length: 1.2
5
+num_questions: 1319
6
+num_fewshot: 5
7
+startup_max_wait_seconds: 1800
8
+server_args: >-
9
+ --tokenizer-mode deepseek_v4
10
+ --trust-remote-code
11
+ --max-model-len 8192
12
+ --tensor-parallel-size 4
13
+ --enable-expert-parallel
14
+ --block-size 256
15
+ --gpu-memory-utilization 0.5
16
+ --kv-cache-dtype fp8
17
+ --attention_config.indexer_kv_dtype=fp8
18
+ --max-num-batched-tokens 16384
19
+ --max-num-seqs 128
20
+ --moe-backend aiter
21
+ --speculative-config '{"method":"dspark",
22
+ "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark",
23
+ "num_speculative_tokens":7,
24
+ "draft_sample_method":"probabilistic",
25
+ "enable_adaptive_verification":false}'
26
+env:
27
+ VLLM_USE_V2_MODEL_RUNNER: "1"
28
+ VLLM_ROCM_USE_AITER: "1"
29
+ VLLM_ROCM_USE_AITER_MOE: "1"
tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-AITER-TP4.yaml
new
+28
@@ -0,0 +1,28 @@
1
+model_name: "mgoin/Kimi-K3-pruned75"
2
+accuracy_threshold: 0.33
3
+min_acceptance_length: 4.8
4
+num_questions: 1319
5
+num_fewshot: 5
6
+startup_max_wait_seconds: 1800
7
+server_args: >-
8
+ --trust-remote-code
9
+ --gpu-memory-utilization 0.90
10
+ --tensor-parallel-size 4
11
+ --load-format fastsafetensors
12
+ --max-model-len 1048576
13
+ --max-num-batched-tokens 16384
14
+ --max-num-seqs 128
15
+ --kv-cache-dtype fp8
16
+ --moe-backend aiter
17
+ --attention-config '{"backend":"ROCM_AITER_MLA","mla_prefill_backend":"ROCM_AITER_FA"}'
18
+ --enable-prefix-caching
19
+ --enable-auto-tool-choice
20
+ --tool-call-parser kimi_k3
21
+ --reasoning-parser kimi_k3
22
+ --speculative-config '{"method":"dspark",
23
+ "model":"RedHatAI/Kimi-K3-speculator.dspark",
24
+ "attention_backend":"ROCM_AITER_FA","num_speculative_tokens":8,
25
+ "draft_sample_method":"probabilistic",
26
+ "rejection_sample_method":"block"}'
27
+env:
28
+ VLLM_ROCM_USE_AITER: "1"
tests/evals/gsm8k/configs/models-spec-decode-rocm.txt
new
+2
@@ -0,0 +1,2 @@
1
+Kimi-K3-pruned75-DSpark-AITER-TP4.yaml
2
+DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml