A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 07:15 UTC
[ROCm][CI] Fix and stabilize EAGLE3 acceptance tests (#41294)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: Micah Williamson <micah.williamson@amd.com>
Andreas Karatzas committed
Jun 1, 2026 at 12:40 UTC
fd9e91d7e4116c9f3d1a3fc237677c925bf9d6d9
1 file changed
+30
-14
tests/v1/spec_decode/test_acceptance_length.py
+30
-14
@@ -39,6 +39,8 @@ class Eagle3ModelConfig:
39
marks: list = field(default_factory=list)
40
# Custom relative tolerance (defaults to DEFAULT_RTOL if None)
41
rtol: float | None = None
42
+ # ROCm-specific test configuration
43
+ rocm_expected_acceptance_lengths_per_pos: list[float] = field(default_factory=list)
44
45
46
# Model configurations for EAGLE3 acceptance length tests.
@@ -69,6 +71,7 @@ EAGLE3_MODEL_CONFIGS = [
71
# FLASHINFER incompatible: gpt-oss-20b uses sink attention which
72
# FLASHINFER does not support ("sink setting not supported")
73
excluded_backends={AttentionBackendEnum.FLASHINFER},
74
+ rocm_expected_acceptance_lengths_per_pos=[0.7040, 0.4820, 0.3350],
75
),
76
Eagle3ModelConfig(
77
verifier="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
@@ -99,16 +102,14 @@ EXCLUDED_BACKENDS = {AttentionBackendEnum.FLEX_ATTENTION}
102
103
104
def get_available_attention_backends() -> list[str]:
105
+ if current_platform.is_rocm():
106
+ return ["auto"]
107
+
108
# Check if get_valid_backends is actually defined in the platform class
109
# (not just returning None from __getattr__)
110
get_valid_backends = getattr(current_platform.__class__, "get_valid_backends", None)
111
if get_valid_backends is None:
106
- if current_platform.is_rocm():
107
- # ROCm uses Triton as its default attention backend since
108
- # Flash Attention is not supported.
109
- return ["TRITON_ATTN"]
110
- else:
111
- return ["FLASH_ATTN"]
112
+ return ["FLASH_ATTN"]
113
114
device_capability = current_platform.get_device_capability()
115
if device_capability is None:
@@ -167,6 +168,8 @@ def get_mt_bench_prompts(
168
disable_shuffle=False,
169
skip_chat_template=False,
170
trust_remote_code=False,
171
+ enable_multimodal_chat=False,
172
+ request_id_prefix="",
173
)
174
samples = get_samples(args, tokenizer)
175
prompt_ids = [
@@ -233,9 +236,12 @@ def test_eagle3_acceptance_length(
236
monkeypatch: pytest.MonkeyPatch,
237
):
238
# Skip if this backend is incompatible with the model
236
- backend_enum = AttentionBackendEnum[attention_backend]
237
- if backend_enum in model_config.excluded_backends:
238
- pytest.skip(f"{attention_backend} is incompatible with {model_config.id}")
239
+ attention_config = None
240
+ if attention_backend != "auto":
241
+ backend_enum = AttentionBackendEnum[attention_backend]
242
+ if backend_enum in model_config.excluded_backends:
243
+ pytest.skip(f"{attention_backend} is incompatible with {model_config.id}")
244
+ attention_config = {"backend": attention_backend}
245
246
with monkeypatch.context() as m:
247
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
@@ -247,11 +253,16 @@ def test_eagle3_acceptance_length(
253
"model": model_config.drafter,
254
"num_speculative_tokens": num_spec_tokens,
255
},
250
- attention_config={"backend": attention_backend},
256
+ attention_config=attention_config,
257
tensor_parallel_size=tp_size,
258
gpu_memory_utilization=0.7,
259
disable_log_stats=False,
260
max_model_len=DEFAULT_MAX_MODEL_LEN,
261
+ # Qwen/Qwen3-30B-A3B-FP8 with TP=4 needs EP
262
+ # https://github.com/vllm-project/vllm/issues/25292
263
+ enable_expert_parallel=(
264
+ tp_size == 4 and "Qwen3-VL" in model_config.verifier
265
+ ),
266
) as vllm_runner:
267
tokenizer = vllm_runner.llm.get_tokenizer()
268
prompt_ids = get_mt_bench_prompts(tokenizer, DEFAULT_NUM_PROMPTS)
@@ -272,6 +283,11 @@ def test_eagle3_acceptance_length(
283
expected = model_config.expected_acceptance_length
284
actual_per_pos = results["acceptance_lengths_per_pos"]
285
expected_per_pos = model_config.expected_acceptance_lengths_per_pos
286
+ if (
287
+ current_platform.is_rocm()
288
+ and model_config.rocm_expected_acceptance_lengths_per_pos
289
+ ):
290
+ expected_per_pos = model_config.rocm_expected_acceptance_lengths_per_pos
291
292
rel_error = abs(actual_acceptance_length - expected) / expected
293
@@ -294,14 +310,14 @@ def test_eagle3_acceptance_length(
310
zip(actual_per_pos, expected_per_pos)
311
):
312
if exp > 0:
297
- pos_rel_error = abs(actual - exp) / exp
298
- assert pos_rel_error <= rtol, (
313
+ min_expected = exp * (1 - rtol)
314
+ assert actual >= min_expected, (
315
f"Per-position acceptance length regression at pos {pos} "
316
f"for {model_config.id}!\n"
317
f" Expected: {exp:.3f}\n"
318
f" Actual: {actual:.3f}\n"
303
- f" Relative error: {pos_rel_error:.2%} "
304
- f"(tolerance: {rtol:.2%})"
319
+ f" Minimum: {min_expected:.3f}\n"
320
+ f" Tolerance: rtol={rtol:.2%}"
321
)
322
323
print(