A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[ROCm][CI] Retrying in case of batch variance effects and reducing flakiness (#36442)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Mar 16, 2026 at 03:08 UTC
a2956a0f8e8f44cc79c14a9d3b45167631b7c249
1 file changed
+6
-2
tests/v1/e2e/general/test_async_scheduling.py
+6
-2
@@ -7,7 +7,10 @@ from typing import Any
7
import pytest
8
import torch._dynamo.config as dynamo_config
9
10
-from tests.utils import large_gpu_mark, single_gpu_only
10
+from tests.utils import (
11
+ large_gpu_mark,
12
+ single_gpu_only,
13
+)
14
from vllm import SamplingParams
15
from vllm.logprobs import Logprob
16
from vllm.platforms import current_platform
@@ -150,6 +153,7 @@ def test_with_eagle3_spec_decoding(sample_json_schema, monkeypatch: pytest.Monke
153
run_tests(monkeypatch, MTP_MODEL, test_configs, test_sampling_params)
154
155
156
+@pytest.mark.flaky(reruns=2, only_on=current_platform.is_rocm())
157
def test_with_ngram_gpu_spec_decoding(monkeypatch: pytest.MonkeyPatch):
158
"""Test ngram_gpu speculative decoding with different configurations.
159
@@ -202,7 +206,6 @@ def run_tests(
206
with monkeypatch.context() as m:
207
# lock matmul precision to full FP32 (IEEE)
208
m.setenv("VLLM_FLOAT32_MATMUL_PRECISION", "highest")
205
- # m.setenv("VLLM_BATCH_INVARIANT", "1")
209
outputs: list[tuple[str, list, list]] = []
210
for n, (
211
test_preemption,
@@ -351,6 +354,7 @@ def run_test(
354
speculative_config=spec_config,
355
disable_log_stats=False,
356
attention_config=attention_config,
357
+ enable_prefix_caching=False if current_platform.is_rocm() else None,
358
**cache_arg,
359
) as vllm_model:
360
results = []