A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 22:45 UTC
[ROCm][CI] Fix spec decode profile assertion and logprob test determinism (#35043)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 23, 2026 at 07:05 UTC
5f68464f92af79b0b851edc7f2bba7c861069713
1 file changed
+20
tests/v1/sample/test_logprobs.py
+20
@@ -20,6 +20,7 @@ from tests.v1.sample.utils import (
20
from vllm import SamplingParams
21
from vllm.config.model import LogprobsMode
22
from vllm.distributed import cleanup_dist_env_and_memory
23
+from vllm.platforms import current_platform
24
25
from ...conftest import HfRunner, VllmRunner
26
@@ -31,6 +32,23 @@ SAMPLE = BatchLogprobsComposition.SAMPLE
32
PROMPT = BatchLogprobsComposition.PROMPT
33
SAMPLE_PROMPT = BatchLogprobsComposition.SAMPLE_PROMPT
34
35
+# On ROCm, floating-point reductions in attention and GEMM kernels are
36
+# non-associative and sensitive to batch geometry. The ref LLM (no spec
37
+# decode, default scheduling) and the spec-decode LLM (chunked prefill,
38
+# different effective batch sizes) follow different reduction orders,
39
+# producing numerically divergent logprobs that get mis-attributed to
40
+# spec-decode incorrectness.
41
+#
42
+# Force LLM instances into an identical, deterministic execution
43
+# mode so the test isolates spec-decode correctness only:
44
+ROCM_DETERMINISM_KWARGS: dict = (
45
+ dict(
46
+ max_num_seqs=1,
47
+ )
48
+ if current_platform.is_rocm()
49
+ else {}
50
+)
51
+
52
53
@pytest.fixture(
54
scope="module",
@@ -1035,6 +1053,7 @@ def test_spec_decode_logprobs(
1053
logprobs_mode=logprobs_mode,
1054
gpu_memory_utilization=0.4,
1055
enable_prefix_caching=False,
1056
+ **ROCM_DETERMINISM_KWARGS,
1057
)
1058
ref_results = ref_llm.generate(
1059
[prompt, prompt], [sampling_params, penalty_sampling_params]
@@ -1064,6 +1083,7 @@ def test_spec_decode_logprobs(
1083
enable_chunked_prefill=True,
1084
max_num_batched_tokens=32,
1085
enable_prefix_caching=False,
1086
+ **ROCM_DETERMINISM_KWARGS,
1087
)
1088
spec_results = spec_llm.generate(
1089
[prompt, prompt], [sampling_params, penalty_sampling_params]