A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 08:45 UTC
[ROCm][CI] Fix serving tokens test failures (#34047)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 12, 2026 at 21:27 UTC
6afa587d31e911c4be495f16916d45d98ebd600c
1 file changed
+28
-1
tests/entrypoints/openai/test_serving_tokens.py
+28
-1
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import os
5
+
6
import httpx
7
import pytest
8
import pytest_asyncio
@@ -46,6 +48,27 @@ def server(request):
48
"--max-model-len",
49
"1024",
50
"--enforce-eager",
51
+ # On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
52
+ # 1 ULP when the batch dimension (M) changes, because different M
53
+ # values cause the Tensile backend to select different tile
54
+ # configurations with different fp32 accumulation orders. With
55
+ # prefix caching, cache-miss prefills compute all tokens in one
56
+ # pass (large M) while cache-hit requests compute only the
57
+ # uncached suffix (small M), seeding a divergence that amplifies
58
+ # through the residual stream and flips argmax tokens.
59
+ # See: https://github.com/vllm-project/vllm/issues/33123
60
+ #
61
+ # Either disable prefix caching entirely, or enable it with
62
+ # --deterministic-prefix-caching which forces cache-miss prefills
63
+ # to split at block boundaries so the suffix GEMM shape is always
64
+ # identical regardless of cache state.
65
+ #
66
+ # Option A: disable prefix caching
67
+ "--no-enable-prefix-caching",
68
+ #
69
+ # Option B: deterministic prefix caching
70
+ # "--enable-prefix-caching",
71
+ # "--deterministic-prefix-caching",
72
]
73
74
extra_args = getattr(request, "param", None)
@@ -56,7 +79,11 @@ def server(request):
79
else [str(extra_args)]
80
)
81
59
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
82
+ envs = os.environ.copy()
83
+ # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3888060787
84
+ envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
85
+
86
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
87
yield remote_server
88
89