@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e69c04887
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 12:45 UTC

[ROCm][CI] Parametrize vision score tests across attention backends with per-backend tolerances (#35571)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

Andreas Karatzas committed Feb 28, 2026 at 02:59 UTC 1e69c048877335e92720772cac704650ad99b219
1 file changed +153 -40
tests/entrypoints/pooling/score/test_online_score_vision.py
+153 -40
@@ -1,12 +1,15 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +import json
5 +
6 import pytest
7 import requests
8
9 from tests.utils import VLLM_PATH, RemoteOpenAIServer
10 from vllm.entrypoints.pooling.score.protocol import RerankResponse, ScoreResponse
11 from vllm.multimodal.utils import encode_image_url, fetch_image
12 +from vllm.platforms import current_platform
13
14 MODEL_NAME = "Qwen/Qwen3-VL-Reranker-2B"
15 HF_OVERRIDES = {
@@ -15,6 +18,60 @@ HF_OVERRIDES = {
18 "is_original_qwen3_reranker": True,
19 }
20
21 +ROCM_ATTN_BACKENDS = [
22 + "ROCM_ATTN",
23 + "ROCM_AITER_FA",
24 + "TRITON_ATTN",
25 + "FLEX_ATTENTION",
26 +]
27 +
28 +ATTN_BACKENDS = ROCM_ATTN_BACKENDS if current_platform.is_rocm() else []
29 +
30 +# Per-backend tolerance with explicit entries; "default" is the fallback
31 +BACKEND_TOL: dict[str, float] = {
32 + "default": 0.05, # 5% tolerance for other backends (e.g. FLASH_ATTN)
33 + # Relaxed tolerances for ROCm attn
34 + # See: https://github.com/vllm-project/vllm/issues/35569
35 + "ROCM_ATTN": 0.09, # gfx950:~8.45%, gfx942:~3.70%
36 + "ROCM_AITER_FA": 0.045, # gfx950:~2.00%, gfx942:~0.80%
37 + "TRITON_ATTN": 0.045, # gfx950:~3.00%, gfx942:~2.20%
38 + "FLEX_ATTENTION": 0.045, # gfx950:~3.25%, gfx942:~1.10%
39 +}
40 +
41 +# ROCm: disable skinny GEMM to avoid non-deterministic results from
42 +# atomic reductions in wvSplitKrc kernel.
43 +# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
44 +ROCM_ENV_OVERRIDES = (
45 + {"VLLM_ROCM_USE_SKINNY_GEMM": "0"} if current_platform.is_rocm() else {}
46 +)
47 +# ROCm: disable prefix caching and eliminate batch variance to reduce
48 +# test flakiness.
49 +ROCM_EXTRA_ARGS = (
50 + ["--no-enable-prefix-caching", "--max-num-seqs", "1"]
51 + if current_platform.is_rocm()
52 + else []
53 +)
54 +
55 +
56 +def get_tol(backend: str) -> float:
57 + return BACKEND_TOL.get(backend, BACKEND_TOL["default"])
58 +
59 +
60 +def assert_score(actual: float, expected: float, backend: str, label: str):
61 + tol = get_tol(backend)
62 + diff = abs(actual - expected)
63 + rel_diff = diff / abs(expected) if expected != 0 else diff
64 + print(
65 + f"[{backend}] {label}: actual={actual:.6f} expected={expected:.6f} "
66 + f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol}"
67 + )
68 + assert actual == pytest.approx(expected, rel=tol), (
69 + f"[{backend}] {label}: score mismatch — "
70 + f"actual={actual:.6f}, expected={expected:.6f}, "
71 + f"rel_diff={rel_diff:.4f}, tol={tol}"
72 + )
73 +
74 +
75 query = "A cat standing in the snow."
76 document = "This product was excellent and exceeded my expectations."
77 image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
@@ -36,28 +93,37 @@ documents = [
93 TEXT_VS_TEXT = 0.10040374100208282
94 TEXT_VS_IMAGE = 0.7423753142356873
95 TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838
39 -TOL = 0.05
96
97
42 -@pytest.fixture(scope="module")
43 -def server():
98 +@pytest.fixture(scope="module", params=ATTN_BACKENDS)
99 +def server(request):
100 + backend = request.param
101 + print(f"\n=== Starting server with attention backend: {backend} ===")
102 args = [
103 "--enforce-eager",
104 "--max-model-len",
105 "8192",
106 "--chat-template",
107 str(VLLM_PATH / "examples/pooling/score/template/qwen3_vl_reranker.jinja"),
50 - ]
108 + "--attention-config",
109 + json.dumps({"backend": backend}),
110 + ] + ROCM_EXTRA_ARGS
111 +
112 + env = dict(ROCM_ENV_OVERRIDES)
113 + if backend != "ROCM_AITER_FA":
114 + env["VLLM_ROCM_USE_AITER"] = "0"
115
116 with RemoteOpenAIServer(
53 - MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
117 + MODEL_NAME, args, override_hf_configs=HF_OVERRIDES, env_dict=env
118 ) as remote_server:
55 - yield remote_server
119 + print(f"=== Server ready with backend: {backend} ===")
120 + yield remote_server, backend
121
122
58 -def test_score_api_queries_str_documents_str(server: RemoteOpenAIServer):
123 +def test_score_api_queries_str_documents_str(server: tuple[RemoteOpenAIServer, str]):
124 + remote_server, backend = server
125 score_response = requests.post(
60 - server.url_for("score"),
126 + remote_server.url_for("score"),
127 json={
128 "model": MODEL_NAME,
129 "queries": query,
@@ -71,12 +137,15 @@ def test_score_api_queries_str_documents_str(server: RemoteOpenAIServer):
137 assert score.data is not None
138 assert len(score.data) == 1
139 assert score.usage.prompt_tokens == 81
74 - assert score.data[0].score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
140 + assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
141
142
77 -def test_score_api_queries_str_documents_text_content(server: RemoteOpenAIServer):
143 +def test_score_api_queries_str_documents_text_content(
144 + server: tuple[RemoteOpenAIServer, str],
145 +):
146 + remote_server, backend = server
147 score_response = requests.post(
79 - server.url_for("score"),
148 + remote_server.url_for("score"),
149 json={
150 "model": MODEL_NAME,
151 "queries": query,
@@ -90,12 +159,15 @@ def test_score_api_queries_str_documents_text_content(server: RemoteOpenAIServer
159 assert score.data is not None
160 assert len(score.data) == 1
161 assert score.usage.prompt_tokens == 81
93 - assert score.data[0].score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
162 + assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
163
164
96 -def test_score_api_queries_str_documents_image_url_content(server: RemoteOpenAIServer):
165 +def test_score_api_queries_str_documents_image_url_content(
166 + server: tuple[RemoteOpenAIServer, str],
167 +):
168 + remote_server, backend = server
169 score_response = requests.post(
98 - server.url_for("score"),
170 + remote_server.url_for("score"),
171 json={
172 "model": MODEL_NAME,
173 "queries": query,
@@ -109,14 +181,15 @@ def test_score_api_queries_str_documents_image_url_content(server: RemoteOpenAIS
181 assert score.data is not None
182 assert len(score.data) == 1
183 assert score.usage.prompt_tokens == 98
112 - assert score.data[0].score == pytest.approx(TEXT_VS_IMAGE, rel=TOL)
184 + assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image")
185
186
187 def test_score_api_queries_str_documents_image_base64_content(
116 - server: RemoteOpenAIServer,
188 + server: tuple[RemoteOpenAIServer, str],
189 ):
190 + remote_server, backend = server
191 score_response = requests.post(
119 - server.url_for("score"),
192 + remote_server.url_for("score"),
193 json={
194 "model": MODEL_NAME,
195 "queries": query,
@@ -130,14 +203,15 @@ def test_score_api_queries_str_documents_image_base64_content(
203 assert score.data is not None
204 assert len(score.data) == 1
205 assert score.usage.prompt_tokens == 98
133 - assert score.data[0].score == pytest.approx(TEXT_VS_IMAGE, rel=TOL)
206 + assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image_base64")
207
208
209 def test_score_api_queries_str_documents_image_url_plus_text_content(
137 - server: RemoteOpenAIServer,
210 + server: tuple[RemoteOpenAIServer, str],
211 ):
212 + remote_server, backend = server
213 score_response = requests.post(
140 - server.url_for("score"),
214 + remote_server.url_for("score"),
215 json={
216 "model": MODEL_NAME,
217 "queries": query,
@@ -151,12 +225,17 @@ def test_score_api_queries_str_documents_image_url_plus_text_content(
225 assert score.data is not None
226 assert len(score.data) == 1
227 assert score.usage.prompt_tokens == 108
154 - assert score.data[0].score == pytest.approx(TEXT_VS_TEXT_PLUS_IMAGE, rel=TOL)
228 + assert_score(
229 + score.data[0].score, TEXT_VS_TEXT_PLUS_IMAGE, backend, "text_vs_text_plus_image"
230 + )
231
232
157 -def test_score_api_queries_str_documents_list(server: RemoteOpenAIServer):
233 +def test_score_api_queries_str_documents_list(
234 + server: tuple[RemoteOpenAIServer, str],
235 +):
236 + remote_server, backend = server
237 score_response = requests.post(
159 - server.url_for("score"),
238 + remote_server.url_for("score"),
239 json={
240 "model": MODEL_NAME,
241 "queries": query,
@@ -175,15 +254,23 @@ def test_score_api_queries_str_documents_list(server: RemoteOpenAIServer):
254 assert score.data is not None
255 assert len(score.data) == 4
256 assert score.usage.prompt_tokens == 368
178 - assert score.data[0].score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
179 - assert score.data[1].score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
180 - assert score.data[2].score == pytest.approx(TEXT_VS_IMAGE, rel=TOL)
181 - assert score.data[3].score == pytest.approx(TEXT_VS_TEXT_PLUS_IMAGE, rel=TOL)
257 + assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "list[0]_text_vs_text")
258 + assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "list[1]_text_vs_text")
259 + assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "list[2]_text_vs_image")
260 + assert_score(
261 + score.data[3].score,
262 + TEXT_VS_TEXT_PLUS_IMAGE,
263 + backend,
264 + "list[3]_text_vs_text_plus_image",
265 + )
266
267
184 -def test_rerank_api_queries_str_documents_list(server: RemoteOpenAIServer):
268 +def test_rerank_api_queries_str_documents_list(
269 + server: tuple[RemoteOpenAIServer, str],
270 +):
271 + remote_server, backend = server
272 rerank_response = requests.post(
186 - server.url_for("rerank"),
273 + remote_server.url_for("rerank"),
274 json={
275 "model": MODEL_NAME,
276 "query": query,
@@ -204,17 +291,38 @@ def test_rerank_api_queries_str_documents_list(server: RemoteOpenAIServer):
291 assert len(rerank.results) == 4
292
293 rerank.results.sort(key=lambda x: x.index)
207 - assert rerank.results[0].relevance_score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
208 - assert rerank.results[1].relevance_score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
209 - assert rerank.results[2].relevance_score == pytest.approx(TEXT_VS_IMAGE, rel=TOL)
210 - assert rerank.results[3].relevance_score == pytest.approx(
211 - TEXT_VS_TEXT_PLUS_IMAGE, rel=TOL
294 + assert_score(
295 + rerank.results[0].relevance_score,
296 + TEXT_VS_TEXT,
297 + backend,
298 + "rerank[0]_text_vs_text",
299 + )
300 + assert_score(
301 + rerank.results[1].relevance_score,
302 + TEXT_VS_TEXT,
303 + backend,
304 + "rerank[1]_text_vs_text",
305 + )
306 + assert_score(
307 + rerank.results[2].relevance_score,
308 + TEXT_VS_IMAGE,
309 + backend,
310 + "rerank[2]_text_vs_image",
311 + )
312 + assert_score(
313 + rerank.results[3].relevance_score,
314 + TEXT_VS_TEXT_PLUS_IMAGE,
315 + backend,
316 + "rerank[3]_text_vs_text_plus_image",
317 )
318
319
215 -def test_score_api_queries_list_documents_list(server: RemoteOpenAIServer):
320 +def test_score_api_queries_list_documents_list(
321 + server: tuple[RemoteOpenAIServer, str],
322 +):
323 + remote_server, backend = server
324 score_response = requests.post(
217 - server.url_for("score"),
325 + remote_server.url_for("score"),
326 json={
327 "model": MODEL_NAME,
328 "queries": [query] * 4,
@@ -233,7 +341,12 @@ def test_score_api_queries_list_documents_list(server: RemoteOpenAIServer):
341 assert score.data is not None
342 assert len(score.data) == 4
343 assert score.usage.prompt_tokens == 368
236 - assert score.data[0].score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
237 - assert score.data[1].score == pytest.approx(TEXT_VS_TEXT, rel=TOL)
238 - assert score.data[2].score == pytest.approx(TEXT_VS_IMAGE, rel=TOL)
239 - assert score.data[3].score == pytest.approx(TEXT_VS_TEXT_PLUS_IMAGE, rel=TOL)
344 + assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "paired[0]_text_vs_text")
345 + assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "paired[1]_text_vs_text")
346 + assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "paired[2]_text_vs_image")
347 + assert_score(
348 + score.data[3].score,
349 + TEXT_VS_TEXT_PLUS_IMAGE,
350 + backend,
351 + "paired[3]_text_vs_text_plus_image",
352 + )