A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[CI] Fix flaky vision beam search test with flexible semantic validation (#31324)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Dec 25, 2025 at 22:39 UTC
c79dbfa9ad2f771c7122e5aa80005f9f03ee6864
1 file changed
+50
-48
tests/entrypoints/openai/test_vision.py
+50
-48
@@ -25,45 +25,33 @@ TEST_IMAGE_ASSETS = [
25
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
26
]
27
28
-EXPECTED_MM_BEAM_SEARCH_RES = [
29
- [
30
- "The image shows a wooden boardwalk leading through a",
31
- "The image shows a wooden boardwalk extending into a",
32
- ],
33
- [
34
- "The image shows two parrots perched on",
35
- "The image shows two birds perched on a cur",
36
- ],
37
- [
38
- "The image shows a Venn diagram with three over",
39
- "The image displays a Venn diagram with three over",
40
- ],
41
- [
42
- "This image displays a gradient of colors ranging from",
43
- "This image displays a gradient of colors forming a spectrum",
44
- ],
28
+# Required terms for beam search validation
29
+# Each entry is a list of term groups - ALL groups must match
30
+# Each group is a list of alternatives - at least ONE term in the group must appear
31
+# This provides semantic validation while allowing wording variation
32
+REQUIRED_BEAM_SEARCH_TERMS = [
33
+ # Boardwalk image: must have "boardwalk" AND ("wooden" or "wood")
34
+ [["boardwalk"], ["wooden", "wood"]],
35
+ # Parrots image: must have ("parrot" or "bird") AND "two"
36
+ [["parrot", "bird"], ["two"]],
37
+ # Venn diagram: must have "venn" AND "diagram"
38
+ [["venn"], ["diagram"]],
39
+ # Gradient image: must have "gradient" AND ("color" or "spectrum")
40
+ [["gradient"], ["color", "spectrum"]],
41
]
42
47
-EXPECTED_MM_BEAM_SEARCH_RES_ROCM = [
48
- # MultiHeadAttention attn_backend: FLASH_ATTN
49
- # with Triton Attention backend
50
- [
51
- "The image shows a wooden boardwalk leading through a",
52
- "The image shows a wooden boardwalk extending into a",
53
- ],
54
- [
55
- "The image shows two parrots perched on",
56
- "The image shows two birds perched on a cur",
57
- ],
58
- [
59
- "The image shows a Venn diagram with three over",
60
- "The image contains a Venn diagram with three over",
61
- ],
62
- [
63
- "This image displays a gradient of colors ranging from",
64
- "This image displays a gradient of colors transitioning from",
65
- ],
66
-]
43
+
44
+def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bool:
45
+ """
46
+ Check if content matches all required term groups.
47
+ Each term group requires at least one of its terms to be present.
48
+ All term groups must be satisfied.
49
+ """
50
+ content_lower = content.lower()
51
+ for group in term_groups:
52
+ if not any(term.lower() in content_lower for term in group):
53
+ return False
54
+ return True
55
56
57
@pytest.fixture(scope="module")
@@ -319,16 +307,9 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
307
image_idx: int,
308
url_encoded_image: dict[str, str],
309
):
322
- # ROCm: Switch expected results based on platform
323
- from vllm.platforms import current_platform
324
-
325
- # NOTE: This test also validates that we pass MM data through beam search
310
+ # NOTE: This test validates that we pass MM data through beam search
311
raw_image_url = TEST_IMAGE_ASSETS[image_idx]
327
-
328
- if current_platform.is_rocm():
329
- expected_res = EXPECTED_MM_BEAM_SEARCH_RES_ROCM[image_idx]
330
- else:
331
- expected_res = EXPECTED_MM_BEAM_SEARCH_RES[image_idx]
312
+ required_terms = REQUIRED_BEAM_SEARCH_TERMS[image_idx]
313
314
messages = dummy_messages_from_image_url(url_encoded_image[raw_image_url])
315
@@ -341,8 +322,29 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
322
extra_body=dict(use_beam_search=True),
323
)
324
assert len(chat_completion.choices) == 2
344
- for actual, expected_str in zip(chat_completion.choices, expected_res):
345
- assert actual.message.content == expected_str
325
+
326
+ # Verify beam search produces two different non-empty outputs
327
+ content_0 = chat_completion.choices[0].message.content
328
+ content_1 = chat_completion.choices[1].message.content
329
+
330
+ # Emit beam search outputs for debugging
331
+ print(
332
+ f"Beam search outputs for image {image_idx} ({raw_image_url}): "
333
+ f"Output 0: {content_0!r}, Output 1: {content_1!r}"
334
+ )
335
+
336
+ assert content_0, "First beam search output should not be empty"
337
+ assert content_1, "Second beam search output should not be empty"
338
+ assert content_0 != content_1, "Beam search should produce different outputs"
339
+
340
+ # Verify each output contains the required terms for this image
341
+ for i, content in enumerate([content_0, content_1]):
342
+ if not check_output_matches_terms(content, required_terms):
343
+ pytest.fail(
344
+ f"Output {i} '{content}' doesn't contain required terms. "
345
+ f"Expected all of these term groups (at least one from each): "
346
+ f"{required_terms}"
347
+ )
348
349
350
@pytest.mark.asyncio