A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 01:30 UTC
[CI] fix flaky empty responses and add diagnostic assertions in vision chat tests (#36341)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Mar 8, 2026 at 00:42 UTC
40077ea3defdf2b0997245ca8999097eede2308f
2 files changed
+317
-192
tests/entrypoints/openai/test_transcription_validation.py
+73
-33
@@ -6,7 +6,7 @@ import json
6
7
import pytest
8
9
-from ...utils import RemoteOpenAIServer
9
+from ...utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
10
from .conftest import add_attention_backend
11
12
MISTRAL_FORMAT_ARGS = [
@@ -19,12 +19,55 @@ MISTRAL_FORMAT_ARGS = [
19
]
20
21
22
+async def transcribe_and_check(
23
+ client,
24
+ model_name: str,
25
+ file,
26
+ *,
27
+ language: str,
28
+ expected_text: str,
29
+ expected_seconds: int | None = None,
30
+ case_sensitive: bool = False,
31
+):
32
+ """Run a transcription request and assert the output contains
33
+ *expected_text* and optionally that usage reports *expected_seconds*.
34
+
35
+ Provides detailed failure messages with the actual transcription output.
36
+ """
37
+ transcription = await client.audio.transcriptions.create(
38
+ model=model_name,
39
+ file=file,
40
+ language=language,
41
+ response_format="text",
42
+ temperature=0.0,
43
+ )
44
+ out = json.loads(transcription)
45
+ out_text = out["text"]
46
+ out_usage = out["usage"]
47
+
48
+ if case_sensitive:
49
+ assert expected_text in out_text, (
50
+ f"Expected {expected_text!r} in transcription output, got: {out_text!r}"
51
+ )
52
+ else:
53
+ assert expected_text.lower() in out_text.lower(), (
54
+ f"Expected {expected_text!r} (case-insensitive) in transcription "
55
+ f"output, got: {out_text!r}"
56
+ )
57
+
58
+ if expected_seconds is not None:
59
+ assert out_usage["seconds"] == expected_seconds, (
60
+ f"Expected {expected_seconds}s of audio, "
61
+ f"got {out_usage['seconds']}s. Full usage: {out_usage!r}"
62
+ )
63
+
64
+
65
@pytest.mark.asyncio
66
@pytest.mark.parametrize(
67
"model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]
68
)
69
async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
27
- server_args = ["--enforce-eager"]
70
+ server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
71
72
if model_name.startswith("mistralai"):
73
server_args += MISTRAL_FORMAT_ARGS
@@ -32,20 +75,18 @@ async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
75
add_attention_backend(server_args, rocm_aiter_fa_attention)
76
77
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
35
- with RemoteOpenAIServer(model_name, server_args) as remote_server:
78
+ with RemoteOpenAIServer(
79
+ model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
80
+ ) as remote_server:
81
client = remote_server.get_async_client()
37
- transcription = await client.audio.transcriptions.create(
38
- model=model_name,
39
- file=mary_had_lamb,
82
+ await transcribe_and_check(
83
+ client,
84
+ model_name,
85
+ mary_had_lamb,
86
language="en",
41
- response_format="text",
42
- temperature=0.0,
87
+ expected_text="Mary had a little lamb",
88
+ expected_seconds=16,
89
)
44
- out = json.loads(transcription)
45
- out_text = out["text"]
46
- out_usage = out["usage"]
47
- assert "Mary had a little lamb" in out_text
48
- assert out_usage["seconds"] == 16, out_usage["seconds"]
90
91
92
@pytest.mark.asyncio
@@ -74,20 +115,18 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
115
add_attention_backend(server_args, rocm_aiter_fa_attention)
116
117
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
77
- with RemoteOpenAIServer(model_name, server_args) as remote_server:
118
+ with RemoteOpenAIServer(
119
+ model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
120
+ ) as remote_server:
121
client = remote_server.get_async_client()
79
- transcription = await client.audio.transcriptions.create(
80
- model=lora_model_name,
81
- file=mary_had_lamb,
122
+ await transcribe_and_check(
123
+ client,
124
+ lora_model_name,
125
+ mary_had_lamb,
126
language="en",
83
- response_format="text",
84
- temperature=0.0,
127
+ expected_text="mary had a little lamb",
128
+ expected_seconds=16,
129
)
86
- out = json.loads(transcription)
87
- out_text = out["text"]
88
- out_usage = out["usage"]
89
- assert "mary had a little lamb" in out_text
90
- assert out_usage["seconds"] == 16, out_usage["seconds"]
130
131
132
@pytest.mark.asyncio
@@ -97,20 +136,21 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
136
async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):
137
# Gemma accuracy on some of the audio samples we use is particularly bad,
138
# hence we use a different one here. WER is evaluated separately.
100
- server_args = ["--enforce-eager"]
139
+ server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
140
141
add_attention_backend(server_args, rocm_aiter_fa_attention)
142
143
with RemoteOpenAIServer(
105
- model_name, server_args, max_wait_seconds=480
144
+ model_name,
145
+ server_args,
146
+ max_wait_seconds=480,
147
+ env_dict=ROCM_ENV_OVERRIDES,
148
) as remote_server:
149
client = remote_server.get_async_client()
108
- transcription = await client.audio.transcriptions.create(
109
- model=model_name,
110
- file=foscolo,
150
+ await transcribe_and_check(
151
+ client,
152
+ model_name,
153
+ foscolo,
154
language="it",
112
- response_format="text",
113
- temperature=0.0,
155
+ expected_text="ove il mio corpo fanciulletto giacque",
156
)
115
- out = json.loads(transcription)["text"]
116
- assert "ove il mio corpo fanciulletto giacque" in out
tests/entrypoints/openai/test_vision.py
+244
-159
@@ -12,7 +12,7 @@ from vllm.multimodal.media import MediaWithBytes
12
from vllm.multimodal.utils import encode_image_url, fetch_image
13
from vllm.platforms import current_platform
14
15
-from ...utils import RemoteOpenAIServer
15
+from ...utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
16
17
MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
18
MAXIMUM_IMAGES = 2
@@ -48,10 +48,37 @@ def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bo
48
All term groups must be satisfied.
49
"""
50
content_lower = content.lower()
51
- for group in term_groups:
52
- if not any(term.lower() in content_lower for term in group):
53
- return False
54
- return True
51
+ return all(
52
+ any(term.lower() in content_lower for term in group) for group in term_groups
53
+ )
54
+
55
+
56
+def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
57
+ """Assert the first choice has non-empty string content; return it.
58
+
59
+ Provides a detailed failure message including the full ChatCompletion
60
+ response so flaky / model-quality issues are easy to diagnose.
61
+ """
62
+ prefix = f"[{context}] " if context else ""
63
+ choice = chat_completion.choices[0]
64
+ content = choice.message.content
65
+
66
+ assert content is not None, (
67
+ f"{prefix}Expected non-None content but got None. "
68
+ f"finish_reason={choice.finish_reason!r}, "
69
+ f"full message={choice.message!r}, "
70
+ f"usage={chat_completion.usage!r}"
71
+ )
72
+ assert isinstance(content, str), (
73
+ f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
74
+ )
75
+ assert len(content) > 0, (
76
+ f"{prefix}Expected non-empty content but got empty string. "
77
+ f"finish_reason={choice.finish_reason!r}, "
78
+ f"full message={choice.message!r}, "
79
+ f"usage={chat_completion.usage!r}"
80
+ )
81
+ return content
82
83
84
@pytest.fixture(scope="module")
@@ -67,16 +94,22 @@ def server():
94
"--trust-remote-code",
95
"--limit-mm-per-prompt",
96
json.dumps({"image": MAXIMUM_IMAGES}),
97
+ *ROCM_EXTRA_ARGS,
98
]
99
100
# ROCm: Increase timeouts to handle potential network delays and slower
101
# video processing when downloading multiple videos from external sources
74
- env_overrides = {}
75
- if current_platform.is_rocm():
76
- env_overrides = {
77
- "VLLM_VIDEO_FETCH_TIMEOUT": "120",
78
- "VLLM_ENGINE_ITERATION_TIMEOUT_S": "300",
79
- }
102
+ env_overrides = {
103
+ **ROCM_ENV_OVERRIDES,
104
+ **(
105
+ {
106
+ "VLLM_VIDEO_FETCH_TIMEOUT": "120",
107
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": "300",
108
+ }
109
+ if current_platform.is_rocm()
110
+ else {}
111
+ ),
112
+ }
113
114
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
115
yield remote_server
@@ -117,6 +150,51 @@ def dummy_messages_from_image_url(
150
]
151
152
153
+def describe_image_messages(
154
+ image_url: str, *, extra_image_fields: dict | None = None
155
+) -> list[dict]:
156
+ """Build the system + user messages used by the completions-with-image
157
+ family of tests. *extra_image_fields* is merged into the top-level
158
+ image content block (for uuid / bad-key tests)."""
159
+ image_block: dict = {
160
+ "type": "image_url",
161
+ "image_url": {"url": image_url},
162
+ }
163
+ if extra_image_fields:
164
+ image_block.update(extra_image_fields)
165
+
166
+ return [
167
+ {"role": "system", "content": "You are a helpful assistant."},
168
+ {
169
+ "role": "user",
170
+ "content": [
171
+ {"type": "text", "text": "Describe this image."},
172
+ image_block,
173
+ ],
174
+ },
175
+ ]
176
+
177
+
178
+async def complete_and_check(
179
+ client: openai.AsyncOpenAI,
180
+ model_name: str,
181
+ messages: list[dict],
182
+ *,
183
+ context: str,
184
+ max_completion_tokens: int = 50,
185
+ temperature: float = 0.0,
186
+) -> str:
187
+ """Run a chat completion and assert the output is non-empty.
188
+ Returns the content string."""
189
+ chat_completion = await client.chat.completions.create(
190
+ model=model_name,
191
+ messages=messages,
192
+ max_completion_tokens=max_completion_tokens,
193
+ temperature=temperature,
194
+ )
195
+ return assert_non_empty_content(chat_completion, context=context)
196
+
197
+
198
def get_hf_prompt_tokens(model_name, content, image_url):
199
processor = AutoProcessor.from_pretrained(
200
model_name, trust_remote_code=True, num_crops=4
@@ -153,7 +231,6 @@ async def test_single_chat_session_image(
231
messages = dummy_messages_from_image_url(image_url, content_text)
232
233
max_completion_tokens = 10
156
- # test single completion
234
chat_completion = await client.chat.completions.create(
235
model=model_name,
236
messages=messages,
@@ -162,32 +239,46 @@ async def test_single_chat_session_image(
239
temperature=0.0,
240
top_logprobs=5,
241
)
165
- assert len(chat_completion.choices) == 1
242
+ assert len(chat_completion.choices) == 1, (
243
+ f"Expected 1 choice, got {len(chat_completion.choices)}"
244
+ )
245
246
choice = chat_completion.choices[0]
168
- assert choice.finish_reason == "length"
247
+ assert choice.finish_reason == "length", (
248
+ f"Expected finish_reason='length' (capped at {max_completion_tokens} "
249
+ f"tokens), got {choice.finish_reason!r}. "
250
+ f"content={choice.message.content!r}"
251
+ )
252
+
253
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
170
- assert chat_completion.usage == openai.types.CompletionUsage(
254
+ expected_usage = openai.types.CompletionUsage(
255
completion_tokens=max_completion_tokens,
256
prompt_tokens=hf_prompt_tokens,
257
total_tokens=hf_prompt_tokens + max_completion_tokens,
258
)
259
+ assert chat_completion.usage == expected_usage, (
260
+ f"Usage mismatch: got {chat_completion.usage!r}, expected {expected_usage!r}"
261
+ )
262
263
message = choice.message
177
- message = chat_completion.choices[0].message
178
- assert message.content is not None and len(message.content) >= 10
179
- assert message.role == "assistant"
264
+ assert message.content is not None and len(message.content) >= 10, (
265
+ f"Expected content with >=10 chars, got {message.content!r}"
266
+ )
267
+ assert message.role == "assistant", (
268
+ f"Expected role='assistant', got {message.role!r}"
269
+ )
270
+
271
messages.append({"role": "assistant", "content": message.content})
272
273
# test multi-turn dialogue
274
messages.append({"role": "user", "content": "express your result in json"})
184
- chat_completion = await client.chat.completions.create(
185
- model=model_name,
186
- messages=messages,
275
+ await complete_and_check(
276
+ client,
277
+ model_name,
278
+ messages,
279
+ context=f"multi-turn follow-up for {image_url}",
280
max_completion_tokens=10,
281
)
189
- message = chat_completion.choices[0].message
190
- assert message.content is not None and len(message.content) >= 0
282
283
284
@pytest.mark.asyncio
@@ -209,7 +300,7 @@ async def test_error_on_invalid_image_url_type(
300
301
# image_url should be a dict {"url": "some url"}, not directly a string
302
with pytest.raises(openai.BadRequestError):
212
- _ = await client.chat.completions.create(
303
+ await client.chat.completions.create(
304
model=model_name,
305
messages=messages,
306
max_completion_tokens=10,
@@ -235,10 +326,15 @@ async def test_single_chat_session_image_beamsearch(
326
top_logprobs=5,
327
extra_body=dict(use_beam_search=True),
328
)
238
- assert len(chat_completion.choices) == 2
239
- assert (
240
- chat_completion.choices[0].message.content
241
- != chat_completion.choices[1].message.content
329
+ assert len(chat_completion.choices) == 2, (
330
+ f"Expected 2 beam search choices, got {len(chat_completion.choices)}"
331
+ )
332
+
333
+ content_0 = chat_completion.choices[0].message.content
334
+ content_1 = chat_completion.choices[1].message.content
335
+ assert content_0 != content_1, (
336
+ f"Beam search should produce different outputs for {image_url}, "
337
+ f"but both returned: {content_0!r}"
338
)
339
340
@@ -269,33 +365,46 @@ async def test_single_chat_session_image_base64encoded(
365
temperature=0.0,
366
top_logprobs=5,
367
)
272
- assert len(chat_completion.choices) == 1
368
+ assert len(chat_completion.choices) == 1, (
369
+ f"Expected 1 choice, got {len(chat_completion.choices)}"
370
+ )
371
372
choice = chat_completion.choices[0]
275
- assert choice.finish_reason == "length"
373
+ assert choice.finish_reason == "length", (
374
+ f"Expected finish_reason='length', got {choice.finish_reason!r}. "
375
+ f"content={choice.message.content!r}"
376
+ )
377
+
378
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
277
- assert chat_completion.usage == openai.types.CompletionUsage(
379
+ expected_usage = openai.types.CompletionUsage(
380
completion_tokens=max_completion_tokens,
381
prompt_tokens=hf_prompt_tokens,
382
total_tokens=hf_prompt_tokens + max_completion_tokens,
383
)
384
+ assert chat_completion.usage == expected_usage, (
385
+ f"Usage mismatch: got {chat_completion.usage!r}, expected {expected_usage!r}"
386
+ )
387
388
message = choice.message
284
- message = chat_completion.choices[0].message
285
- assert message.content is not None and len(message.content) >= 10
286
- assert message.role == "assistant"
389
+ assert message.content is not None and len(message.content) >= 10, (
390
+ f"Expected content with >=10 chars, got {message.content!r}"
391
+ )
392
+ assert message.role == "assistant", (
393
+ f"Expected role='assistant', got {message.role!r}"
394
+ )
395
+
396
messages.append({"role": "assistant", "content": message.content})
397
398
# test multi-turn dialogue
399
messages.append({"role": "user", "content": "express your result in json"})
291
- chat_completion = await client.chat.completions.create(
292
- model=model_name,
293
- messages=messages,
400
+ await complete_and_check(
401
+ client,
402
+ model_name,
403
+ messages,
404
+ context=f"multi-turn base64 follow-up for {raw_image_url}",
405
max_completion_tokens=10,
406
temperature=0.0,
407
)
297
- message = chat_completion.choices[0].message
298
- assert message.content is not None and len(message.content) >= 0
408
409
410
@pytest.mark.asyncio
@@ -321,7 +430,10 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
430
temperature=0.0,
431
extra_body=dict(use_beam_search=True),
432
)
324
- assert len(chat_completion.choices) == 2
433
+ assert len(chat_completion.choices) == 2, (
434
+ f"Expected 2 beam search choices for image {image_idx} "
435
+ f"({raw_image_url}), got {len(chat_completion.choices)}"
436
+ )
437
438
# Verify beam search produces two different non-empty outputs
439
content_0 = chat_completion.choices[0].message.content
@@ -333,18 +445,28 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
445
f"Output 0: {content_0!r}, Output 1: {content_1!r}"
446
)
447
336
- assert content_0, "First beam search output should not be empty"
337
- assert content_1, "Second beam search output should not be empty"
338
- assert content_0 != content_1, "Beam search should produce different outputs"
448
+ assert content_0, (
449
+ f"First beam output is empty for image {image_idx} ({raw_image_url}). "
450
+ f"finish_reason={chat_completion.choices[0].finish_reason!r}"
451
+ )
452
+ assert content_1, (
453
+ f"Second beam output is empty for image {image_idx} "
454
+ f"({raw_image_url}). "
455
+ f"finish_reason={chat_completion.choices[1].finish_reason!r}"
456
+ )
457
+ assert content_0 != content_1, (
458
+ f"Beam search produced identical outputs for image {image_idx} "
459
+ f"({raw_image_url}): {content_0!r}"
460
+ )
461
462
# Verify each output contains the required terms for this image
463
for i, content in enumerate([content_0, content_1]):
342
- if not check_output_matches_terms(content, required_terms):
343
- pytest.fail(
344
- f"Output {i} '{content}' doesn't contain required terms. "
345
- f"Expected all of these term groups (at least one from each): "
346
- f"{required_terms}"
347
- )
464
+ assert check_output_matches_terms(content, required_terms), (
465
+ f"Beam output {i} for image {image_idx} ({raw_image_url}) "
466
+ f"doesn't match required terms.\n"
467
+ f" content: {content!r}\n"
468
+ f" required (all groups, >=1 per group): {required_terms}"
469
+ )
470
471
472
@pytest.mark.asyncio
@@ -378,16 +500,29 @@ async def test_chat_streaming_image(
500
async for chunk in stream:
501
delta = chunk.choices[0].delta
502
if delta.role:
381
- assert delta.role == "assistant"
503
+ assert delta.role == "assistant", (
504
+ f"Expected role='assistant' in stream delta, got {delta.role!r}"
505
+ )
506
if delta.content:
507
chunks.append(delta.content)
508
if chunk.choices[0].finish_reason is not None:
509
finish_reason_count += 1
510
# finish reason should only return in last block
387
- assert finish_reason_count == 1
388
- assert chunk.choices[0].finish_reason == stop_reason
389
- assert delta.content
390
- assert "".join(chunks) == output
511
+ assert finish_reason_count == 1, (
512
+ f"Expected exactly 1 finish_reason across stream chunks, "
513
+ f"got {finish_reason_count}"
514
+ )
515
+ assert chunk.choices[0].finish_reason == stop_reason, (
516
+ f"Stream finish_reason={chunk.choices[0].finish_reason!r} "
517
+ f"doesn't match non-stream finish_reason={stop_reason!r}"
518
+ )
519
+
520
+ streamed_text = "".join(chunks)
521
+ assert streamed_text == output, (
522
+ f"Streamed output doesn't match non-streamed for {image_url}.\n"
523
+ f" streamed: {streamed_text!r}\n"
524
+ f" non-streamed: {output!r}"
525
+ )
526
527
528
@pytest.mark.asyncio
@@ -418,17 +553,19 @@ async def test_multi_image_input(
553
max_tokens=5,
554
temperature=0.0,
555
)
421
- completion = completion.choices[0].text
422
- assert completion is not None and len(completion) >= 0
556
+ assert completion.choices[0].text is not None, (
557
+ "Server failed to produce output after rejecting over-limit "
558
+ "multi-image request"
559
+ )
560
else:
424
- chat_completion = await client.chat.completions.create(
425
- model=model_name,
426
- messages=messages,
561
+ await complete_and_check(
562
+ client,
563
+ model_name,
564
+ messages,
565
+ context=f"multi-image input ({len(image_urls)} images)",
566
max_completion_tokens=10,
567
temperature=0.0,
568
)
430
- message = chat_completion.choices[0].message
431
- assert message.content is not None and len(message.content) >= 0
569
570
571
@pytest.mark.asyncio
@@ -444,30 +581,13 @@ async def test_completions_with_image(
581
image_urls: list[str],
582
):
583
for image_url in image_urls:
447
- chat_completion = await client.chat.completions.create(
448
- messages=[
449
- {"role": "system", "content": "You are a helpful assistant."},
450
- {
451
- "role": "user",
452
- "content": [
453
- {
454
- "type": "text",
455
- "text": "Describe this image.",
456
- },
457
- {
458
- "type": "image_url",
459
- "image_url": {
460
- "url": image_url,
461
- },
462
- },
463
- ],
464
- },
465
- ],
466
- model=model_name,
584
+ messages = describe_image_messages(image_url)
585
+ await complete_and_check(
586
+ client,
587
+ model_name,
588
+ messages,
589
+ context=f"completions_with_image url={image_url}",
590
)
468
- assert chat_completion.choices[0].message.content is not None
469
- assert isinstance(chat_completion.choices[0].message.content, str)
470
- assert len(chat_completion.choices[0].message.content) > 0
591
592
593
@pytest.mark.asyncio
@@ -483,54 +603,33 @@ async def test_completions_with_image_with_uuid(
603
image_urls: list[str],
604
):
605
for image_url in image_urls:
486
- chat_completion = await client.chat.completions.create(
487
- messages=[
488
- {"role": "system", "content": "You are a helpful assistant."},
489
- {
490
- "role": "user",
491
- "content": [
492
- {
493
- "type": "text",
494
- "text": "Describe this image.",
495
- },
496
- {
497
- "type": "image_url",
498
- "image_url": {
499
- "url": image_url,
500
- },
501
- "uuid": image_url,
502
- },
503
- ],
504
- },
505
- ],
506
- model=model_name,
606
+ messages = describe_image_messages(
607
+ image_url,
608
+ extra_image_fields={"uuid": image_url},
609
)
508
- assert chat_completion.choices[0].message.content is not None
509
- assert isinstance(chat_completion.choices[0].message.content, str)
510
- assert len(chat_completion.choices[0].message.content) > 0
511
-
512
- # Second request, with empty image but the same uuid.
513
- chat_completion_with_empty_image = await client.chat.completions.create(
514
- messages=[
515
- {"role": "system", "content": "You are a helpful assistant."},
516
- {
517
- "role": "user",
518
- "content": [
519
- {
520
- "type": "text",
521
- "text": "Describe this image.",
522
- },
523
- {"type": "image_url", "image_url": {}, "uuid": image_url},
524
- ],
525
- },
526
- ],
527
- model=model_name,
610
+ await complete_and_check(
611
+ client,
612
+ model_name,
613
+ messages,
614
+ context=f"uuid first request url={image_url}",
615
)
529
- assert chat_completion_with_empty_image.choices[0].message.content is not None
530
- assert isinstance(
531
- chat_completion_with_empty_image.choices[0].message.content, str
616
+
617
+ cached_messages: list[dict] = [
618
+ {"role": "system", "content": "You are a helpful assistant."},
619
+ {
620
+ "role": "user",
621
+ "content": [
622
+ {"type": "text", "text": "Describe this image."},
623
+ {"type": "image_url", "image_url": {}, "uuid": image_url},
624
+ ],
625
+ },
626
+ ]
627
+ await complete_and_check(
628
+ client,
629
+ model_name,
630
+ cached_messages,
631
+ context=f"uuid cached (empty image) uuid={image_url}",
632
)
533
- assert len(chat_completion_with_empty_image.choices[0].message.content) > 0
633
634
635
@pytest.mark.asyncio
@@ -540,16 +639,13 @@ async def test_completions_with_empty_image_with_uuid_without_cache_hit(
639
model_name: str,
640
):
641
with pytest.raises(openai.BadRequestError):
543
- _ = await client.chat.completions.create(
642
+ await client.chat.completions.create(
643
messages=[
644
{"role": "system", "content": "You are a helpful assistant."},
645
{
646
"role": "user",
647
"content": [
549
- {
550
- "type": "text",
551
- "text": "Describe this image.",
552
- },
648
+ {"type": "text", "text": "Describe this image."},
649
{
650
"type": "image_url",
651
"image_url": {},
@@ -575,29 +671,18 @@ async def test_completions_with_image_with_incorrect_uuid_format(
671
image_urls: list[str],
672
):
673
for image_url in image_urls:
578
- chat_completion = await client.chat.completions.create(
579
- messages=[
580
- {"role": "system", "content": "You are a helpful assistant."},
581
- {
582
- "role": "user",
583
- "content": [
584
- {
585
- "type": "text",
586
- "text": "Describe this image.",
587
- },
588
- {
589
- "type": "image_url",
590
- "image_url": {
591
- "url": image_url,
592
- "incorrect_uuid_key": image_url,
593
- },
594
- "also_incorrect_uuid_key": image_url,
595
- },
596
- ],
597
- },
598
- ],
599
- model=model_name,
674
+ messages = describe_image_messages(
675
+ image_url,
676
+ extra_image_fields={
677
+ "also_incorrect_uuid_key": image_url,
678
+ },
679
+ )
680
+ # Inject the bad key inside image_url dict too
681
+ messages[1]["content"][1]["image_url"]["incorrect_uuid_key"] = image_url
682
+
683
+ await complete_and_check(
684
+ client,
685
+ model_name,
686
+ messages,
687
+ context=f"incorrect uuid format url={image_url}",
688
)
601
- assert chat_completion.choices[0].message.content is not None
602
- assert isinstance(chat_completion.choices[0].message.content, str)
603
- assert len(chat_completion.choices[0].message.content) > 0