[Feature][MM_UUIDs] Allow empty video URLs when using multi-modal UUIDs (#54220)
Signed-off-by: Jan Kwiecinski <jkwiecinski@nvidia.com>
Jan Kwieciński committed
Sep 1, 2026 at 07:55 UTC
ec32f669bb557e85d3f05a814d4e4a68f7e3b634
4 files changed
+146
-1
docs/features/multimodal_inputs.md
+7
@@ -1366,4 +1366,11 @@ Just like with offline inference, you can skip sending media if you expect cache
1366
"uuid": image_uuid,
1367
},
1368
1369
+ # video_url:
1370
+ {
1371
+ "type": "video_url",
1372
+ "video_url": {},
1373
+ "uuid": video_uuid,
1374
+ },
1375
+
1376
```
tests/entrypoints/multimodal/openai/chat_completion/test_video.py
+112
@@ -66,6 +66,77 @@ def url_encoded_video() -> dict[str, str]:
66
}
67
68
69
+def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
70
+ """Assert the first choice has non-empty string content; return it."""
71
+ prefix = f"[{context}] " if context else ""
72
+ choice = chat_completion.choices[0]
73
+ content = choice.message.content
74
+
75
+ assert content is not None, (
76
+ f"{prefix}Expected non-None content but got None. "
77
+ f"finish_reason={choice.finish_reason!r}, "
78
+ f"full message={choice.message!r}, "
79
+ f"usage={chat_completion.usage!r}"
80
+ )
81
+ assert isinstance(content, str), (
82
+ f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
83
+ )
84
+ assert len(content) > 0, (
85
+ f"{prefix}Expected non-empty content but got empty string. "
86
+ f"finish_reason={choice.finish_reason!r}, "
87
+ f"full message={choice.message!r}, "
88
+ f"usage={chat_completion.usage!r}"
89
+ )
90
+ return content
91
+
92
+
93
+def describe_video_messages(
94
+ video_url: str | None, *, extra_video_fields: dict | None = None
95
+) -> list[dict]:
96
+ """Build the system + user messages used by the completions-with-video
97
+ family of tests. *extra_video_fields* is merged into the top-level
98
+ video content block (for uuid / bad-key tests).
99
+
100
+ Pass ``None`` for *video_url* to omit media (uuid-backed cache hits).
101
+ """
102
+ video_block: dict = {
103
+ "type": "video_url",
104
+ "video_url": {} if video_url is None else {"url": video_url},
105
+ }
106
+ if extra_video_fields:
107
+ video_block.update(extra_video_fields)
108
+
109
+ return [
110
+ {"role": "system", "content": "You are a helpful assistant."},
111
+ {
112
+ "role": "user",
113
+ "content": [
114
+ {"type": "text", "text": "Describe this video."},
115
+ video_block,
116
+ ],
117
+ },
118
+ ]
119
+
120
+
121
+async def complete_and_check(
122
+ client: openai.AsyncOpenAI,
123
+ model_name: str,
124
+ messages: list[dict],
125
+ *,
126
+ context: str,
127
+ max_completion_tokens: int = 50,
128
+ temperature: float = 0.0,
129
+) -> str:
130
+ """Run a chat completion and assert the output is non-empty."""
131
+ chat_completion = await client.chat.completions.create(
132
+ model=model_name,
133
+ messages=messages,
134
+ max_completion_tokens=max_completion_tokens,
135
+ temperature=temperature,
136
+ )
137
+ return assert_non_empty_content(chat_completion, context=context)
138
+
139
+
140
def dummy_messages_from_video_url(
141
video_urls: str | list[str],
142
content_text: str = "What's in this video?",
@@ -402,3 +473,44 @@ async def test_multi_video_input(
473
)
474
message = chat_completion.choices[0].message
475
assert message.content is not None and len(message.content) >= 0
476
+
477
+
478
+@pytest.mark.asyncio
479
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
480
+async def test_omitted_video_url_requires_cached_uuid(
481
+ client: openai.AsyncOpenAI,
482
+ model_name: str,
483
+):
484
+ video_uuid = "test-video-uuid"
485
+ video_url = TEST_VIDEO_URLS[0]
486
+
487
+ omitted_video_messages = describe_video_messages(
488
+ None,
489
+ extra_video_fields={"uuid": video_uuid},
490
+ )
491
+
492
+ with pytest.raises(
493
+ openai.BadRequestError,
494
+ match="Cache miss for video at index 0 but data is not provided",
495
+ ):
496
+ await client.chat.completions.create(
497
+ model=model_name,
498
+ messages=omitted_video_messages,
499
+ )
500
+
501
+ await complete_and_check(
502
+ client,
503
+ model_name,
504
+ describe_video_messages(
505
+ video_url,
506
+ extra_video_fields={"uuid": video_uuid},
507
+ ),
508
+ context="populate UUID cache",
509
+ )
510
+
511
+ await complete_and_check(
512
+ client,
513
+ model_name,
514
+ omitted_video_messages,
515
+ context="reuse cached video without URL",
516
+ )
tests/multimodal/test_parse.py
+20
-1
@@ -5,7 +5,11 @@ import pytest
5
import torch
6
from PIL import Image
7
8
-from vllm.multimodal.parse import ImageProcessorItems, VideoProcessorItems
8
+from vllm.multimodal.parse import (
9
+ ImageProcessorItems,
10
+ MultiModalDataParser,
11
+ VideoProcessorItems,
12
+)
13
14
H, W = 480, 640
15
@@ -49,3 +53,18 @@ def test_frame_size_hwc_chw(frame):
53
items = VideoProcessorItems([[frame]])
54
55
assert items.get_frame_size(0) == (W, H)
56
+
57
+
58
+@pytest.mark.parametrize(
59
+ "modality,processor_cls",
60
+ [
61
+ ("image", ImageProcessorItems),
62
+ ("video", VideoProcessorItems),
63
+ ],
64
+)
65
+def test_parse_mm_data_accepts_none_cached_item(modality, processor_cls):
66
+ mm_items = MultiModalDataParser().parse_mm_data({modality: [None]})
67
+ items = mm_items[modality]
68
+ assert isinstance(items, processor_cls)
69
+ assert len(items) == 1
70
+ assert items.get(0) is None
vllm/multimodal/parse.py
+7
@@ -756,9 +756,16 @@ class MultiModalDataParser:
756
np.ndarray
757
| MediaWithBytes[np.ndarray]
758
| tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
759
+ | None
760
]()
761
metadata_lst: list[dict[str, Any] | None] = []
762
for data_item in data_items:
763
+ # Allow None video items, valid requests can contain empty URLs
764
+ # if they use multi-modal uuids.
765
+ if data_item is None:
766
+ new_videos.append(None)
767
+ metadata_lst.append(None)
768
+ continue
769
video, metadata = self._get_video_with_metadata(data_item)
770
if self.video_needs_metadata:
771
if metadata is None: