A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[V1] Support interleaved modality items (#15605)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Mar 29, 2025 at 06:30 UTC
c67abd614fe670b1cc771097658dd7efe4a33747
6 files changed
+211
-121
.buildkite/test-pipeline.yaml
+1
@@ -431,6 +431,7 @@ steps:
431
- pytest -v -s models/encoder_decoder/audio_language -m core_model
432
- pytest -v -s models/encoder_decoder/language -m core_model
433
- pytest -v -s models/encoder_decoder/vision_language -m core_model
434
+ - pytest -v -s models/decoder_only/vision_language/test_interleaved.py
435
436
- label: Multi-Modal Models Test (Extended) 1 # 48m
437
optional: true
tests/conftest.py
+19
-22
@@ -747,30 +747,27 @@ class VllmRunner:
747
videos: Optional[PromptVideoInput] = None,
748
audios: Optional[PromptAudioInput] = None,
749
) -> list[TextPrompt]:
750
- if images is not None:
751
- assert len(prompts) == len(images)
752
-
753
- if videos is not None:
754
- assert len(prompts) == len(videos)
750
756
- if audios is not None:
757
- assert len(prompts) == len(audios)
751
+ if any(x is not None and len(x) != len(prompts)
752
+ for x in [images, videos, audios]):
753
+ raise ValueError(
754
+ "All non-None multimodal inputs must have the same length as "
755
+ "prompts")
756
759
- inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
760
- if images is not None:
761
- for i, image in enumerate(images):
762
- if image is not None:
763
- inputs[i]["multi_modal_data"] = {"image": image}
764
-
765
- if videos is not None:
766
- for i, video in enumerate(videos):
767
- if video is not None:
768
- inputs[i]["multi_modal_data"] = {"video": video}
769
-
770
- if audios is not None:
771
- for i, audio in enumerate(audios):
772
- if audio is not None:
773
- inputs[i]["multi_modal_data"] = {"audio": audio}
757
+ inputs = []
758
+ for i, prompt in enumerate(prompts):
759
+ multi_modal_data = {}
760
+ if images is not None and (image := images[i]) is not None:
761
+ multi_modal_data["image"] = image
762
+ if videos is not None and (video := videos[i]) is not None:
763
+ multi_modal_data["video"] = video
764
+ if audios is not None and (audio := audios[i]) is not None:
765
+ multi_modal_data["audio"] = audio
766
+
767
+ inputs.append(
768
+ TextPrompt(prompt=prompt,
769
+ multi_modal_data=multi_modal_data
770
+ if multi_modal_data else None))
771
772
return inputs
773
tests/models/decoder_only/vision_language/test_interleaved.py
new
+77
@@ -0,0 +1,77 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import pytest
4
+
5
+from vllm.assets.image import ImageAsset
6
+from vllm.assets.video import VideoAsset
7
+
8
+models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
9
+
10
+
11
+def base_prompt(modalities_str: str) -> str:
12
+ return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n" # noqa: E501
13
+
14
+
15
+INTERLEAVED_PROMPT = base_prompt("<image><video><image>\n")
16
+NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
17
+
18
+
19
+@pytest.mark.parametrize("model", models)
20
+@pytest.mark.parametrize("dtype", ["float16"])
21
+@pytest.mark.parametrize("max_tokens", [128])
22
+def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
23
+ """
24
+ This is a simple test to check if interleaved and non-interleaved prompts
25
+ give the same result.
26
+ """
27
+
28
+ image_cherry = ImageAsset("cherry_blossom").pil_image.convert("RGB")
29
+ image_stop = ImageAsset("stop_sign").pil_image.convert("RGB")
30
+ images = [image_cherry, image_stop]
31
+ video = VideoAsset(name="sample_demo_1.mp4", num_frames=16).np_ndarrays
32
+
33
+ inputs = [
34
+ (
35
+ [INTERLEAVED_PROMPT],
36
+ [images],
37
+ [video],
38
+ ),
39
+ (
40
+ [NONINTERLEAVED_PROMPT],
41
+ [images],
42
+ [video],
43
+ ),
44
+ ]
45
+
46
+ with vllm_runner(model,
47
+ task="generate",
48
+ dtype=dtype,
49
+ limit_mm_per_prompt={"image": 2},
50
+ max_model_len=32768,
51
+ max_num_seqs=2,
52
+ tensor_parallel_size=1,
53
+ enforce_eager=True) as vllm_model:
54
+ vllm_outputs_per_case = [
55
+ vllm_model.generate_greedy(prompts,
56
+ max_tokens,
57
+ images=images,
58
+ videos=videos)
59
+ for prompts, images, videos in inputs
60
+ ]
61
+
62
+ all_results = [output[0][1] for output in vllm_outputs_per_case]
63
+ outputs = [(total_str, total_str.find("assistant\n") + len("assistant\n"))
64
+ for total_str in all_results]
65
+ prompt_lengths = [prompt_len for _, prompt_len in outputs]
66
+ generated_strs = [
67
+ total_str[prompt_len:] for total_str, prompt_len in outputs
68
+ ]
69
+ interleaved_prompt_len, noninterleaved_prompt_len = prompt_lengths
70
+ interleaved_output_str, noninterleaved_output_str = generated_strs
71
+
72
+ # The two prompts are identical except for the order of modality tokens.
73
+ assert interleaved_prompt_len == noninterleaved_prompt_len
74
+
75
+ # The two generated strings should be different because of the
76
+ # interleaved modality tokens.
77
+ assert interleaved_output_str != noninterleaved_output_str
tests/multimodal/test_utils.py
+64
-16
@@ -155,7 +155,7 @@ def test_merge_and_sort_multimodal_metadata():
155
]
156
},
157
mm_hashes={"image": ["hash1", "hash2"]},
158
- expected_modalities=["image"],
158
+ expected_modalities=["image", "image"],
159
expected_ranges=[
160
PlaceholderRange(offset=0, length=2),
161
PlaceholderRange(offset=3, length=2),
@@ -172,7 +172,7 @@ def test_merge_and_sort_multimodal_metadata():
172
]
173
},
174
mm_hashes=None,
175
- expected_modalities=["image"],
175
+ expected_modalities=["image", "image"],
176
expected_ranges=[
177
PlaceholderRange(offset=0, length=2),
178
PlaceholderRange(offset=2, length=2),
@@ -197,7 +197,7 @@ def test_merge_and_sort_multimodal_metadata():
197
"image": ["image_hash1", "image_hash2"],
198
"audio": ["audio_hash1", "audio_hash2"],
199
},
200
- expected_modalities=["audio", "image"],
200
+ expected_modalities=["audio", "audio", "image", "image"],
201
expected_ranges=[
202
PlaceholderRange(offset=0, length=2),
203
PlaceholderRange(offset=2, length=3),
@@ -223,7 +223,7 @@ def test_merge_and_sort_multimodal_metadata():
223
]
224
},
225
mm_hashes=None,
226
- expected_modalities=["audio", "image"],
226
+ expected_modalities=["audio", "audio", "image", "image"],
227
expected_ranges=[
228
PlaceholderRange(offset=0, length=2),
229
PlaceholderRange(offset=2, length=3),
@@ -254,7 +254,9 @@ def test_merge_and_sort_multimodal_metadata():
254
"audio": ["audio_hash1"],
255
"video": ["video_hash1", "video_hash2", "video_hash3"]
256
},
257
- expected_modalities=["audio", "video", "image"],
257
+ expected_modalities=[
258
+ "audio", "video", "video", "video", "image", "image"
259
+ ],
260
expected_ranges=[
261
PlaceholderRange(offset=0, length=2),
262
PlaceholderRange(offset=3, length=4),
@@ -300,12 +302,19 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
302
"image": ["image_hash1", "image_hash2"],
303
"audio": ["audio_hash1", "audio_hash2"],
304
},
303
- expected_modalities=[],
304
- expected_ranges=[],
305
- expected_hashes=None,
305
+ expected_modalities=["image", "audio", "image", "audio"],
306
+ expected_ranges=[
307
+ PlaceholderRange(offset=0, length=4),
308
+ PlaceholderRange(offset=5, length=2),
309
+ PlaceholderRange(offset=8, length=2),
310
+ PlaceholderRange(offset=11, length=4),
311
+ ],
312
+ expected_hashes=[
313
+ "image_hash1", "audio_hash1", "image_hash2", "audio_hash2"
314
+ ],
315
),
316
308
- # <image> <image> <video> <audio> <image>
317
+ # <image> <image> <audio> <video> <image>
318
TestCase(
319
mm_positions={
320
"image": [
@@ -321,15 +330,54 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
330
]
331
},
332
mm_hashes=None,
324
- expected_modalities=[],
325
- expected_ranges=[],
333
+ expected_modalities=["image", "image", "audio", "video", "image"],
334
+ expected_ranges=[
335
+ PlaceholderRange(offset=0, length=2),
336
+ PlaceholderRange(offset=2, length=3),
337
+ PlaceholderRange(offset=5, length=2),
338
+ PlaceholderRange(offset=8, length=5),
339
+ PlaceholderRange(offset=20, length=4),
340
+ ],
341
expected_hashes=None,
342
),
343
+
344
+ # <image> <audio> <video> <image> with hashes
345
+ TestCase(
346
+ mm_positions={
347
+ "image": [
348
+ PlaceholderRange(offset=0, length=2),
349
+ PlaceholderRange(offset=18, length=4),
350
+ ],
351
+ "audio": [
352
+ PlaceholderRange(offset=6, length=2),
353
+ ],
354
+ "video": [
355
+ PlaceholderRange(offset=10, length=5),
356
+ ]
357
+ },
358
+ mm_hashes={
359
+ "image": ["image_hash1", "image_hash2"],
360
+ "audio": ["audio_hash1"],
361
+ "video": ["video_hash1"],
362
+ },
363
+ expected_modalities=["image", "audio", "video", "image"],
364
+ expected_ranges=[
365
+ PlaceholderRange(offset=0, length=2),
366
+ PlaceholderRange(offset=6, length=2),
367
+ PlaceholderRange(offset=10, length=5),
368
+ PlaceholderRange(offset=18, length=4),
369
+ ],
370
+ expected_hashes=[
371
+ "image_hash1", "audio_hash1", "video_hash1", "image_hash2"
372
+ ],
373
+ ),
374
]
375
330
- for case in test_cases:
331
- with pytest.raises(ValueError) as ex_info:
332
- merge_and_sort_multimodal_metadata(case.mm_positions,
333
- case.mm_hashes)
376
+ for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
377
+ expected_hashes) in test_cases:
378
+ modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
379
+ mm_positions, mm_hashes)
380
335
- assert "Interleaved mixed-modality" in str(ex_info.value)
381
+ assert modalities == expected_modalities
382
+ assert ranges == expected_ranges
383
+ assert hashes == expected_hashes
vllm/multimodal/utils.py
+30
-52
@@ -303,14 +303,10 @@ def merge_and_sort_multimodal_metadata(
303
304
Optionally if a MultiModalHashDict is given, same operation will be
305
applied to the object and the sorted list of hashes will be returned.
306
-
307
- Raises:
308
- ValueError: If the input prompt has interleaved placeholders from
309
- different modalities (e.g, "<image><audio><image> Describe the
310
- content.")
306
307
Returns:
313
- list[str]: Sorted list of involved modalities.
308
+ list[str]: List of item modalities in order of their positions in
309
+ the input sequence.
310
list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from
311
mm_positions.
312
Optional[list[str]]: Sorted list of all hashes from mm_hashes if
@@ -324,50 +320,33 @@ def merge_and_sort_multimodal_metadata(
320
# For single modality, placeholder ranges and hashes are already sorted
321
# so we can return the list directly.
322
if len(modalities) == 1:
327
- if mm_hashes is None:
328
- return modalities, list(mm_positions[modalities[0]]), None
329
- else:
330
- return modalities, list(mm_positions[modalities[0]]), list(
331
- mm_hashes[modalities[0]])
332
-
333
- placeholder_lists_with_modality = [(modality, mm_positions[modality])
334
- for modality in modalities]
335
-
336
- if mm_hashes is None:
337
- sorted_placeholder_lists = sorted(placeholder_lists_with_modality,
338
- key=lambda x: x[1][0]['offset'])
339
- sorted_hash_lists = None
340
- else:
341
- hashes_lists = [
342
- mm_hashes[modality] for modality in modalities
343
- if modality in mm_hashes
344
- ]
345
- sorted_pairs = sorted(zip(placeholder_lists_with_modality,
346
- hashes_lists),
347
- key=lambda x: x[0][1][0]['offset'])
348
- sorted_placeholder_tuple, sorted_hash_tuple = zip(*sorted_pairs)
349
- sorted_placeholder_lists = list(sorted_placeholder_tuple)
350
- sorted_hash_lists = list(sorted_hash_tuple)
351
-
352
- sorted_modalities = [modality for modality, _ in sorted_placeholder_lists]
353
-
354
- # Flatten sorted list of lists to a single list and verify there is no
355
- # interleaving of placeholders from different modalities.
356
- merged_placeholders: list[PlaceholderRange] = []
357
- for modality, placeholder_list in sorted_placeholder_lists:
358
- if merged_placeholders and placeholder_list[0][
359
- 'offset'] < merged_placeholders[-1]['offset']:
360
- raise ValueError(
361
- "Interleaved mixed-modality inference is currently not "
362
- "supported.")
363
- merged_placeholders.extend(placeholder_list)
364
-
365
- if sorted_hash_lists is not None:
366
- merged_hashes = []
367
- for hash_list in sorted_hash_lists:
368
- merged_hashes.extend(hash_list)
369
- else:
370
- merged_hashes = None
323
+ modality = modalities[0]
324
+ placeholder_list = list(mm_positions[modality])
325
+
326
+ return [modality] * len(
327
+ placeholder_list
328
+ ), placeholder_list, None if not mm_hashes else mm_hashes[modality]
329
+
330
+ # Create a list of (modality, placeholder, hash) tuples for all placeholders
331
+ all_items = []
332
+ for modality in modalities:
333
+ placeholder_list = list(mm_positions[modality])
334
+ hash_list: list[Optional[str]] = list(
335
+ mm_hashes[modality]) if mm_hashes and modality in mm_hashes else [
336
+ None
337
+ ] * len(placeholder_list)
338
+
339
+ for placeholder, hash_value in zip(placeholder_list, hash_list):
340
+ all_items.append((modality, placeholder, hash_value))
341
+
342
+ # Sort all items by offset
343
+ all_items.sort(key=lambda x: x[1]['offset'])
344
+
345
+ # Split into separate lists
346
+ sorted_modalities = [item[0] for item in all_items]
347
+ merged_placeholders = [item[1] for item in all_items]
348
+ merged_hashes = [str(item[2])
349
+ for item in all_items] if mm_hashes is not None else None
350
351
return sorted_modalities, merged_placeholders, merged_hashes
352
@@ -383,8 +362,7 @@ def group_mm_inputs_by_modality(
362
363
Returns:
364
list[list[MultiModalKwargs]]: List of list of MultiModalKwargs, each
386
- inner list contains consecutive MultiModalKwargs with same modality, or
387
- one with multimodal modalities.
365
+ inner list contains consecutive MultiModalKwargs with same modality.
366
"""
367
if not mm_inputs:
368
return []
vllm/v1/engine/processor.py
+20
-31
@@ -234,22 +234,11 @@ class Processor:
234
if decoder_inputs["type"] == "multimodal":
235
decoder_mm_inputs = decoder_inputs["mm_kwargs"]
236
237
- # The output of merged multi-modal processor (`decoder_mm_inputs`)
238
- # contains the kwargs for all items from all modalities.
239
- # This code separates them so that there is one set of kwargs
240
- # per item per modality.
241
- individual_mm_inputs = [
242
- MultiModalKwargs.from_items([item])
243
- for modality in decoder_mm_inputs.modalities
244
- for item in decoder_mm_inputs.get_items(modality)
245
- ]
246
-
237
# Merge and flatten multimodal placeholders, hashes and inputs
238
# from dictionaries to lists, and sort them by each item's position
239
# in the input sequence.
250
- # NOTE: interleaved modalities are not supported.
240
(
252
- sorted_modalities,
241
+ sorted_item_modalities,
242
sorted_mm_positions,
243
sorted_mm_hashes,
244
) = merge_and_sort_multimodal_metadata(
@@ -257,26 +246,26 @@ class Processor:
246
decoder_inputs["mm_hashes"] if self.use_hash else None,
247
)
248
260
- # NOTE: Sort multimodal inputs/kwargs ONLY IF there are multiple
261
- # modalities involved.
262
- if len(sorted_modalities) > 1:
263
- modality_order_dict = {
264
- modality: order
265
- for order, modality in enumerate(sorted_modalities)
266
- }
267
-
268
- # Sanity check to make sure each multimodal input has only one
269
- # modality key.
270
- for mm_input in individual_mm_inputs:
271
- assert len(mm_input.modalities) == 1
272
-
273
- # Sort MultiModalKwargs to match sorted_mm_positions
274
- sorted_mm_inputs = sorted(
275
- individual_mm_inputs,
276
- key=lambda mm_input: modality_order_dict[list(
277
- mm_input.modalities)[0]])
249
+ # The output of merged multi-modal processor (`decoder_mm_inputs`)
250
+ # is a single MultiModalKwargs for all items from all modalities.
251
+ # This code flattens kwargs for individual items in a list and
252
+ # sorts them by each item's position in the input sequence if there
253
+ # are multiple modalities.
254
+ unique_modalities = set(sorted_item_modalities)
255
+ if len(unique_modalities) > 1:
256
+ sorted_mm_inputs = []
257
+ used_indices = {modality: 0 for modality in unique_modalities}
258
+ for modality in sorted_item_modalities:
259
+ items = decoder_mm_inputs.get_items(modality)
260
+ item = items[used_indices[modality]]
261
+ sorted_mm_inputs.append(MultiModalKwargs.from_items([item
262
+ ]))
263
+ used_indices[modality] += 1
264
else:
279
- sorted_mm_inputs = individual_mm_inputs
265
+ sorted_mm_inputs = [
266
+ MultiModalKwargs.from_items([item]) for item in
267
+ decoder_mm_inputs.get_items(sorted_item_modalities[0])
268
+ ]
269
270
return EngineCoreRequest(
271
request_id=request_id,