A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 03:45 UTC
[Renderer] Move MM Hash parsing into Renderer (#34711)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 18, 2026 at 11:18 UTC
a0d8d944e2659cedd52bccef63fbc7f764be4cf6
16 files changed
+255
-218
tests/renderers/test_process_multi_modal_uuids.py
+22
-13
@@ -6,6 +6,7 @@ import pytest
6
from vllm.assets.image import ImageAsset
7
from vllm.assets.video import VideoAsset
8
from vllm.config import CacheConfig, ModelConfig, VllmConfig
9
+from vllm.multimodal.parse import parse_mm_uuids
10
from vllm.renderers.hf import HfRenderer
11
from vllm.tokenizers.registry import tokenizer_args_from_config
12
@@ -45,10 +46,11 @@ def test_multi_modal_uuids_length_mismatch_raises():
46
mm_uuids = {"image": ["hash_cherry"]}
47
48
mm_processor = renderer.get_mm_processor()
48
- mm_items = mm_processor.info.parse_mm_data(mm_data)
49
+ mm_data_items = mm_processor.info.parse_mm_data(mm_data)
50
+ mm_uuid_items = parse_mm_uuids(mm_uuids)
51
52
with pytest.raises(ValueError, match="must have same length as"):
51
- renderer._process_mm_uuids(mm_data, mm_items, mm_uuids, "req-1")
53
+ renderer._process_mm_uuids(mm_data, mm_data_items, mm_uuid_items, "req-1")
54
55
56
def test_multi_modal_uuids_missing_modality_raises():
@@ -63,10 +65,11 @@ def test_multi_modal_uuids_missing_modality_raises():
65
mm_uuids = {"image": ["hash_cherry"]}
66
67
mm_processor = renderer.get_mm_processor()
66
- mm_items = mm_processor.info.parse_mm_data(mm_data)
68
+ mm_data_items = mm_processor.info.parse_mm_data(mm_data)
69
+ mm_uuid_items = parse_mm_uuids(mm_uuids)
70
71
with pytest.raises(ValueError, match="is empty but .* is missing"):
69
- renderer._process_mm_uuids(mm_data, mm_items, mm_uuids, "req-2")
72
+ renderer._process_mm_uuids(mm_data, mm_data_items, mm_uuid_items, "req-2")
73
74
75
@pytest.mark.parametrize(
@@ -78,7 +81,7 @@ def test_multi_modal_uuids_missing_modality_raises():
81
],
82
)
83
def test_multi_modal_uuids_accepts_none_and_passes_through(
81
- monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool
84
+ mm_cache_gb: float, enable_prefix_caching: bool
85
):
86
renderer = _build_renderer(
87
mm_cache_gb=mm_cache_gb,
@@ -94,9 +97,11 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
97
mm_uuids = {"image": [None, "hash_stop"], "video": None}
98
99
mm_processor = renderer.get_mm_processor()
97
- mm_items = mm_processor.info.parse_mm_data(mm_data)
100
+ mm_data_items = mm_processor.info.parse_mm_data(mm_data)
101
+ mm_uuid_items = parse_mm_uuids(mm_uuids)
102
+
103
processed_mm_uuids = renderer._process_mm_uuids(
99
- mm_data, mm_items, mm_uuids, "req-3"
104
+ mm_data, mm_data_items, mm_uuid_items, "req-3"
105
)
106
107
assert processed_mm_uuids == mm_uuids
@@ -111,7 +116,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
116
],
117
)
118
def test_multi_modal_uuids_accepts_empty(
114
- monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool
119
+ mm_cache_gb: float, enable_prefix_caching: bool
120
):
121
renderer = _build_renderer(
122
mm_cache_gb=mm_cache_gb,
@@ -124,15 +129,17 @@ def test_multi_modal_uuids_accepts_empty(
129
mm_uuids = {"image": [], "video": None} # type: ignore[var-annotated]
130
131
mm_processor = renderer.get_mm_processor()
127
- mm_items = mm_processor.info.parse_mm_data(mm_data)
132
+ mm_data_items = mm_processor.info.parse_mm_data(mm_data)
133
+ mm_uuid_items = parse_mm_uuids(mm_uuids)
134
+
135
processed_mm_uuids = renderer._process_mm_uuids(
129
- mm_data, mm_items, mm_uuids, "req-4"
136
+ mm_data, mm_data_items, mm_uuid_items, "req-4"
137
)
138
139
assert processed_mm_uuids == mm_uuids
140
141
135
-def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
142
+def test_multi_modal_uuids_ignored_when_caching_disabled():
143
# When both processor cache is 0 and prefix caching disabled, the
144
# processor builds overrides from request id instead of using user UUIDs.
145
renderer = _build_renderer(mm_cache_gb=0.0, enable_prefix_caching=False)
@@ -145,9 +152,11 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
152
mm_uuids = {"image": ["hash_cherry", "hash_stop"], "video": ["hash_video"]}
153
154
mm_processor = renderer.get_mm_processor()
148
- mm_items = mm_processor.info.parse_mm_data(mm_data)
155
+ mm_data_items = mm_processor.info.parse_mm_data(mm_data)
156
+ mm_uuid_items = parse_mm_uuids(mm_uuids)
157
+
158
processed_mm_uuids = renderer._process_mm_uuids(
150
- mm_data, mm_items, mm_uuids, request_id
159
+ mm_data, mm_data_items, mm_uuid_items, request_id
160
)
161
162
# Expect request-id-based overrides are passed through
vllm/inputs/preprocess.py
+3
-3
@@ -91,7 +91,7 @@ class InputPreprocessor:
91
self,
92
prompt: str | list[int],
93
mm_data: MultiModalDataDict,
94
- mm_processor_kwargs: Mapping[str, object] | None,
94
+ mm_processor_kwargs: Mapping[str, object] | None = None,
95
tokenization_kwargs: dict[str, Any] | None = None,
96
*,
97
mm_uuids: MultiModalUUIDDict | None = None,
@@ -103,9 +103,9 @@ class InputPreprocessor:
103
return self.renderer._process_multimodal(
104
prompt,
105
mm_data,
106
+ mm_uuids=mm_uuids,
107
mm_processor_kwargs=mm_processor_kwargs,
108
tokenization_kwargs=tokenization_kwargs,
108
- mm_uuids=mm_uuids,
109
)
110
111
def _process_embeds(
@@ -144,7 +144,7 @@ class InputPreprocessor:
144
inputs = self._process_multimodal(
145
prompt_token_ids,
146
multi_modal_data,
147
- parsed_content.get("mm_processor_kwargs") or {},
147
+ parsed_content.get("mm_processor_kwargs"),
148
tokenization_kwargs=tokenization_kwargs,
149
mm_uuids=parsed_content.get("multi_modal_uuids"),
150
)
vllm/model_executor/models/clip.py
+9
-6
@@ -36,9 +36,13 @@ from vllm.multimodal.inputs import (
36
MultiModalFieldConfig,
37
MultiModalInputs,
38
MultiModalKwargsItems,
39
- MultiModalUUIDDict,
39
)
41
-from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
40
+from vllm.multimodal.parse import (
41
+ ImageProcessorItems,
42
+ ImageSize,
43
+ MultiModalDataItems,
44
+ MultiModalUUIDItems,
45
+)
46
from vllm.multimodal.processing import (
47
BaseDummyInputsBuilder,
48
BaseMultiModalProcessor,
@@ -203,10 +207,9 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
207
self,
208
prompt: str | list[int],
209
mm_items: MultiModalDataItems,
206
- hf_processor_mm_kwargs: Mapping[str, object],
210
+ mm_uuid_items: MultiModalUUIDItems | None = None,
211
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
212
tokenization_kwargs: Mapping[str, object] | None = None,
208
- *,
209
- mm_uuids: MultiModalUUIDDict | None = None,
213
) -> MultiModalInputs:
214
if mm_items:
215
if isinstance(prompt, str):
@@ -235,9 +238,9 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
238
return super().apply(
239
prompt=prompt,
240
mm_items=mm_items,
241
+ mm_uuid_items=mm_uuid_items,
242
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
243
tokenization_kwargs=tokenization_kwargs,
240
- mm_uuids=mm_uuids,
244
)
245
246
def _hf_processor_applies_updates(
vllm/model_executor/models/deepseek_vl2.py
+4
-4
@@ -24,13 +24,13 @@ from vllm.multimodal.inputs import (
24
MultiModalDataDict,
25
MultiModalFieldConfig,
26
MultiModalKwargsItems,
27
- MultiModalUUIDDict,
27
)
28
from vllm.multimodal.parse import (
29
ImageEmbeddingItems,
30
ImageProcessorItems,
31
ImageSize,
32
MultiModalDataItems,
33
+ MultiModalUUIDItems,
34
)
35
from vllm.multimodal.processing import BaseDummyInputsBuilder
36
from vllm.multimodal.processing.processor import (
@@ -313,9 +313,9 @@ class DeepseekVL2MultiModalProcessor(
313
self,
314
prompt: str | list[int],
315
mm_data_items: MultiModalDataItems,
316
+ mm_uuid_items: MultiModalUUIDItems | None,
317
hf_processor_mm_kwargs: Mapping[str, object],
318
tokenization_kwargs: Mapping[str, object],
318
- mm_uuids: MultiModalUUIDDict | None = None,
319
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
320
# The processor logic is different for len(images) <= 2 vs > 2
321
# Since the processing cache assumes that the processor output is
@@ -325,17 +325,17 @@ class DeepseekVL2MultiModalProcessor(
325
return self._apply_hf_processor(
326
prompt=prompt,
327
mm_data_items=mm_data_items,
328
+ mm_uuid_items=mm_uuid_items,
329
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
330
tokenization_kwargs=tokenization_kwargs,
330
- mm_uuids=mm_uuids,
331
)
332
333
return super()._cached_apply_hf_processor(
334
prompt=prompt,
335
mm_data_items=mm_data_items,
336
+ mm_uuid_items=mm_uuid_items,
337
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
338
tokenization_kwargs=tokenization_kwargs,
338
- mm_uuids=mm_uuids,
339
)
340
341
vllm/model_executor/models/h2ovl.py
+5
-4
@@ -16,11 +16,12 @@ from transformers import PretrainedConfig
16
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
-from vllm.multimodal.inputs import MultiModalKwargsItems, MultiModalUUIDDict
19
+from vllm.multimodal.inputs import MultiModalKwargsItems
20
from vllm.multimodal.parse import (
21
ImageEmbeddingItems,
22
ImageProcessorItems,
23
MultiModalDataItems,
24
+ MultiModalUUIDItems,
25
)
26
from vllm.multimodal.processing.processor import (
27
MultiModalProcessingInfo,
@@ -491,9 +492,9 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
492
self,
493
prompt: str | list[int],
494
mm_data_items: MultiModalDataItems,
495
+ mm_uuid_items: MultiModalUUIDItems | None,
496
hf_processor_mm_kwargs: Mapping[str, object],
497
tokenization_kwargs: Mapping[str, object],
496
- mm_uuids: MultiModalUUIDDict | None = None,
498
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
499
# The processor logic is different for len(images) <= 1 vs > 1
500
# Since the processing cache assumes that the processor output is
@@ -503,17 +504,17 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
504
return self._apply_hf_processor(
505
prompt=prompt,
506
mm_data_items=mm_data_items,
507
+ mm_uuid_items=mm_uuid_items,
508
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
509
tokenization_kwargs=tokenization_kwargs,
508
- mm_uuids=mm_uuids,
510
)
511
512
return super()._cached_apply_hf_processor(
513
prompt=prompt,
514
mm_data_items=mm_data_items,
515
+ mm_uuid_items=mm_uuid_items,
516
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
517
tokenization_kwargs=tokenization_kwargs,
516
- mm_uuids=mm_uuids,
518
)
519
520
vllm/model_executor/models/llava.py
+6
-6
@@ -30,7 +30,6 @@ from vllm.multimodal.inputs import (
30
MultiModalFieldConfig,
31
MultiModalInputs,
32
MultiModalKwargsItems,
33
- MultiModalUUIDDict,
33
mm_inputs,
34
)
35
from vllm.multimodal.parse import (
@@ -38,6 +37,7 @@ from vllm.multimodal.parse import (
37
ImageProcessorItems,
38
ImageSize,
39
MultiModalDataItems,
40
+ MultiModalUUIDItems,
41
)
42
from vllm.multimodal.processing import (
43
BaseDummyInputsBuilder,
@@ -773,9 +773,9 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
773
self,
774
prompt: str | list[int],
775
mm_items: MultiModalDataItems,
776
- hf_processor_mm_kwargs: Mapping[str, object],
776
+ mm_uuid_items: MultiModalUUIDItems | None = None,
777
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
778
tokenization_kwargs: Mapping[str, object] | None = None,
778
- mm_uuids: MultiModalUUIDDict | None = None,
779
) -> MultiModalInputs:
780
hf_config = self.info.get_hf_config()
781
image_token_id = hf_config.image_token_index
@@ -789,9 +789,9 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
789
result = super().apply(
790
prompt,
791
mm_items,
792
- hf_processor_mm_kwargs,
793
- tokenization_kwargs,
794
- mm_uuids=mm_uuids,
792
+ mm_uuid_items,
793
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
794
+ tokenization_kwargs=tokenization_kwargs,
795
)
796
797
mm_item_counts = mm_items.get_all_counts()
vllm/model_executor/models/paligemma.py
+6
-6
@@ -16,12 +16,12 @@ from vllm.multimodal.inputs import (
16
MultiModalFieldConfig,
17
MultiModalInputs,
18
MultiModalKwargsItems,
19
- MultiModalUUIDDict,
19
)
20
from vllm.multimodal.parse import (
21
ImageEmbeddingItems,
22
ImageProcessorItems,
23
MultiModalDataItems,
24
+ MultiModalUUIDItems,
25
)
26
from vllm.multimodal.processing import (
27
BaseDummyInputsBuilder,
@@ -231,16 +231,16 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
231
self,
232
prompt: str | list[int],
233
mm_items: MultiModalDataItems,
234
- hf_processor_mm_kwargs: Mapping[str, object],
234
+ mm_uuid_items: MultiModalUUIDItems | None = None,
235
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
236
tokenization_kwargs: Mapping[str, object] | None = None,
236
- mm_uuids: MultiModalUUIDDict | None = None,
237
) -> MultiModalInputs:
238
mm_inputs = super().apply(
239
prompt,
240
mm_items,
241
- hf_processor_mm_kwargs,
242
- tokenization_kwargs,
243
- mm_uuids=mm_uuids,
241
+ mm_uuid_items,
242
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
243
+ tokenization_kwargs=tokenization_kwargs,
244
)
245
prompt_token_ids = mm_inputs["prompt_token_ids"]
246
vllm/model_executor/models/pixtral.py
+8
-4
@@ -44,10 +44,14 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
44
from vllm.multimodal.inputs import (
45
MultiModalDataDict,
46
MultiModalFieldConfig,
47
- MultiModalUUIDDict,
47
NestedTensors,
48
)
50
-from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
49
+from vllm.multimodal.parse import (
50
+ ImageProcessorItems,
51
+ ImageSize,
52
+ MultiModalDataItems,
53
+ MultiModalUUIDItems,
54
+)
55
from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs
56
from vllm.multimodal.processing.processor import (
57
BaseMultiModalProcessor,
@@ -344,16 +348,16 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
348
self,
349
prompt: str | list[int],
350
mm_data_items: MultiModalDataItems,
351
+ mm_uuid_items: MultiModalUUIDItems | None,
352
hf_processor_mm_kwargs: Mapping[str, object],
353
tokenization_kwargs: Mapping[str, object],
349
- mm_uuids: MultiModalUUIDDict | None = None,
354
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
355
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
356
prompt=prompt,
357
mm_data_items=mm_data_items,
358
+ mm_uuid_items=mm_uuid_items,
359
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
360
tokenization_kwargs=tokenization_kwargs,
356
- mm_uuids=mm_uuids,
361
)
362
363
# NOTE: The tokens are already inserted by the chat template
vllm/model_executor/models/siglip.py
+9
-6
@@ -42,9 +42,13 @@ from vllm.multimodal.inputs import (
42
MultiModalFieldConfig,
43
MultiModalInputs,
44
MultiModalKwargsItems,
45
- MultiModalUUIDDict,
45
)
47
-from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
46
+from vllm.multimodal.parse import (
47
+ ImageProcessorItems,
48
+ ImageSize,
49
+ MultiModalDataItems,
50
+ MultiModalUUIDItems,
51
+)
52
from vllm.multimodal.processing import (
53
BaseDummyInputsBuilder,
54
BaseMultiModalProcessor,
@@ -189,10 +193,9 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
193
self,
194
prompt: str | list[int],
195
mm_items: MultiModalDataItems,
192
- hf_processor_mm_kwargs: Mapping[str, object],
196
+ mm_uuid_items: MultiModalUUIDItems | None = None,
197
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
198
tokenization_kwargs: Mapping[str, object] | None = None,
194
- *,
195
- mm_uuids: MultiModalUUIDDict | None = None,
199
) -> MultiModalInputs:
200
if mm_items:
201
if isinstance(prompt, str):
@@ -221,9 +224,9 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
224
return super().apply(
225
prompt=prompt,
226
mm_items=mm_items,
227
+ mm_uuid_items=mm_uuid_items,
228
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
229
tokenization_kwargs=tokenization_kwargs,
226
- mm_uuids=mm_uuids,
230
)
231
232
def _hf_processor_applies_updates(
vllm/model_executor/models/terratorch.py
+8
-4
@@ -46,7 +46,6 @@ from vllm.multimodal.inputs import (
46
MultiModalFieldConfig,
47
MultiModalInputs,
48
MultiModalKwargsItems,
49
- MultiModalUUIDDict,
49
PlaceholderRange,
50
mm_inputs,
51
)
@@ -55,6 +54,7 @@ from vllm.multimodal.parse import (
54
ModalityDataItems,
55
MultiModalDataItems,
56
MultiModalDataParser,
57
+ MultiModalUUIDItems,
58
)
59
from vllm.multimodal.processing import (
60
BaseDummyInputsBuilder,
@@ -196,15 +196,19 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
196
self,
197
prompt: str | list[int],
198
mm_items: MultiModalDataItems,
199
- hf_processor_mm_kwargs: Mapping[str, object],
199
+ mm_uuid_items: MultiModalUUIDItems | None = None,
200
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
201
tokenization_kwargs: Mapping[str, object] | None = None,
201
- mm_uuids: MultiModalUUIDDict | None = None,
202
) -> MultiModalInputs:
203
+ if hf_processor_mm_kwargs is None:
204
+ hf_processor_mm_kwargs = {}
205
if tokenization_kwargs is None:
206
tokenization_kwargs = {}
207
208
mm_hashes = self._hash_mm_items(
207
- mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
209
+ mm_items,
210
+ mm_uuid_items,
211
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
212
)
213
214
_, passthrough_data = self._get_hf_mm_data(mm_items)
vllm/model_executor/models/transformers/multimodal.py
+12
-5
@@ -31,11 +31,14 @@ from vllm.multimodal.inputs import (
31
MultiModalFeatureSpec,
32
MultiModalFieldConfig,
33
MultiModalInputs,
34
- MultiModalUUIDDict,
34
PlaceholderRange,
35
mm_inputs,
36
)
38
-from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
37
+from vllm.multimodal.parse import (
38
+ ImageProcessorItems,
39
+ MultiModalDataItems,
40
+ MultiModalUUIDItems,
41
+)
42
from vllm.multimodal.processing import (
43
BaseDummyInputsBuilder,
44
BaseMultiModalProcessor,
@@ -177,9 +180,9 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
180
self,
181
prompt: str | list[int],
182
mm_items: MultiModalDataItems,
180
- hf_processor_mm_kwargs: Mapping[str, object],
183
+ mm_uuid_items: MultiModalUUIDItems | None = None,
184
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
185
tokenization_kwargs: Mapping[str, object] | None = None,
182
- mm_uuids: MultiModalUUIDDict | None = None,
186
) -> MultiModalInputs:
187
"""
188
Process multi-modal inputs to be used in vLLM.
@@ -187,6 +190,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
190
Apply HF Processor on prompt text and multi-modal data together,
191
outputting token IDs and processed tensors.
192
"""
193
+ if hf_processor_mm_kwargs is None:
194
+ hf_processor_mm_kwargs = {}
195
if tokenization_kwargs is None:
196
tokenization_kwargs = {}
197
@@ -258,7 +263,9 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
263
264
# Use overrides if provided; fallback to data-dependent hashing.
265
mm_hashes = self._hash_mm_items(
261
- mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
266
+ mm_items,
267
+ mm_uuid_items,
268
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
269
)
270
271
return mm_inputs(
vllm/model_executor/models/voxtral.py
+3
-3
@@ -41,13 +41,13 @@ from vllm.multimodal.inputs import (
41
MultiModalDataDict,
42
MultiModalFieldConfig,
43
MultiModalKwargsItems,
44
- MultiModalUUIDDict,
44
NestedTensors,
45
)
46
from vllm.multimodal.parse import (
47
AudioProcessorItems,
48
MultiModalDataItems,
49
MultiModalDataParser,
50
+ MultiModalUUIDItems,
51
)
52
from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs
53
from vllm.multimodal.processing.processor import (
@@ -363,16 +363,16 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
363
self,
364
prompt: str | list[int],
365
mm_data_items: MultiModalDataItems,
366
+ mm_uuid_items: MultiModalUUIDItems | None,
367
hf_processor_mm_kwargs: Mapping[str, object],
368
tokenization_kwargs: Mapping[str, object],
368
- mm_uuids: MultiModalUUIDDict | None = None,
369
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
370
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
371
prompt=prompt,
372
mm_data_items=mm_data_items,
373
+ mm_uuid_items=mm_uuid_items,
374
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
375
tokenization_kwargs=tokenization_kwargs,
375
- mm_uuids=mm_uuids,
376
)
377
378
# NOTE: The tokens are already inserted by the chat template
vllm/multimodal/inputs.py
+1
-1
@@ -155,7 +155,7 @@ The built-in modalities are defined by
155
[`MultiModalDataBuiltins`][vllm.multimodal.inputs.MultiModalDataBuiltins].
156
"""
157
158
-MultiModalUUIDDict: TypeAlias = Mapping[str, list[str | None] | str]
158
+MultiModalUUIDDict: TypeAlias = Mapping[str, Sequence[str | None] | str]
159
"""
160
A dictionary containing user-provided UUIDs for items in each modality.
161
If a UUID for an item is not provided, its entry will be `None` and
vllm/multimodal/parse.py
+63
-48
@@ -3,7 +3,7 @@
3
4
from abc import ABC, abstractmethod
5
from collections import UserDict
6
-from collections.abc import Callable, Iterator, Mapping, Sequence
6
+from collections.abc import Callable, Iterator, Mapping, Sequence, Set
7
from typing import (
8
TYPE_CHECKING,
9
Any,
@@ -33,6 +33,7 @@ from .inputs import (
33
MultiModalDataDict,
34
MultiModalFieldConfig,
35
MultiModalKwargsItems,
36
+ MultiModalUUIDDict,
37
VideoItem,
38
)
39
from .media import MediaWithBytes
@@ -297,14 +298,15 @@ class DictEmbeddingItems(
298
return self.data
299
300
300
-class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
301
- def __init__(self, data: Sequence[HfAudioItem] | None) -> None:
302
- if data is None:
303
- data = [None]
301
+class AudioProcessorItems(ProcessorBatchItems[HfAudioItem | None]):
302
+ def __init__(self, data: Sequence[HfAudioItem | None]) -> None:
303
super().__init__(data, "audio")
304
305
def get_audio_length(self, item_idx: int) -> int:
306
audio = self.get(item_idx)
307
+ if audio is None:
308
+ raise ValueError(f"Cannot get length of cached audio at {item_idx}")
309
+
310
return len(audio)
311
312
@@ -322,14 +324,14 @@ class ImageSize(NamedTuple):
324
height: int
325
326
325
-class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
326
- def __init__(self, data: Sequence[HfImageItem] | None) -> None:
327
- if data is None:
328
- data = [None]
327
+class ImageProcessorItems(ProcessorBatchItems[HfImageItem | None]):
328
+ def __init__(self, data: Sequence[HfImageItem | None]) -> None:
329
super().__init__(data, "image")
330
331
def get_image_size(self, item_idx: int) -> ImageSize:
332
image = self.get(item_idx)
333
+ if image is None:
334
+ raise ValueError(f"Cannot get size of cached image at {item_idx}")
335
336
if isinstance(image, PILImage.Image):
337
return ImageSize(*image.size)
@@ -349,22 +351,31 @@ class ImageEmbeddingItems(EmbeddingItems):
351
super().__init__(data, "image", expected_hidden_size)
352
353
352
-class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
354
+class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]):
355
def __init__(
356
self,
355
- data: Sequence[HfVideoItem] | None,
357
+ data: Sequence[HfVideoItem | None],
358
metadata: dict[str, Any] | list[dict[str, Any] | None] | None = None,
359
) -> None:
358
- if data is None:
359
- data = [None]
360
super().__init__(data, "video")
361
+
362
self.metadata = metadata
363
364
def get_num_frames(self, item_idx: int) -> int:
364
- return len(self.get(item_idx))
365
+ video = self.get(item_idx)
366
+ if video is None:
367
+ raise ValueError(f"Cannot get length of cached video at {item_idx}")
368
+
369
+ return len(video)
370
371
def get_frame_size(self, item_idx: int) -> ImageSize:
367
- image = self.get(item_idx)[0] # Assume that the video isn't empty
372
+ video = self.get(item_idx)
373
+ if video is None:
374
+ raise ValueError(f"Cannot get size of cached video at {item_idx}")
375
+ if len(video) == 0:
376
+ raise ValueError(f"Cannot get size of empty video at {item_idx}")
377
+
378
+ image = video[0]
379
380
if isinstance(image, PILImage.Image):
381
return ImageSize(*image.size)
@@ -400,6 +411,15 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
411
normalized such that each entry corresponds to a list.
412
"""
413
414
+ def select(self, modalities: Set[str]):
415
+ """
416
+ Construct a new `MultiModalDataItems` instance containing only the
417
+ selected modalities.
418
+ """
419
+ return MultiModalDataItems(
420
+ {modality: self[modality] for modality in modalities}
421
+ )
422
+
423
def get_count(self, modality: str, *, strict: bool = True) -> int:
424
"""
425
Get the number of data items belonging to a modality.
@@ -497,19 +517,11 @@ class MultiModalDataParser:
517
) -> TypeGuard[torch.Tensor | list[torch.Tensor]]:
518
if isinstance(data, torch.Tensor):
519
return data.ndim == 3
500
- if is_list_of(data, torch.Tensor):
520
+ if is_list_of(data, torch.Tensor) and len(data) > 0:
521
return data[0].ndim == 2 # type: ignore[index]
522
523
return False
524
505
- def _is_empty(self, data: object) -> TypeGuard[None]:
506
- if isinstance(data, list):
507
- return len(data) == 0
508
- if isinstance(data, (np.ndarray, torch.Tensor)):
509
- return data.size == 0
510
-
511
- return False
512
-
525
def _get_audio_with_sr(
526
self,
527
audio: AudioItem,
@@ -545,12 +557,6 @@ class MultiModalDataParser:
557
data: ModalityData[AudioItem],
558
) -> ModalityDataItems[Any, Any] | None:
559
if data is None:
548
- return AudioProcessorItems(None)
549
-
550
- # also check single audio item with sampling rate
551
- if self._is_empty(data) or (
552
- isinstance(data, tuple) and self._is_empty(data[0])
553
- ):
560
return None
561
562
if self.is_embeddings(data):
@@ -558,9 +564,8 @@ class MultiModalDataParser:
564
565
data_items: list[AudioItem]
566
if (
561
- is_list_of(data, float)
562
- or isinstance(data, (np.ndarray, torch.Tensor))
563
- and data.ndim == 1
567
+ (is_list_of(data, float) and len(data) > 0)
568
+ or (isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 1)
569
or isinstance(data, tuple)
570
):
571
data_items = [data]
@@ -591,18 +596,13 @@ class MultiModalDataParser:
596
data: ModalityData[ImageItem],
597
) -> ModalityDataItems[Any, Any] | None:
598
if data is None:
594
- return ImageProcessorItems(None)
595
-
596
- if self._is_empty(data):
599
return None
600
601
if self.is_embeddings(data):
602
return ImageEmbeddingItems(data, self.expected_hidden_size)
603
602
- if (
603
- isinstance(data, (PILImage.Image, MediaWithBytes))
604
- or isinstance(data, (np.ndarray, torch.Tensor))
605
- and data.ndim == 3
604
+ if isinstance(data, (PILImage.Image, MediaWithBytes)) or (
605
+ isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 3
606
):
607
data_items = [data]
608
elif isinstance(data, (np.ndarray, torch.Tensor)):
@@ -617,19 +617,14 @@ class MultiModalDataParser:
617
data: ModalityData[VideoItem],
618
) -> ModalityDataItems[Any, Any] | None:
619
if data is None:
620
- return VideoProcessorItems(None)
621
-
622
- if self._is_empty(data):
620
return None
621
622
if self.is_embeddings(data):
623
return VideoEmbeddingItems(data, self.expected_hidden_size)
624
625
data_items: list[VideoItem]
629
- if (
630
- is_list_of(data, PILImage.Image)
631
- or isinstance(data, (np.ndarray, torch.Tensor))
632
- and data.ndim == 4
626
+ if (is_list_of(data, PILImage.Image) and len(data) > 0) or (
627
+ isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 4
628
):
629
data_items = [data]
630
elif isinstance(data, (np.ndarray, torch.Tensor)):
@@ -664,12 +659,15 @@ class MultiModalDataParser:
659
data: ModalityData[Any],
660
) -> ModalityDataItems[Any, Any] | None:
661
"""Parse vision chunk data (unified image and video chunks)."""
667
- if data is None or self._is_empty(data):
662
+ if data is None:
663
return None
664
+
665
if self.is_embeddings(data):
666
raise ValueError("Do not support embedding data for vision_chunk right now")
667
+
668
if isinstance(data, dict):
669
data = [data]
670
+
671
return VisionChunkProcessorItems(data)
672
673
def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
@@ -693,3 +691,20 @@ class MultiModalDataParser:
691
mm_items[k] = parsed_data
692
693
return mm_items
694
+
695
+
696
+MultiModalUUIDItems: TypeAlias = dict[str, Sequence[str | None]]
697
+"""
698
+As [`MultiModalUUIDDict`][vllm.multimodal.inputs.MultiModalUUIDDict], but
699
+normalized such that each entry corresponds to a list.
700
+"""
701
+
702
+
703
+def parse_mm_uuids(mm_uuids: MultiModalUUIDDict | None) -> MultiModalUUIDItems:
704
+ if mm_uuids is None:
705
+ return {}
706
+
707
+ return {
708
+ modality: [uuids] if isinstance(uuids, str) else uuids
709
+ for modality, uuids in mm_uuids.items()
710
+ }
vllm/multimodal/processing/processor.py
+59
-68
@@ -32,7 +32,6 @@ from ..inputs import (
32
MultiModalKwargsItem,
33
MultiModalKwargsItems,
34
MultiModalKwargsOptionalItems,
35
- MultiModalUUIDDict,
35
PlaceholderRange,
36
mm_enc_dec_inputs,
37
mm_inputs,
@@ -41,6 +40,7 @@ from ..parse import (
40
DictEmbeddingItems,
41
EmbeddingItems,
42
MultiModalDataItems,
43
+ MultiModalUUIDItems,
44
)
45
from .context import (
46
BaseProcessingInfo,
@@ -1014,11 +1014,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1014
self,
1015
prompt: str,
1016
mm_items: MultiModalDataItems,
1017
- hf_processor_mm_kwargs: Mapping[str, object],
1018
- *,
1019
- mm_uuids: MultiModalUUIDDict | None = None,
1017
+ mm_uuid_items: MultiModalUUIDItems | None = None,
1018
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1019
) -> MultiModalInputs:
1021
- return self.apply(prompt, mm_items, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1020
+ return self.apply(
1021
+ prompt,
1022
+ mm_items,
1023
+ mm_uuid_items,
1024
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1025
+ )
1026
1027
@abstractmethod
1028
def _get_mm_fields_config(
@@ -1174,7 +1178,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1178
1179
In addition, return whether prompt updates have been applied.
1180
"""
1177
- processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
1181
+ valid_mm_items = mm_items.select(
1182
+ {k for k, c in mm_items.get_all_counts().items() if c > 0}
1183
+ )
1184
+ processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
1185
1186
processed_data = self._call_hf_processor(
1187
prompt=prompt_text,
@@ -1301,69 +1308,57 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1308
1309
def _hash_mm_items(
1310
self,
1304
- mm_items: MultiModalDataItems,
1311
+ mm_data_items: MultiModalDataItems,
1312
+ mm_uuid_items: MultiModalUUIDItems | None,
1313
hf_processor_mm_kwargs: Mapping[str, object],
1306
- tokenization_kwargs: Mapping[str, object],
1307
- *,
1308
- mm_uuids: MultiModalUUIDDict | None = None,
1314
) -> MultiModalHashes:
1310
- """Create MM hashes to be returned.
1311
-
1312
-
1313
- Note: When overrides are provided via callers of `apply`,
1314
- `_hash_mm_items` will be bypassed and the overrides will be used.
1315
- """
1315
model_id = self.info.model_id
1316
1318
- hashes: MultiModalHashes = {}
1319
- mm_uuids = mm_uuids or {}
1317
+ if mm_uuid_items is None:
1318
+ mm_uuid_items = {}
1319
1321
- for modality, items in mm_items.items():
1322
- if modality in mm_uuids:
1323
- mm_uuids_per_modality = mm_uuids[modality]
1324
- if isinstance(mm_uuids_per_modality, str):
1325
- mm_uuids_per_modality = [mm_uuids_per_modality]
1320
+ mm_hashes: MultiModalHashes = {}
1321
+ hasher = MultiModalHasher
1322
+
1323
+ for modality, data_items in mm_data_items.items():
1324
+ if modality in mm_uuid_items:
1325
+ uuid_items = mm_uuid_items[modality]
1326
1327
# For None entries, compute a hash; otherwise, use provided ID.
1328
- computed: list[str] = []
1329
- for i, item in enumerate(items.get_all_items_for_hash()):
1330
- item_uuid = mm_uuids_per_modality[i]
1331
-
1332
- # NOTE: Even if a item_uuid is provided, we still compute a
1333
- # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
1334
- # are provided. This is because the processed multimodal
1335
- # inputs can be different depending on the processor kwargs.
1336
- if (
1337
- item_uuid is None
1338
- or hf_processor_mm_kwargs
1339
- or tokenization_kwargs
1340
- ):
1328
+ hashes: list[str] = []
1329
+ for i, item in enumerate(data_items.get_all_items_for_hash()):
1330
+ uuid_item = uuid_items[i]
1331
+
1332
+ # NOTE: Even if a uuid_item is provided, we still compute a hash
1333
+ # if `hf_processor_mm_kwargs` is provided.
1334
+ # This is because the processed multimodal inputs can be different
1335
+ # depending on the processor kwargs.
1336
+ if uuid_item is None or hf_processor_mm_kwargs:
1337
# NOTE: use provided hash string to hash with kwargs
1338
# if available for better performance.
1343
- item = item_uuid if item_uuid is not None else item
1344
- computed.append(
1345
- MultiModalHasher.hash_kwargs(
1339
+ item = uuid_item if uuid_item is not None else item
1340
+ hashes.append(
1341
+ hasher.hash_kwargs(
1342
model_id=model_id,
1343
**{modality: item},
1344
**hf_processor_mm_kwargs,
1349
- **tokenization_kwargs,
1345
)
1346
)
1347
else:
1353
- computed.append(item_uuid)
1354
- hashes[modality] = computed
1348
+ hashes.append(uuid_item)
1349
+
1350
+ mm_hashes[modality] = hashes
1351
else:
1356
- hashes[modality] = [
1357
- MultiModalHasher.hash_kwargs(
1352
+ mm_hashes[modality] = [
1353
+ hasher.hash_kwargs(
1354
model_id=model_id,
1355
**{modality: item},
1356
**hf_processor_mm_kwargs,
1361
- **tokenization_kwargs,
1357
)
1363
- for item in items
1358
+ for item in data_items
1359
]
1360
1366
- return hashes
1361
+ return mm_hashes
1362
1363
def _get_cache_missing_items(
1364
self,
@@ -1468,10 +1463,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1463
self,
1464
prompt: str | list[int],
1465
mm_data_items: MultiModalDataItems,
1466
+ mm_uuid_items: MultiModalUUIDItems | None,
1467
hf_processor_mm_kwargs: Mapping[str, object],
1468
tokenization_kwargs: Mapping[str, object],
1473
- *,
1474
- mm_uuids: MultiModalUUIDDict | None = None,
1469
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1470
(
1471
prompt_ids,
@@ -1494,9 +1488,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1488
with timed_preprocessor_operation(self.info.ctx, "hashing"):
1489
mm_hashes = self._hash_mm_items(
1490
mm_data_items,
1497
- hf_processor_mm_kwargs,
1498
- tokenization_kwargs,
1499
- mm_uuids=mm_uuids,
1491
+ mm_uuid_items,
1492
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1493
)
1494
1495
mm_prompt_updates = self._get_mm_prompt_updates(
@@ -1517,10 +1510,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1510
self,
1511
prompt: str | list[int],
1512
mm_data_items: MultiModalDataItems,
1513
+ mm_uuid_items: MultiModalUUIDItems | None,
1514
hf_processor_mm_kwargs: Mapping[str, object],
1515
tokenization_kwargs: Mapping[str, object],
1522
- *,
1523
- mm_uuids: MultiModalUUIDDict | None = None,
1516
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1517
"""
1518
Apply the HF processor on the full prompt text,
@@ -1533,17 +1525,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1525
return self._apply_hf_processor(
1526
prompt=prompt,
1527
mm_data_items=mm_data_items,
1528
+ mm_uuid_items=mm_uuid_items,
1529
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1530
tokenization_kwargs=tokenization_kwargs,
1538
- mm_uuids=mm_uuids,
1531
)
1532
1533
with timed_preprocessor_operation(self.info.ctx, "hashing"):
1534
mm_hashes = self._hash_mm_items(
1535
mm_data_items,
1544
- hf_processor_mm_kwargs,
1545
- tokenization_kwargs,
1546
- mm_uuids=mm_uuids,
1536
+ mm_uuid_items,
1537
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1538
)
1539
1540
with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
@@ -1753,10 +1744,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1744
self,
1745
prompt: str | list[int],
1746
mm_items: MultiModalDataItems,
1756
- hf_processor_mm_kwargs: Mapping[str, object],
1747
+ mm_uuid_items: MultiModalUUIDItems | None = None,
1748
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1749
tokenization_kwargs: Mapping[str, object] | None = None,
1758
- *,
1759
- mm_uuids: MultiModalUUIDDict | None = None,
1750
) -> MultiModalInputs:
1751
"""
1752
Process multi-modal inputs to be used in vLLM.
@@ -1775,6 +1765,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1765
if request_id is not None:
1766
self.info.ctx.create_timing_stats(request_id)
1767
1768
+ if hf_processor_mm_kwargs is None:
1769
+ hf_processor_mm_kwargs = {}
1770
if tokenization_kwargs is None:
1771
tokenization_kwargs = {}
1772
@@ -1785,9 +1777,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1777
) = self._cached_apply_hf_processor(
1778
prompt,
1779
mm_items,
1780
+ mm_uuid_items,
1781
hf_processor_mm_kwargs,
1782
tokenization_kwargs=tokenization_kwargs,
1790
- mm_uuids=mm_uuids,
1783
)
1784
1785
# NOTE: tokenization_kwargs are not required to init processor
@@ -1861,10 +1853,9 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1853
self,
1854
prompt: str | list[int],
1855
mm_items: MultiModalDataItems,
1864
- hf_processor_mm_kwargs: Mapping[str, object],
1856
+ mm_uuid_items: MultiModalUUIDItems | None = None,
1857
+ hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1858
tokenization_kwargs: Mapping[str, object] | None = None,
1866
- *,
1867
- mm_uuids: MultiModalUUIDDict | None = None,
1859
) -> MultiModalEncDecInputs:
1860
"""
1861
Process multi-modal inputs to be used in vLLM.
@@ -1877,9 +1868,9 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1868
encoder_inputs = super().apply(
1869
encoder_prompt,
1870
mm_items,
1880
- hf_processor_mm_kwargs,
1881
- tokenization_kwargs,
1882
- mm_uuids=mm_uuids,
1871
+ mm_uuid_items,
1872
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1873
+ tokenization_kwargs=tokenization_kwargs,
1874
)
1875
1876
return self._get_enc_dec_inputs(
vllm/renderers/base.py
+37
-37
@@ -51,7 +51,7 @@ if TYPE_CHECKING:
51
MultiModalInputs,
52
MultiModalUUIDDict,
53
)
54
- from vllm.multimodal.parse import MultiModalDataItems
54
+ from vllm.multimodal.parse import MultiModalDataItems, MultiModalUUIDItems
55
from vllm.multimodal.processing import BaseMultiModalProcessor
56
57
logger = init_logger(__name__)
@@ -463,23 +463,25 @@ class BaseRenderer(ABC, Generic[_T]):
463
def _validate_mm_uuids(
464
self,
465
mm_data: "MultiModalDataDict",
466
- mm_items: "MultiModalDataItems",
467
- mm_uuids: "MultiModalUUIDDict | None",
466
+ mm_data_items: "MultiModalDataItems",
467
+ mm_uuid_items: "MultiModalUUIDItems",
468
) -> None:
469
- if mm_uuids is None:
470
- mm_uuids = {}
471
-
472
- # NOTE: Keys corresponding to `None` in `mm_data` don't appear in `mm_items`
473
- modalities = mm_data.keys() | mm_uuids.keys()
469
+ # NOTE: Keys corresponding to `None` in `mm_data` don't appear in
470
+ # `mm_data_items`
471
+ modalities = mm_data.keys() | mm_uuid_items.keys()
472
473
for modality in modalities:
476
- data_items = mm_items.get(modality) or list[Any]()
474
+ data_items = mm_data_items.get(modality)
475
+ uuid_items = mm_uuid_items.get(modality)
476
478
- uuid_items = mm_uuids.get(modality) or list[str | None]()
479
- if isinstance(uuid_items, str):
480
- uuid_items = [uuid_items]
477
+ if data_items is None:
478
+ if uuid_items is None:
479
+ raise ValueError(
480
+ f"multi_modal_data[{modality!r}] is empty but "
481
+ f"multi_modal_uuids[{modality!r}] is missing."
482
+ )
483
482
- if len(data_items) > 0:
484
+ elif uuid_items is not None:
485
if len(uuid_items) > 0 and len(data_items) != len(uuid_items):
486
raise ValueError(
487
f"If given, multi_modal_uuids[{modality!r}] must have "
@@ -488,24 +490,17 @@ class BaseRenderer(ABC, Generic[_T]):
490
)
491
492
for i, item in enumerate(data_items):
491
- if item is None:
492
- if not uuid_items:
493
- raise ValueError(
494
- f"multi_modal_data[{modality!r}][{i}] is empty but "
495
- f"multi_modal_uuids[{modality!r}] is missing."
496
- )
497
-
498
- if uuid_items[i] is None:
499
- raise ValueError(
500
- f"multi_modal_data[{modality!r}][{i}] is empty but "
501
- f"multi_modal_uuids[{modality!r}][{i}] is missing."
502
- )
493
+ if item is None and uuid_items[i] is None:
494
+ raise ValueError(
495
+ f"multi_modal_data[{modality!r}][{i}] is empty but "
496
+ f"multi_modal_uuids[{modality!r}][{i}] is missing."
497
+ )
498
499
def _process_mm_uuids(
500
self,
501
mm_data: "MultiModalDataDict",
507
- mm_items: "MultiModalDataItems",
508
- mm_uuids: "MultiModalUUIDDict | None",
502
+ mm_data_items: "MultiModalDataItems",
503
+ mm_uuid_items: "MultiModalUUIDItems",
504
mm_req_id: str,
505
):
506
model_config = self.model_config
@@ -520,40 +515,45 @@ class BaseRenderer(ABC, Generic[_T]):
515
and model_config.multimodal_config.mm_processor_cache_gb == 0
516
and not self.config.cache_config.enable_prefix_caching
517
):
523
- mm_uuids = {
518
+ mm_uuid_items = {
519
modality: [f"{mm_req_id}-{modality}-{i}" for i in range(data_count)]
525
- for modality, data_count in mm_items.get_all_counts().items()
520
+ for modality, data_count in mm_data_items.get_all_counts().items()
521
}
522
528
- self._validate_mm_uuids(mm_data, mm_items, mm_uuids)
523
+ self._validate_mm_uuids(mm_data, mm_data_items, mm_uuid_items)
524
530
- return mm_uuids
525
+ return mm_uuid_items
526
527
# TODO: Remove str and tokenization_kwargs after deprecating InputPreprocessor
528
def _process_multimodal(
529
self,
530
prompt: list[int] | str,
531
mm_data: "MultiModalDataDict",
532
+ mm_uuids: "MultiModalUUIDDict | None",
533
mm_processor_kwargs: Mapping[str, object] | None,
534
tokenization_kwargs: dict[str, Any] | None,
539
- mm_uuids: "MultiModalUUIDDict | None",
535
) -> "MultiModalInputs":
536
+ from vllm.multimodal.parse import parse_mm_uuids
537
from vllm.multimodal.processing.context import set_request_id
538
539
mm_req_id = f"renderer-mm-{self._mm_req_counter.inc(1)}"
540
541
mm_processor = self.get_mm_processor()
542
547
- mm_items = mm_processor.info.parse_mm_data(mm_data)
548
- mm_uuids = self._process_mm_uuids(mm_data, mm_items, mm_uuids, mm_req_id)
543
+ mm_data_items = mm_processor.info.parse_mm_data(mm_data)
544
+ mm_uuid_items = parse_mm_uuids(mm_uuids)
545
+
546
+ mm_uuids = self._process_mm_uuids(
547
+ mm_data, mm_data_items, mm_uuid_items, mm_req_id
548
+ )
549
550
with set_request_id(mm_req_id), set_default_torch_num_threads():
551
mm_inputs = mm_processor.apply(
552
prompt,
553
- mm_items,
554
- hf_processor_mm_kwargs=mm_processor_kwargs or {},
553
+ mm_data_items,
554
+ mm_uuid_items,
555
+ hf_processor_mm_kwargs=mm_processor_kwargs,
556
tokenization_kwargs=tokenization_kwargs,
556
- mm_uuids=mm_uuids,
557
)
558
559
self.update_mm_cache_stats()