@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 6e85feb1b7

[3/N] Harden Transformers modelling backend multi-modal path (#51827)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Aug 20, 2026 at 11:24 UTC 6e85feb1b7592e2bb73ff9227b2e529b8a270ec2
4 files changed +838 -161
tests/models/multimodal/processing/test_transformers_audio.py
+31 -31
@@ -4,8 +4,11 @@ import numpy as np
4 import pytest
5
6 from vllm.config import ModelConfig
7 +from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
8 from vllm.multimodal import MULTIMODAL_REGISTRY
9
10 +from .transformers_backend import PROCESSOR_CLASSES, create_processor
11 +
12 AUDIO_MODEL_SETTINGS = {
13 "ibm-granite/granite-speech-3.3-2b": {
14 "prompt": (
@@ -53,6 +56,7 @@ AUDIO_MODEL_SETTINGS = {
56 }
57
58
59 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
60 @pytest.mark.parametrize(
61 "model_id",
62 [
@@ -61,12 +65,12 @@ AUDIO_MODEL_SETTINGS = {
65 pytest.param(
66 "mistralai/Voxtral-Mini-3B-2507",
67 marks=pytest.mark.xfail(
64 - reason="MistralCommonBackend.encode does not produce the audio "
65 - "placeholder token (ID 24) from raw text. apply_chat_template "
66 - "yields token IDs with placeholders, but MultiModalProcessor."
67 - "apply() decodes the prompt back to text and re-tokenizes, at "
68 - "which point the placeholders are lost. Fix belongs in "
69 - "mistral_common or in the Voxtral-specific path.",
68 + reason="Voxtral's mistral_common processor does not compose with "
69 + "the Transformers modelling backend. Loading it currently fails "
70 + "outright, because MistralCommonBackend.from_pretrained rejects "
71 + "the kwargs vLLM passes, and it implements no "
72 + "`replace_audio_token`, so it would report no replacement "
73 + "offsets. Both fixes belong in mistral_common or transformers.",
74 strict=False,
75 ),
76 ),
@@ -74,15 +78,10 @@ AUDIO_MODEL_SETTINGS = {
78 "zai-org/GLM-ASR-Nano-2512",
79 ],
80 )
77 -def test_audio_multimodal_processor(model_id):
81 +def test_audio_multimodal_processor(model_id, processor_cls):
82 settings = AUDIO_MODEL_SETTINGS[model_id]
83
80 - model_config = ModelConfig(
81 - model=model_id,
82 - model_impl="transformers",
83 - )
84 -
85 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
84 + mm_processor = create_processor(model_id, processor_cls)
85
86 audio = np.zeros(16000, dtype=np.float32)
87 mm_data = {"audio": (audio, 16000)}
@@ -113,10 +112,13 @@ def test_audio_multimodal_processor(model_id):
112 )
113
114
116 -def _process_granite_speech(separator: str):
115 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
116 +@pytest.mark.parametrize("separator", [" and ", ""])
117 +def test_audio_multiple_inputs(separator, processor_cls):
118 + """Multiple audios per prompt are each detected as a separate placeholder
119 + and multi-modal item by the Transformers modelling backend."""
120 model_id = "ibm-granite/granite-speech-3.3-2b"
118 - model_config = ModelConfig(model=model_id, model_impl="transformers")
119 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
121 + mm_processor = create_processor(model_id, processor_cls)
122
123 audio_token = mm_processor.info.get_hf_processor().audio_token
124 # One token per audio; the processor expands each to its placeholder run.
@@ -126,17 +128,21 @@ def _process_granite_speech(separator: str):
128 )
129 audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
130
129 - return mm_processor(
130 - prompt=prompt,
131 - mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
132 - hf_processor_mm_kwargs={},
133 - )
131 + def process():
132 + return mm_processor(
133 + prompt=prompt,
134 + mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
135 + hf_processor_mm_kwargs={},
136 + )
137
138 + # The legacy path reads placeholders off contiguous runs of the audio token, so
139 + # it cannot tell adjacent ones apart and says so instead of merging them
140 + if processor_cls is LegacyMultiModalProcessor and not separator:
141 + with pytest.raises(ValueError, match="Separate them in the prompt"):
142 + process()
143 + return
144
136 -def test_audio_multiple_inputs():
137 - """Multiple audios per prompt are each detected as a separate placeholder
138 - and multi-modal item by the Transformers modelling backend."""
139 - result = _process_granite_speech(separator=" and ")
145 + result = process()
146
147 assert len(result["mm_placeholders"]["audio"]) == 2
148 assert len(result["mm_kwargs"]["audio"]) == 2
@@ -167,9 +173,3 @@ def test_unclaimed_fields_warn_rather_than_raise():
173
174 assert owned["audio"] == ["input_features"]
175 assert owned["image"] == []
170 -
171 -
172 -def test_audio_adjacent_inputs():
173 - """Adjacent audios are rejected rather than silently merged into one placeholder."""
174 - with pytest.raises(ValueError, match="told apart"):
175 - _process_granite_speech(separator="")
tests/models/multimodal/processing/test_transformers_image.py
+228 -19
@@ -1,20 +1,27 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from unittest.mock import patch
4 +
5 import pytest
6
7 from vllm.assets.image import ImageAsset
6 -from vllm.config import ModelConfig
7 -from vllm.multimodal import MULTIMODAL_REGISTRY
8 +from vllm.model_executor.models.transformers.multimodal import (
9 + LegacyMultiModalProcessor,
10 + OffsetsMultiModalProcessor,
11 +)
12
13 +from .transformers_backend import (
14 + PROCESSOR_CLASSES,
15 + create_cached_processor,
16 + create_processor,
17 + offsets_only,
18 +)
19
10 -@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
11 -def test_multimodal_processor(model_id):
12 - model_config = ModelConfig(
13 - model=model_id,
14 - model_impl="transformers",
15 - )
20
17 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
21 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
22 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
23 +def test_multimodal_processor(model_id, processor_cls):
24 + mm_processor = create_processor(model_id, processor_cls)
25
26 image_pil = ImageAsset("cherry_blossom").pil_image
27 mm_data = {"image": image_pil}
@@ -56,10 +63,9 @@ def test_multimodal_processor(model_id):
63 )
64
65
59 -def _process_two_images(separator: str):
66 +def _process_two_images(processor_cls, separator: str):
67 model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
61 - model_config = ModelConfig(model=model_id, model_impl="transformers")
62 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
68 + mm_processor = create_processor(model_id, processor_cls)
69
70 image = ImageAsset("cherry_blossom").pil_image
71 prompt = (
@@ -74,28 +80,147 @@ def _process_two_images(separator: str):
80 )
81
82
77 -def test_image_multiple_inputs():
83 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
84 +def test_image_multiple_inputs(processor_cls):
85 """Multiple images per prompt are each detected as a separate placeholder
86 and multi-modal item by the Transformers modelling backend."""
80 - result = _process_two_images(separator="\n and ")
87 + result = _process_two_images(processor_cls, separator="\n and ")
88
89 assert len(result["mm_placeholders"]["image"]) == 2
90 assert len(result["mm_kwargs"]["image"]) == 2
91
92
86 -def test_image_adjacent_inputs():
93 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
94 +def test_image_adjacent_inputs(processor_cls):
95 """Adjacent images stay separate placeholders rather than merging into one."""
88 - result = _process_two_images(separator="")
96 + result = _process_two_images(processor_cls, separator="")
97
98 assert len(result["mm_placeholders"]["image"]) == 2
99 assert len(result["mm_kwargs"]["image"]) == 2
100
101
94 -def test_text_only_prompt():
102 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
103 +def test_batch_padding_removed_from_image_items(processor_cls):
104 + """Emu3 pads every image up to the largest in the batch, which would leave an
105 + item's data dependent on what it was processed with and so uncacheable."""
106 + mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
107 + image_token = mm_processor.info.get_hf_processor().image_token
108 +
109 + images = [
110 + ImageAsset("cherry_blossom").pil_image,
111 + ImageAsset("cherry_blossom").pil_image.resize((256, 1024)),
112 + ]
113 + result = mm_processor(
114 + prompt=f"{image_token} and {image_token}",
115 + mm_items=mm_processor.info.parse_mm_data({"image": images}),
116 + hf_processor_mm_kwargs={},
117 + )
118 +
119 + items = result["mm_kwargs"]["image"]
120 + shapes = set()
121 + for item in items:
122 + height, width = item["image_sizes"].data.flatten().tolist()
123 + pixel_values = item["pixel_values"].data
124 + assert tuple(pixel_values.shape[-2:]) == (height, width)
125 + shapes.add(tuple(pixel_values.shape))
126 +
127 + # Both images would have been padded to a common shape had they been kept
128 + assert len(shapes) == 2
129 +
130 +
131 +def _process_one_gemma3_image(processor_cls):
132 + mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
133 + hf_processor = mm_processor.info.get_hf_processor()
134 + result = mm_processor(
135 + prompt=f"{hf_processor.boi_token} What is this?",
136 + mm_items=mm_processor.info.parse_mm_data(
137 + {"image": ImageAsset("cherry_blossom").pil_image}
138 + ),
139 + hf_processor_mm_kwargs={},
140 + )
141 + return hf_processor, result
142 +
143 +
144 +@offsets_only
145 +def test_non_embedding_tokens_excluded_from_placeholders():
146 + """Gemma3 wraps each image in text that carries no embeddings, which must be
147 + inside the placeholder range but masked out of it."""
148 + _, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
149 +
150 + (placeholder,) = result["mm_placeholders"]["image"]
151 + assert placeholder.is_embed is not None
152 + assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
153 +
154 +
155 +def test_legacy_placeholders_hold_only_image_tokens():
156 + """The legacy path spans whatever `mm_token_type_ids` attributes to the image,
157 + which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
158 + path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
159 + marker in the unexpanded prompt, not the token the expansion repeats."""
160 + hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
161 +
162 + (placeholder,) = result["mm_placeholders"]["image"]
163 + assert placeholder.length == hf_processor.image_seq_length
164 + prompt_ids = result["prompt_token_ids"]
165 + covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
166 + assert set(covered) == {hf_processor.tokenizer.image_token_id}
167 +
168 +
169 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
170 +def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
171 + """SmolVLM splits each image into tiles introduced by tokens carrying no
172 + embeddings. Those belong inside the placeholder and masked out, because the token
173 + count the processor reports is over the whole span. Idefics3 also refuses a prompt
174 + holding `<image>` when no images are passed, which is how the offsets path has to
175 + tokenize it before splicing in the expansion."""
176 + mm_processor = create_processor(
177 + "HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
178 + )
179 + result = mm_processor(
180 + prompt="<image>What is this?",
181 + mm_items=mm_processor.info.parse_mm_data(
182 + {"image": ImageAsset("cherry_blossom").pil_image}
183 + ),
184 + hf_processor_mm_kwargs={},
185 + )
186 +
187 + (placeholder,) = result["mm_placeholders"]["image"]
188 + assert placeholder.is_embed is not None
189 + assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
190 +
191 +
192 +@offsets_only
193 +def test_missing_replacement_offsets_names_the_processor():
194 + """A processor that reports no replacement offsets cannot be served, which must
195 + be said plainly rather than surfacing later as a field config mismatch."""
196 + model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
197 + mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
198 + hf_processor_cls = type(mm_processor.info.get_hf_processor())
199 + hf_call = hf_processor_cls.__call__
200 +
201 + def without_offsets(self, *args, **kwargs):
202 + hf_inputs = hf_call(self, *args, **kwargs)
203 + hf_inputs.pop("text_replacement_offsets", None)
204 + return hf_inputs
205 +
206 + with (
207 + patch.object(hf_processor_cls, "__call__", without_offsets),
208 + pytest.raises(ValueError, match="LlavaOnevisionProcessor returned no"),
209 + ):
210 + mm_processor(
211 + prompt="<image>\nWhat is the content of this image?",
212 + mm_items=mm_processor.info.parse_mm_data(
213 + {"image": ImageAsset("cherry_blossom").pil_image}
214 + ),
215 + hf_processor_mm_kwargs={},
216 + )
217 +
218 +
219 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
220 +def test_text_only_prompt(processor_cls):
221 """An image model still accepts a prompt with no images."""
222 model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
97 - model_config = ModelConfig(model=model_id, model_impl="transformers")
98 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
223 + mm_processor = create_processor(model_id, processor_cls)
224
225 result = mm_processor(
226 prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
@@ -105,3 +230,87 @@ def test_text_only_prompt():
230
231 assert len(result["prompt_token_ids"]) > 0
232 assert not result["mm_placeholders"]
233 +
234 +
235 +@offsets_only
236 +def test_repeated_image_hits_the_processor_cache():
237 + """Check that mm caching is actually working."""
238 + mm_processor, cache = create_cached_processor(
239 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
240 + )
241 + image = ImageAsset("cherry_blossom").pil_image
242 +
243 + def process():
244 + return mm_processor(
245 + prompt="<image>\nWhat is this?",
246 + mm_items=mm_processor.info.parse_mm_data({"image": image}),
247 + hf_processor_mm_kwargs={},
248 + )
249 +
250 + first, second = process(), process()
251 +
252 + assert cache.make_stats().hits > 0
253 + assert first["prompt_token_ids"] == second["prompt_token_ids"]
254 + assert first["mm_hashes"] == second["mm_hashes"]
255 +
256 +
257 +@offsets_only
258 +@pytest.mark.parametrize(
259 + ("model_id", "prompt"),
260 + [
261 + ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "<image>\nWhat is this?"),
262 + ("google/gemma-3-4b-it", "<start_of_image> What is this?"),
263 + ("HuggingFaceTB/SmolVLM-256M-Instruct", "<image>What is this?"),
264 + pytest.param(
265 + "BAAI/Emu3-Chat-hf",
266 + "<image> and more text",
267 + marks=pytest.mark.xfail(
268 + reason="Emu3Processor prepends its BOS token only when images are "
269 + "passed, so the unexpanded prompt the offsets path tokenizes never "
270 + "gets one. Fixed by huggingface/transformers#47924, unreleased.",
271 + strict=False,
272 + ),
273 + ),
274 + ],
275 +)
276 +def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
277 + """The offsets path splices the expansion into a prompt tokenized without any
278 + multi-modal data, so its token ids have to come out the same as the ones the HF
279 + processor produces itself, which is what the legacy path returns."""
280 + prompt_ids = []
281 + for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
282 + mm_processor = create_processor(model_id, processor_cls)
283 + prompt_ids.append(
284 + mm_processor(
285 + prompt=prompt,
286 + mm_items=mm_processor.info.parse_mm_data(
287 + {"image": ImageAsset("cherry_blossom").pil_image}
288 + ),
289 + hf_processor_mm_kwargs={},
290 + )["prompt_token_ids"]
291 + )
292 +
293 + legacy_ids, offsets_ids = prompt_ids
294 + assert legacy_ids == offsets_ids
295 +
296 +
297 +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
298 +def test_nested_image_fields_split_per_image(processor_cls):
299 + """Idefics3 returns image fields with a leading batch dimension, putting the rows
300 + belonging to each image one dimension further in. Slicing the batch dimension
301 + instead handed the first image every row and the second an empty tensor."""
302 + mm_processor = create_processor(
303 + "HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
304 + )
305 + image = ImageAsset("cherry_blossom").pil_image
306 + result = mm_processor(
307 + prompt="<image> and <image>",
308 + mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
309 + hf_processor_mm_kwargs={},
310 + )
311 +
312 + items = result["mm_kwargs"]["image"]
313 + assert len(items) == 2
314 + for item in items:
315 + pixel_values = item["pixel_values"].data
316 + assert pixel_values.shape[1] == int(item["num_image_patches"].data)
tests/models/multimodal/processing/transformers_backend.py new
+51
@@ -0,0 +1,51 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Helpers for testing both processors of the Transformers modelling backend."""
4 +
5 +import pytest
6 +
7 +from vllm.config import ModelConfig
8 +from vllm.model_executor.models.transformers.multimodal import (
9 + LegacyMultiModalProcessor,
10 + MultiModalDummyInputsBuilder,
11 + MultiModalProcessingInfo,
12 + MultiModalProcessor,
13 + OffsetsMultiModalProcessor,
14 +)
15 +from vllm.multimodal.cache import MultiModalProcessorOnlyCache
16 +from vllm.multimodal.processing import InputProcessingContext
17 +from vllm.tokenizers.registry import cached_tokenizer_from_config
18 +
19 +offsets_only = pytest.mark.skipif(
20 + MultiModalProcessor is not OffsetsMultiModalProcessor,
21 + reason="Replacement offsets are only used from transformers 5.15.0 onwards",
22 +)
23 +
24 +PROCESSOR_CLASSES = [
25 + pytest.param(LegacyMultiModalProcessor, id="legacy"),
26 + pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
27 +]
28 +
29 +
30 +def create_processor(model_id: str, processor_cls):
31 + """Build a processor directly, because the registry only ever builds the one the
32 + installed transformers version selects, leaving the other path untested."""
33 + model_config = ModelConfig(model=model_id, model_impl="transformers")
34 + ctx = InputProcessingContext(
35 + model_config, cached_tokenizer_from_config(model_config)
36 + )
37 + info = MultiModalProcessingInfo(ctx)
38 + return processor_cls(info, MultiModalDummyInputsBuilder(info))
39 +
40 +
41 +def create_cached_processor(model_id: str, processor_cls):
42 + """Build a processor backed by a real multi-modal processor cache, and hand the
43 + cache back so a test can check it was actually used."""
44 + model_config = ModelConfig(model=model_id, model_impl="transformers")
45 + model_config.multimodal_config.mm_processor_cache_gb = 4
46 + ctx = InputProcessingContext(
47 + model_config, cached_tokenizer_from_config(model_config)
48 + )
49 + info = MultiModalProcessingInfo(ctx)
50 + cache = MultiModalProcessorOnlyCache(model_config)
51 + return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache
vllm/model_executor/models/transformers/multimodal.py
+528 -111
@@ -22,6 +22,8 @@ from contextlib import ExitStack, contextmanager
22 from typing import TYPE_CHECKING, Any
23
24 import torch
25 +import transformers
26 +from packaging.version import Version
27
28 from vllm.compilation.decorators import should_torch_compile_mm_encoder
29 from vllm.config.utils import getattr_iter
@@ -50,7 +52,9 @@ from vllm.multimodal.processing import (
52 BaseMultiModalProcessor,
53 BaseProcessingInfo,
54 ProcessorInputs,
55 + PromptReplacement,
56 PromptUpdate,
57 + PromptUpdateDetails,
58 TimingContext,
59 )
60 from vllm.sequence import IntermediateTensors
@@ -65,6 +69,12 @@ if TYPE_CHECKING:
69 logger = init_logger(__name__)
70
71 _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
72 +_MODALITY_SIZE_KEYS = {"audio": "num_audio_tokens", "image": "num_image_patches"}
73 +
74 +
75 +def _get_embed_token_id(replacement_ids: torch.Tensor) -> int:
76 + """The token an expansion repeats is the one holding the embeddings."""
77 + return int(replacement_ids.mode().values)
78
79
80 class MultiModalProcessingInfo(BaseProcessingInfo):
@@ -94,21 +104,6 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
104 )
105 return modalities
106
97 - def _get_audio_token_id(self) -> int:
98 - processor = self.get_hf_processor()
99 - if hasattr(processor, "audio_token_id"):
100 - return processor.audio_token_id
101 - config = self.get_hf_config()
102 - val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
103 - if val is not None:
104 - return val
105 - if hasattr(processor, "audio_token"):
106 - tokenizer = self.get_tokenizer()
107 - vocab = tokenizer.get_vocab()
108 - if processor.audio_token in vocab:
109 - return vocab[processor.audio_token]
110 - raise ValueError("Cannot find audio_token_id on processor or model config")
111 -
107 def _get_audio_sampling_rate(self) -> float:
108 sub = self._get_audio_processor()
109 if sub is not None and hasattr(sub, "sampling_rate"):
@@ -169,7 +164,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
164 if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
165 processor = self.info.get_hf_processor()
166 audio_token = getattr(processor, "audio_token", "")
172 - # Separated so that `_apply_audio` can tell the placeholders apart
167 + # Separated so that adjacent placeholders stay distinguishable
168 text += " ".join([audio_token] * num_audios)
169 if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
170 processor = self.info.get_hf_processor()
@@ -210,23 +205,26 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
205 return data
206
207
213 -class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
214 - def _get_prompt_updates(
208 +class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
209 + """Processing common to both Transformers backend processors: calling the HF
210 + processor, sizing images, and attributing its outputs to a modality.
211 +
212 + Subclasses add the strategy for locating placeholders in the prompt.
213 + """
214 +
215 + def _get_hf_mm_data(
216 self,
217 mm_items: MultiModalDataItems,
217 - hf_processor_mm_kwargs: Mapping[str, object],
218 - out_mm_kwargs: MultiModalKwargsItems,
219 - ) -> Sequence[PromptUpdate]:
220 - """No updates: `apply` locates placeholders via `mm_token_type_ids` instead.
221 -
222 - HF processors have no generic contract for the token sequence they insert,
223 - so it cannot be expressed as a `PromptUpdate`. Returning nothing is only
224 - safe because `apply` is overridden; the base class would reject it.
225 - """
226 - return []
218 + ) -> tuple[Mapping[str, object], Mapping[str, object]]:
219 + """Rename the parser's `audios` key to the `audio` argument HF audio
220 + processors take."""
221 + processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
222 + if self.info._is_audio_model() and "audios" in processor_data:
223 + processor_data["audio"] = processor_data.pop("audios")
224 + return processor_data, passthrough_data
225
226 def _get_modality_field_names(self, modality: str) -> set[str]:
229 - """Field names the sub-processor for `modality` produces."""
227 + """Names of the fields the sub-processor for `modality` produces."""
228 # TODO: use else branch only once huggingface/transformers#44394 lands.
229 if modality == "audio":
230 sub_processor = self.info._get_audio_processor()
@@ -273,6 +271,18 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
271
272 return owned
273
274 + def _get_slice_dim(self, data: Any, total_rows: int) -> int:
275 + """Which dimension of a field holds the rows belonging to each item.
276 +
277 + Some processors (e.g., Idefics3) return image fields with a leading batch
278 + dimension, putting the rows one dimension further in.
279 + """
280 + if not isinstance(data, torch.Tensor) or data.ndim < 2:
281 + return 0
282 + if data.shape[0] != total_rows and data.shape[1] == total_rows:
283 + return 1
284 + return 0
285 +
286 def _get_mm_fields_config(
287 self,
288 hf_inputs: "BatchFeature",
@@ -281,29 +291,50 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
291 # HF Processors always return a mask but vLLM doesn't need it
292 hf_inputs.pop("attention_mask", None)
293
284 - # Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
294 + # Absent if the modality had no items
295 sizes = {
286 - "audio": hf_inputs.get("num_audio_tokens"),
287 - "image": hf_inputs.get("num_image_patches"),
296 + modality: hf_inputs.get(key)
297 + for modality, key in _MODALITY_SIZE_KEYS.items()
298 }
299 modalities = [m for m, size in sizes.items() if size is not None]
300
291 - size_keys = {"num_audio_tokens", "num_image_patches"}
292 - keys = [key for key in hf_inputs if key not in size_keys]
301 + # Keys we wrote ourselves, rather than ones a sub-processor produced
302 + own_keys = set(_MODALITY_SIZE_KEYS.values()) | {
303 + f"{modality}_replacement_{suffix}"
304 + for modality in modalities
305 + for suffix in ("ids", "sizes")
306 + }
307 + keys = [key for key in hf_inputs if key not in own_keys]
308 owned = self._partition_keys_by_modality(keys, modalities)
309
310 + # Un-padded fields are already one entry per item, so index rather than slice
311 mm_fields: dict[str, MultiModalFieldConfig] = {
296 - key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
312 + key: MultiModalFieldConfig.batched(modality)
313 + if modality == "audio" or isinstance(hf_inputs[key], list)
314 + else MultiModalFieldConfig.flat_from_sizes(
315 + modality,
316 + sizes[modality],
317 + dim=self._get_slice_dim(hf_inputs[key], int(sizes[modality].sum())),
318 + )
319 for modality in modalities
320 for key in owned[modality]
321 }
322
301 - # Keep these as batched, as they always have batch size as first dim
302 - if "audio" in modalities:
303 - mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
304 - "audio", keep_on_cpu=True
323 + for modality in modalities:
324 + # One row per item, and only ever read on the CPU
325 + mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched(
326 + modality, keep_on_cpu=True
327 )
328 + replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes")
329 + if replacement_sizes is not None:
330 + mm_fields[f"{modality}_replacement_ids"] = (
331 + MultiModalFieldConfig.flat_from_sizes(
332 + modality, replacement_sizes, keep_on_cpu=True
333 + )
334 + )
335 +
336 if "image" in modalities:
337 + # Always one row per item, whatever they describe
338 mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
339 "image", keep_on_cpu=True
340 )
@@ -311,25 +342,154 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
342 mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
343 "image", keep_on_cpu=True
344 )
314 - mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
315 - "image", keep_on_cpu=True
316 - )
345 +
346 return mm_fields
347
319 - def _get_hf_mm_data(
348 + def _call_hf_processor(
349 self,
321 - mm_items: MultiModalDataItems,
322 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:
350 + prompt: str,
351 + mm_data: Mapping[str, object],
352 + mm_kwargs: Mapping[str, object],
353 + tok_kwargs: Mapping[str, object],
354 + ) -> "BatchFeature":
355 + """Run the HF processor and unpad inputs."""
356 + try:
357 + hf_inputs = super()._call_hf_processor(
358 + prompt, mm_data, mm_kwargs, tok_kwargs
359 + )
360 + except ValueError:
361 + if any(mm_data.values()):
362 + raise
363 + # Some processors reject a prompt holding placeholders with
364 + # no data to go with them, so tokenize it without them
365 + prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
366 + hf_inputs = transformers.BatchFeature(
367 + dict(input_ids=[prompt_ids]), tensor_type="pt"
368 + )
369 + self._unpad_images(hf_inputs)
370 + self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
371 + return hf_inputs
372 +
373 + def _unpad_audios(
374 + self,
375 + hf_inputs: "BatchFeature",
376 + mm_data: Mapping[str, object],
377 + mm_kwargs: Mapping[str, object],
378 + tok_kwargs: Mapping[str, object],
379 + ) -> None:
380 + """Replace the audio fields with each audio processed on its own.
381 +
382 + Processors pad every audio up to the longest in the call, which would leave
383 + an item's data dependent on what it was processed with. Unlike images,
384 + nothing in the output states how long each one really is, and processors
385 + pad a lone audio too, so the only way to know what an audio produces by
386 + itself is to process it by itself.
387 + """
388 + audios = mm_data.get("audio")
389 + if not audios or len({len(audio) for audio in audios}) == 1:
390 + return
391 +
392 + alone = [
393 + self.info.ctx.call_hf_processor(
394 + self.info.get_hf_processor(**mm_kwargs),
395 + dict(
396 + text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
397 + ),
398 + dict(**mm_kwargs, **tok_kwargs),
399 + )
400 + for audio in audios
401 + ]
402 +
403 + for key in self._get_modality_field_names("audio"):
404 + if isinstance(hf_inputs.get(key), torch.Tensor):
405 + hf_inputs[key] = [output[key][0] for output in alone]
406 +
407 + def _unpad_images(self, hf_inputs: "BatchFeature") -> None:
408 + """Trim each image back to its own size when the processor padded them all
409 + to the largest in the batch.
410 +
411 + An image's data has to depend on nothing but that image, or the multi-modal
412 + processor cache would store it under that image's hash and later reuse it
413 + beside a different neighbour. Padding is re-applied when the encoder runs.
414 """
324 - In contrast to the base class, this method requests
325 - `return_mm_token_type_ids` and remaps the `audios` key to `audio` for
326 - audio models.
415 + pixel_values = hf_inputs.get("pixel_values")
416 + image_sizes = hf_inputs.get("image_sizes")
417 + if not isinstance(pixel_values, torch.Tensor):
418 + return
419 + if not isinstance(image_sizes, torch.Tensor):
420 + return
421 + if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes):
422 + return
423 +
424 + # The sizes describe the trailing dimensions only if the largest of them is
425 + # what the batch was padded up to. Otherwise they mean something else, as
426 + # in llava-onevision, where they are the sizes before any processing.
427 + maxima = image_sizes.max(dim=0).values
428 + if maxima.tolist() != list(pixel_values.shape[-2:]):
429 + return
430 +
431 + hf_inputs["pixel_values"] = [
432 + image[..., :height, :width]
433 + for image, (height, width) in zip(pixel_values, image_sizes.tolist())
434 + ]
435 +
436 +
437 +class LegacyMultiModalProcessor(_MultiModalProcessorBase):
438 + """Locates placeholders by searching the prompt the HF processor has already
439 + expanded for the tokens of each modality.
440 +
441 + Serves transformers versions with no `return_text_replacement_offsets`.
442 + Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
443 + this processor overrides `apply` and gets no multi-modal processor cache.
444 + Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
445 + """
446 +
447 + def _get_prompt_updates(
448 + self,
449 + mm_items: MultiModalDataItems,
450 + hf_processor_mm_kwargs: Mapping[str, object],
451 + out_mm_kwargs: MultiModalKwargsItems,
452 + ) -> Sequence[PromptUpdate]:
453 + """Empty, because `apply` writes the placeholder ranges itself rather than
454 + deriving them from updates."""
455 + return []
456 +
457 + def _call_hf_processor(
458 + self,
459 + prompt: str,
460 + mm_data: Mapping[str, object],
461 + mm_kwargs: Mapping[str, object],
462 + tok_kwargs: Mapping[str, object],
463 + ) -> "BatchFeature":
464 + """Ask which modality owns each token of the expanded prompt, which is the
465 + only marking of the tokens an expansion adds around an item."""
466 + if any(mm_data.values()):
467 + mm_data = {**mm_data, "return_mm_token_type_ids": True}
468 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
469 +
470 + def _get_mm_token_ids(self, modality: str) -> list[int]:
471 + """Token ids marking where `modality` sits in the prompt, which for some
472 + processors differ from the placeholder written into it.
473 +
474 + The expanded prompt is all this path has to go on, so it takes the
475 + processor at its word about which tokens belong to the modality.
476 """
328 - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
329 - if self.info._is_audio_model() and "audios" in processor_data:
330 - processor_data["audio"] = processor_data.pop("audios")
331 - processor_data["return_mm_token_type_ids"] = True
332 - return processor_data, passthrough_data
477 + info = self.info
478 + processor = info.get_hf_processor()
479 + declared = getattr(processor, f"{modality}_token_ids", None) or ()
480 + if ids := [token_id for token_id in declared if token_id is not None]:
481 + return ids
482 + config = info.get_hf_config()
483 + names = (f"{modality}_token_id", f"{modality}_token_index")
484 + token_id = getattr(processor, names[0], getattr_iter(config, names))
485 + if token_id is None:
486 + token = getattr(processor, f"{modality}_token", None)
487 + token_id = info.get_tokenizer().get_vocab().get(token)
488 + if token_id is None:
489 + raise ValueError(
490 + f"Cannot find {modality}_token_id on processor or model config"
491 + )
492 + return [token_id]
493
494 def _apply_audio(
495 self,
@@ -337,12 +497,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
497 processed_data: "BatchFeature",
498 num_audios: int,
499 ) -> dict[str, list[PlaceholderRange]]:
340 - audio_token_id = self.info._get_audio_token_id()
500 + """Take each contiguous run of the audio token as one item's placeholder,
501 + and record how many tokens the run holds."""
502 + audio_token_ids = self._get_mm_token_ids("audio")
503 prompt_tensor = torch.tensor(prompt_ids)
342 - is_audio = prompt_tensor == audio_token_id
504 + is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
505
506 if not is_audio.any():
345 - return {}
507 + raise ValueError(
508 + f"{num_audios} audio item(s) were passed but the prompt "
509 + "contains no audio token. Add one placeholder per audio item."
510 + )
511
512 padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
513 transitions = padded.int().diff()
@@ -365,6 +530,23 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
530 processed_data["num_audio_tokens"] = lengths
531 return {"audio": ranges}
532
533 + def _get_num_multimodal_tokens(
534 + self,
535 + mm_items: MultiModalDataItems,
536 + hf_processor_mm_kwargs: Mapping[str, object],
537 + ) -> Mapping[str, list[int]]:
538 + """Ask the HF processor how many tokens and patches each image expands to."""
539 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
540 + images = mm_items.get_items("image", ImageProcessorItems)
541 + image_sizes = [
542 + (size.height, size.width)
543 + for size in map(images.get_image_size, range(len(images)))
544 + ]
545 + return processor._get_num_multimodal_tokens(
546 + image_sizes=image_sizes,
547 + **self.info.ctx.get_merged_mm_kwargs({}),
548 + )
549 +
550 def _apply_vision(
551 self,
552 prompt_ids: list[int],
@@ -373,47 +555,46 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
555 hf_processor_mm_kwargs: Mapping[str, object],
556 mm_token_type_ids: torch.Tensor | None,
557 ) -> dict[str, list[PlaceholderRange]]:
376 - # Placeholders can't be located without them, so give up rather than guess
377 - if mm_token_type_ids is None:
378 - return {}
379 -
558 + """Split the positions the processor marks as image into one placeholder per
559 + item, sized by the token count it reports for each image."""
560 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
561 + if mm_token_type_ids is None:
562 + raise ValueError(
563 + f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
564 + "the Transformers modeling backend cannot locate the placeholder of "
565 + "each image."
566 + )
567
382 - # We can infer vLLM style placeholder from token type ids, if we split
383 - # it for each input `mm_data`.
384 - mm_positions = torch.where(mm_token_type_ids == 1)[1]
385 - images = mm_items.get_items("image", ImageProcessorItems)
386 - image_sizes = []
387 - for item_idx in range(len(images)):
388 - image_size = images.get_image_size(item_idx)
389 - image_sizes.append((image_size.height, image_size.width))
390 -
391 - mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
392 - image_sizes=image_sizes,
393 - **self.info.ctx.get_merged_mm_kwargs({}),
568 + # These mark the tokens structuring an image as well as the image itself,
569 + # which is what the counts below are over
570 + mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
571 + mm_tokens_per_modality = self._get_num_multimodal_tokens(
572 + mm_items, hf_processor_mm_kwargs
573 )
574
575 mm_placeholders: dict[str, list[PlaceholderRange]] = {}
576 split_sizes = mm_tokens_per_modality["num_image_tokens"]
398 - if split_sizes:
399 - image_token_ids = getattr(hf_processor, "image_token_ids", None)
400 - if image_token_ids is None:
401 - # Transformers <5.10.0
402 - image_token_ids = [hf_processor.image_token_id]
403 - image_token_ids = torch.tensor(
404 - [i for i in image_token_ids if i is not None]
577 + if sum(split_sizes) != len(mm_positions):
578 + raise ValueError(
579 + f"The expanded prompt holds {len(mm_positions)} image token(s) but "
580 + f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
581 + f"across {mm_items.get_count('image')} image item(s)."
582 )
583
407 - chunked_mm_positions = torch.split(mm_positions, split_sizes)
408 - mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
409 - chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
584 + if split_sizes:
585 + image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
586 + mm_tokens = torch.tensor(prompt_ids)[mm_positions]
587 ranges = [
588 PlaceholderRange(
589 offset=positions[0].item(),
590 length=positions.shape[0],
414 - is_embed=torch.isin(mm_tokens, image_token_ids),
591 + # Only some of the span carries embeddings
592 + is_embed=torch.isin(tokens, image_token_ids),
593 + )
594 + for positions, tokens in zip(
595 + torch.split(mm_positions, split_sizes),
596 + torch.split(mm_tokens, split_sizes),
597 )
416 - for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
598 ]
599 mm_placeholders = {"image": ranges}
600
@@ -427,12 +608,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
608 inputs: ProcessorInputs,
609 timing_ctx: TimingContext,
610 ) -> MultiModalInput:
430 - """
431 - Process multi-modal inputs to be used in vLLM.
432 -
433 - Apply HF Processor on prompt text and multi-modal data together,
434 - outputting token IDs and processed tensors.
435 - """
611 + """Process the prompt and every multi-modal item in one HF processor call,
612 + then read the placeholder ranges out of the token ids it returns."""
613 prompt = inputs.prompt
614 mm_items = inputs.mm_data_items
615 hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
@@ -468,7 +645,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
645 self.info.ctx.get_mm_config().mm_hasher_algorithm,
646 )
647
471 - # For gemma3 we check `token_type_ids` as the key
648 + # Gemma3 reports them under the key the model uses for its own token types
649 mm_token_type_ids = processed_data.pop("token_type_ids", None)
650 mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
651
@@ -516,6 +693,201 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
693 )
694
695
696 +class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
697 + """Locates placeholders from the `text_replacement_offsets` the HF processor
698 + reports, expressing each one as a `PromptUpdate`.
699 +
700 + Stating the expansion as an update is what lets it be rebuilt from an
701 + unexpanded prompt, so this processor takes the base class's processing path
702 + and with it the multi-modal processor cache.
703 + """
704 +
705 + def _get_prompt_updates(
706 + self,
707 + mm_items: MultiModalDataItems,
708 + hf_processor_mm_kwargs: Mapping[str, object],
709 + out_mm_kwargs: MultiModalKwargsItems,
710 + ) -> Sequence[PromptUpdate]:
711 + """Replace each modality's placeholder token with the token ids that item's
712 + replacement text encodes to, marking which of them hold embeddings."""
713 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
714 + updates = []
715 + for modality, items in out_mm_kwargs.items():
716 + # Popped so they are neither cached nor sent to the model; the updates
717 + # they produce are cached alongside the item instead
718 + replacements = [
719 + PromptUpdateDetails.select_token_id(
720 + (ids := item.pop(f"{modality}_replacement_ids").data).tolist(),
721 + _get_embed_token_id(ids),
722 + )
723 + for item in items
724 + ]
725 + updates.append(
726 + PromptReplacement(
727 + modality=modality,
728 + target=getattr(hf_processor, f"{modality}_token"),
729 + replacement=replacements.__getitem__,
730 + )
731 + )
732 + return updates
733 +
734 + def _apply_hf_processor_main(
735 + self,
736 + prompt: str | list[int],
737 + mm_items: MultiModalDataItems,
738 + hf_processor_mm_kwargs: Mapping[str, object],
739 + tokenization_kwargs: Mapping[str, object],
740 + *,
741 + enable_hf_prompt_update: bool,
742 + ) -> tuple[list[int], "BatchFeature", bool]:
743 + """Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
744 + in, whatever `enable_hf_prompt_update` asks for.
745 +
746 + This differs from `super()` only for a text prompt with
747 + `enable_hf_prompt_update`, where `super()` would keep the expanded token
748 + ids the HF processor returns. Both routes give the same ids, so forcing
749 + this one costs an extra processor call and buys the guarantee that a
750 + request bypassing the cache is identical to a cached one.
751 +
752 + A text prompt only arrives with `enable_hf_prompt_update` when there is
753 + no multi-modal processor cache. Requests carrying pre-computed
754 + embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
755 + here, so they fail validation shortly afterwards.
756 + """
757 + if isinstance(prompt, str) and enable_hf_prompt_update:
758 + logger.warning_once(
759 + "Disabling the multi-modal processor cache is extra slow with the "
760 + "Transformers modeling backend: the prompt is still tokenized "
761 + "unexpanded and the expansion spliced in, to keep the token ids "
762 + "identical to what the cache produces, which costs an extra HF "
763 + "processor call per request."
764 + )
765 + return super()._apply_hf_processor_main(
766 + prompt,
767 + mm_items,
768 + hf_processor_mm_kwargs,
769 + tokenization_kwargs,
770 + enable_hf_prompt_update=False,
771 + )
772 +
773 + def _get_num_image_patches(
774 + self,
775 + hf_inputs: "BatchFeature",
776 + mm_data: Mapping[str, object],
777 + num_images: int,
778 + ) -> torch.Tensor:
779 + """How many rows of the image fields belong to each image.
780 +
781 + Taken from whichever per-image count the processor reports,
782 + and checked against the data it has to slice.
783 + """
784 + if (grid := hf_inputs.get("image_grid_thw")) is not None:
785 + num_patches = grid.prod(-1)
786 + elif (counts := self._get_num_patches_per_image(mm_data)) is not None:
787 + num_patches = torch.tensor(counts)
788 + else:
789 + num_patches = torch.ones(num_images, dtype=torch.long)
790 +
791 + image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches"))
792 + if isinstance(image_data, torch.Tensor):
793 + total = int(num_patches.sum())
794 + rows = image_data.shape[self._get_slice_dim(image_data, total)]
795 + if rows != total:
796 + raise ValueError(
797 + f"{type(self.info.get_hf_processor()).__name__} returned "
798 + f"{rows} row(s) of image data for {num_images} image(s), which "
799 + f"cannot be split into the {num_patches.tolist()} row(s) per "
800 + "image derived from its outputs, so the rows cannot be "
801 + "attributed to an image. Gemma3 does this when "
802 + "`do_pan_and_scan` crops an image."
803 + )
804 + return num_patches
805 +
806 + def _get_num_patches_per_image(
807 + self, mm_data: Mapping[str, object]
808 + ) -> list[int] | None:
809 + """Ask the HF processor how many rows of image data each image produces."""
810 + images = mm_data.get("images")
811 + if not images:
812 + return None
813 + try:
814 + sizes = [(image.height, image.width) for image in images]
815 + mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens(
816 + image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({})
817 + )
818 + return list(mm_tokens["num_image_patches"])
819 + except (AttributeError, KeyError, TypeError):
820 + return None
821 +
822 + def _call_hf_processor(
823 + self,
824 + prompt: str,
825 + mm_data: Mapping[str, object],
826 + mm_kwargs: Mapping[str, object],
827 + tok_kwargs: Mapping[str, object],
828 + ) -> "BatchFeature":
829 + """Ask for the replacement each placeholder expands to, and record it as
830 + per-item fields: its token ids, and the tokens or patches behind them."""
831 + if has_mm_data := any(mm_data.values()):
832 + mm_data = {**mm_data, "return_text_replacement_offsets": True}
833 + hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
834 + # Drop the inputs the model would reject
835 + hf_inputs.pop("mm_token_type_ids", None)
836 + hf_inputs.pop("token_type_ids", None)
837 +
838 + offsets = hf_inputs.pop("text_replacement_offsets", None)
839 + # Some processors return an empty batch as a tensor rather than a list
840 + if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0:
841 + if has_mm_data:
842 + raise ValueError(
843 + f"{type(self.info.get_hf_processor()).__name__} returned no "
844 + "text replacement offsets, so the Transformers modeling backend "
845 + "cannot locate the placeholder of each item. Its `__call__` has "
846 + "to reach `ProcessorMixin.get_text_with_replacements` with one "
847 + "replacement per item, which usually means implementing "
848 + "`replace_<modality>_token`. Please report this to transformers "
849 + "so it can be fixed, and install `transformers<5.15.0` in the "
850 + "meantime to locate placeholders in the expanded prompt instead."
851 + )
852 + return hf_inputs
853 +
854 + tokenizer = self.info.get_tokenizer()
855 + replacements = defaultdict[str, list[list[int]]](list)
856 + for entry in offsets[0]:
857 + replacements[entry["type"]].append(
858 + tokenizer.encode(entry["replacement"], add_special_tokens=False)
859 + )
860 +
861 + for modality, seqs in replacements.items():
862 + hf_inputs[f"{modality}_replacement_ids"] = torch.tensor(
863 + [token_id for seq in seqs for token_id in seq]
864 + )
865 + hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor(
866 + [len(seq) for seq in seqs]
867 + )
868 + if modality == "image":
869 + hf_inputs["num_image_patches"] = self._get_num_image_patches(
870 + hf_inputs, mm_data, len(seqs)
871 + )
872 + elif modality == "audio":
873 + counts = []
874 + for seq in seqs:
875 + ids = torch.tensor(seq)
876 + counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
877 + hf_inputs["num_audio_tokens"] = torch.tensor(counts)
878 +
879 + return hf_inputs
880 +
881 +
882 +# From this version on, a processor reporting no offsets is an error rather than a
883 +# fallback to searching the expanded prompt
884 +MultiModalProcessor = (
885 + LegacyMultiModalProcessor
886 + if Version(transformers.__version__) < Version("5.15.0")
887 + else OffsetsMultiModalProcessor
888 +)
889 +
890 +
891 class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
892 def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
893 # Skip SupportsMRoPE.__init__ and call the next class in MRO
@@ -733,18 +1105,32 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
1105 kwargs.pop("token_type_ids", None)
1106 kwargs.pop("mm_token_type_ids", None)
1107
736 - # HuggingFace's `get_audio_features` implementations branch on
737 - # per-sample feature lengths internally.
738 - with gpu_sync_allowed():
739 - audio_output = self.model.get_audio_features(
740 - input_features, return_dict=True, **kwargs
741 - )
742 - audio_embeddings = audio_output.pooler_output
743 -
1108 # Per-audio token counts are needed as Python ints to split.
1109 with gpu_sync_allowed():
1110 split_sizes = num_audio_tokens.flatten().tolist()
747 - return self._split_embeddings(audio_embeddings, split_sizes)
1111 + if isinstance(input_features, torch.Tensor):
1112 + # HuggingFace's `get_audio_features` implementations branch on
1113 + # per-sample feature lengths internally.
1114 + with gpu_sync_allowed():
1115 + audio_output = self.model.get_audio_features(
1116 + input_features, return_dict=True, **kwargs
1117 + )
1118 + return self._split_embeddings(audio_output.pooler_output, split_sizes)
1119 +
1120 + # Audios the processor left un-padded arrive as a list once their
1121 + # lengths differ. Encode them one at a time so that none of them is
1122 + # padded to match another.
1123 + embeddings: list[torch.Tensor] = []
1124 + for index, features in enumerate(input_features):
1125 + audio_output = self.model.get_audio_features(
1126 + features.unsqueeze(0),
1127 + return_dict=True,
1128 + **self._select_item_kwargs(kwargs, index, len(input_features)),
1129 + )
1130 + embeddings.extend(
1131 + self._split_embeddings(audio_output.pooler_output, [split_sizes[index]])
1132 + )
1133 + return embeddings
1134
1135 def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
1136 pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
@@ -761,6 +1147,42 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
1147
1148 num_image_patches = kwargs.pop("num_image_patches")
1149
1150 + split_sizes = num_image_patches.flatten().tolist()
1151 + if isinstance(pixel_values, torch.Tensor):
1152 + vision_embeddings = self._get_image_features(pixel_values, **kwargs)
1153 + if isinstance(vision_embeddings, torch.Tensor):
1154 + return self._split_embeddings(vision_embeddings, split_sizes)
1155 + return list(vision_embeddings)
1156 +
1157 + # Images the processor left un-padded arrive as a list once their
1158 + # shapes differ. Encode them one at a time so that none of them is
1159 + # padded to match another.
1160 + embeddings: list[torch.Tensor] = []
1161 + for index, image in enumerate(pixel_values):
1162 + features = self._get_image_features(
1163 + image.unsqueeze(0),
1164 + **self._select_item_kwargs(kwargs, index, len(pixel_values)),
1165 + )
1166 + # Encoders which return one entry per image return a single entry
1167 + if not isinstance(features, torch.Tensor):
1168 + features = torch.cat(list(features))
1169 + embeddings.extend(self._split_embeddings(features, [split_sizes[index]]))
1170 + return embeddings
1171 +
1172 + def _select_item_kwargs(
1173 + self, kwargs: dict[str, Any], index: int, num_items: int
1174 + ) -> dict[str, Any]:
1175 + """Narrow the entries of `kwargs` that hold one row per item down to the item
1176 + at `index`. Length is all there is to match on, so an unrelated entry of the
1177 + same length is narrowed too."""
1178 + return {
1179 + key: value[index : index + 1]
1180 + if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
1181 + else value
1182 + for key, value in kwargs.items()
1183 + }
1184 +
1185 + def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> Any:
1186 # grid_thw fields are registered keep_on_cpu; restore the on-device
1187 # placement that HF get_image_features implementations expect.
1188 for key, value in kwargs.items():
@@ -772,22 +1194,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
1194 # padding images via boolean-mask indexing, LlavaOnevision
1195 # branches on per-sample batch counts).
1196 with gpu_sync_allowed():
775 - vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
1197 + features = self.model.get_image_features(pixel_values, **kwargs)
1198
1199 # Transformers `v5`, `self.get_image_features` returns a tuple
1200 # containing the features and optionally attentions/hidden_states
1201 # After v5 is settled, we can enable qwen3-vl with several outputs
1202 # from `self.get_image_features`
781 - if isinstance(vision_embeddings, tuple):
782 - vision_embeddings = vision_embeddings[0]
783 - elif isinstance(vision_embeddings, dict):
784 - vision_embeddings = vision_embeddings.pooler_output
785 -
786 - if isinstance(vision_embeddings, torch.Tensor):
787 - split_sizes = num_image_patches.flatten().tolist()
788 - return self._split_embeddings(vision_embeddings, split_sizes)
789 -
790 - return list(vision_embeddings)
1203 + if isinstance(features, tuple):
1204 + return features[0]
1205 + if isinstance(features, dict):
1206 + return features.pooler_output
1207 + return features
1208
1209 def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
1210 # Each helper detects its own inputs. We are called once per modality, so the