[1/N] Harden Transformers modelling backend multi-modal path (#51408)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Aug 7, 2026 at 18:30 UTC
45273b8dcbfb2d2c300c2f4a55c4dc283adca06a
4 files changed
+204
-52
tests/models/multimodal/processing/test_common.py
+32
@@ -134,6 +134,17 @@ def get_model_ids_to_test():
134
return _get_model_ids_to_test(vllm_only_archs)
135
136
137
+def get_transformers_backend_model_ids_to_test():
138
+ return sorted(
139
+ {
140
+ model_id
141
+ for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
142
+ if "MultiModal" in arch
143
+ for model_id in (info.default, *info.extras.values())
144
+ }
145
+ )
146
+
147
+
148
def get_text_token_prompts(
149
processor: BaseMultiModalProcessor,
150
mm_data: MultiModalDataDict,
@@ -211,6 +222,7 @@ def _test_processing_correctness(
222
hit_rate: float,
223
num_batches: int,
224
simplify_rate: float,
225
+ model_impl: str = "auto",
226
):
227
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
228
# Use model architecture to get the default model id
@@ -238,6 +250,7 @@ def _test_processing_correctness(
250
enable_mm_embeds=model_info.require_embed_inputs,
251
enforce_eager=model_info.enforce_eager,
252
dtype=model_info.dtype,
253
+ model_impl=model_impl,
254
)
255
# Ensure that the cache can fit all of the data
256
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
@@ -480,6 +493,25 @@ def test_processing_correctness(
493
)
494
495
496
+@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
497
+@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
498
+@pytest.mark.parametrize("num_batches", [32])
499
+@pytest.mark.parametrize("simplify_rate", [1.0])
500
+def test_processing_correctness_transformers(
501
+ model_id: str,
502
+ hit_rate: float,
503
+ num_batches: int,
504
+ simplify_rate: float,
505
+):
506
+ _test_processing_correctness(
507
+ model_id,
508
+ hit_rate=hit_rate,
509
+ num_batches=num_batches,
510
+ simplify_rate=simplify_rate,
511
+ model_impl="transformers",
512
+ )
513
+
514
+
515
def _assert_inputs_equal(
516
a: MultiModalInput,
517
b: MultiModalInput,
tests/models/multimodal/processing/test_transformers_audio.py
+42
-5
@@ -113,9 +113,7 @@ def test_audio_multimodal_processor(model_id):
113
)
114
115
116
-def test_audio_multiple_inputs():
117
- """Multiple audios per prompt are each detected as a separate placeholder
118
- and multi-modal item by the Transformers backend."""
116
+def _process_granite_speech(separator: str):
117
model_id = "ibm-granite/granite-speech-3.3-2b"
118
model_config = ModelConfig(model=model_id, model_impl="transformers")
119
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
@@ -124,15 +122,54 @@ def test_audio_multiple_inputs():
122
# One token per audio; the processor expands each to its placeholder run.
123
prompt = (
124
"<|start_of_role|>user<|end_of_role|>"
127
- f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
125
+ f"{audio_token}{separator}{audio_token} transcribe<|end_of_text|>\n"
126
)
127
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
128
131
- result = mm_processor(
129
+ return mm_processor(
130
prompt=prompt,
131
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
132
hf_processor_mm_kwargs={},
133
)
134
135
+
136
+def test_audio_multiple_inputs():
137
+ """Multiple audios per prompt are each detected as a separate placeholder
138
+ and multi-modal item by the Transformers modelling backend."""
139
+ result = _process_granite_speech(separator=" and ")
140
+
141
assert len(result["mm_placeholders"]["audio"]) == 2
142
assert len(result["mm_kwargs"]["audio"]) == 2
143
+
144
+
145
+def test_audio_fields_not_claimed_by_image():
146
+ """Audio fields survive when the image branch is also active."""
147
+ model_id = "ibm-granite/granite-speech-3.3-2b"
148
+ model_config = ModelConfig(model=model_id, model_impl="transformers")
149
+ mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
150
+
151
+ audio_keys = ["input_features", "input_features_mask"]
152
+ owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"])
153
+
154
+ assert owned["audio"] == audio_keys
155
+ assert owned["image"] == []
156
+
157
+
158
+def test_unclaimed_fields_warn_rather_than_raise():
159
+ """Keys no sub-processor declares are dropped with a warning, not an error."""
160
+ model_id = "ibm-granite/granite-speech-3.3-2b"
161
+ model_config = ModelConfig(model=model_id, model_impl="transformers")
162
+ mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
163
+
164
+ owned = mm_processor._partition_keys_by_modality(
165
+ ["input_features", "surprise_field"], ["audio", "image"]
166
+ )
167
+
168
+ assert owned["audio"] == ["input_features"]
169
+ assert owned["image"] == []
170
+
171
+
172
+def test_audio_adjacent_inputs():
173
+ """Adjacent audios are rejected rather than silently merged into one placeholder."""
174
+ with pytest.raises(ValueError, match="told apart"):
175
+ _process_granite_speech(separator="")
tests/models/multimodal/processing/test_transformers_image.py
+33
-5
@@ -56,24 +56,52 @@ def test_multimodal_processor(model_id):
56
)
57
58
59
-def test_image_multiple_inputs():
60
- """Multiple images per prompt are each detected as a separate placeholder
61
- and multi-modal item by the Transformers backend."""
59
+def _process_two_images(separator: str):
60
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
61
model_config = ModelConfig(model=model_id, model_impl="transformers")
62
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
63
64
image = ImageAsset("cherry_blossom").pil_image
65
prompt = (
68
- "<|im_start|>user <image>\n and <image>\n"
66
+ f"<|im_start|>user <image>{separator}<image>\n"
67
"What do these images show?<|im_end|><|im_start|>assistant\n"
68
)
69
72
- result = mm_processor(
70
+ return mm_processor(
71
prompt=prompt,
72
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
73
hf_processor_mm_kwargs={},
74
)
75
76
+
77
+def test_image_multiple_inputs():
78
+ """Multiple images per prompt are each detected as a separate placeholder
79
+ and multi-modal item by the Transformers modelling backend."""
80
+ result = _process_two_images(separator="\n and ")
81
+
82
+ assert len(result["mm_placeholders"]["image"]) == 2
83
+ assert len(result["mm_kwargs"]["image"]) == 2
84
+
85
+
86
+def test_image_adjacent_inputs():
87
+ """Adjacent images stay separate placeholders rather than merging into one."""
88
+ result = _process_two_images(separator="")
89
+
90
assert len(result["mm_placeholders"]["image"]) == 2
91
assert len(result["mm_kwargs"]["image"]) == 2
92
+
93
+
94
+def test_text_only_prompt():
95
+ """An image model still accepts a prompt with no images."""
96
+ model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
97
+ model_config = ModelConfig(model=model_id, model_impl="transformers")
98
+ mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
99
+
100
+ result = mm_processor(
101
+ prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
102
+ mm_items=mm_processor.info.parse_mm_data({}),
103
+ hf_processor_mm_kwargs={},
104
+ )
105
+
106
+ assert len(result["prompt_token_ids"]) > 0
107
+ assert not result["mm_placeholders"]
vllm/model_executor/models/transformers/multimodal.py
+97
-42
@@ -168,7 +168,8 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
168
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
169
processor = self.info.get_hf_processor()
170
audio_token = getattr(processor, "audio_token", "")
171
- text += audio_token * num_audios
171
+ # Separated so that `_apply_audio` can tell the placeholders apart
172
+ text += " ".join([audio_token] * num_audios)
173
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
174
processor = self.info.get_hf_processor()
175
if "gemma3" in processor.__class__.__name__.lower():
@@ -230,6 +231,54 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
231
"""
232
return None
233
234
+ def _get_modality_field_names(self, modality: str) -> set[str]:
235
+ """Field names the sub-processor for `modality` produces."""
236
+ # TODO: use else branch only once huggingface/transformers#44394 lands.
237
+ if modality == "audio":
238
+ sub_processor = self.info._get_audio_processor()
239
+ else:
240
+ processor = self.info.get_hf_processor()
241
+ sub_processor = getattr(processor, f"{modality}_processor", None)
242
+
243
+ # Pre-computed embeddings bypass the sub-processor entirely
244
+ names = {f"{modality}_embeds"}
245
+ for name in getattr(sub_processor, "model_input_names", None) or ():
246
+ # Companion masks are emitted but not always declared
247
+ names.update((name, f"{name}_mask"))
248
+ return names
249
+
250
+ def _partition_keys_by_modality(
251
+ self,
252
+ keys: list[str],
253
+ modalities: list[str],
254
+ ) -> dict[str, list[str]]:
255
+ """Attribute each HF processor output key to the modality that produced it."""
256
+ if len(modalities) == 1:
257
+ return {modalities[0]: keys}
258
+
259
+ claimed = {m: self._get_modality_field_names(m) for m in modalities}
260
+
261
+ owned: dict[str, list[str]] = {modality: [] for modality in modalities}
262
+ unclaimed = []
263
+ for key in keys:
264
+ for modality in modalities:
265
+ if key in claimed[modality]:
266
+ owned[modality].append(key)
267
+ break
268
+ else:
269
+ unclaimed.append(key)
270
+
271
+ if unclaimed:
272
+ logger.warning_once(
273
+ "Unable to attribute %s to any of the modalities %s, so they "
274
+ "will not be passed to the model. Add them to the relevant "
275
+ "sub-processor's `model_input_names` to fix this.",
276
+ tuple(unclaimed),
277
+ tuple(modalities),
278
+ )
279
+
280
+ return owned
281
+
282
def _get_mm_fields_config(
283
self,
284
hf_inputs: "BatchFeature",
@@ -238,34 +287,29 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
287
# HF Processors always return a mask but vLLM doesn't need it
288
hf_inputs.pop("attention_mask", None)
289
241
- mm_fields: dict[str, MultiModalFieldConfig] = {}
242
- if self.info._is_audio_model():
243
- num_audio_tokens = hf_inputs.get("num_audio_tokens")
244
- mm_fields.update(
245
- {
246
- key: MultiModalFieldConfig.flat_from_sizes(
247
- "audio", num_audio_tokens
248
- )
249
- for key in hf_inputs
250
- }
251
- )
290
+ # Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
291
+ sizes = {
292
+ "audio": hf_inputs.get("num_audio_tokens"),
293
+ "image": hf_inputs.get("num_image_patches"),
294
+ }
295
+ modalities = [m for m, size in sizes.items() if size is not None]
296
+
297
+ size_keys = {"num_audio_tokens", "num_image_patches"}
298
+ keys = [key for key in hf_inputs if key not in size_keys]
299
+ owned = self._partition_keys_by_modality(keys, modalities)
300
+
301
+ mm_fields: dict[str, MultiModalFieldConfig] = {
302
+ key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
303
+ for modality in modalities
304
+ for key in owned[modality]
305
+ }
306
+
307
+ # Keep these as batched, as they always have batch size as first dim
308
+ if "audio" in modalities:
309
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
253
- if self.info._is_image_model():
254
- num_image_patches = hf_inputs.get("num_image_patches")
255
- mm_fields.update(
256
- {
257
- key: MultiModalFieldConfig.flat_from_sizes(
258
- "image", num_image_patches
259
- )
260
- for key in hf_inputs
261
- }
262
- )
263
- mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
264
- "image", num_image_patches
265
- )
266
-
267
- # Keep these as batched, as they always have batch size as first dim
310
+ if "image" in modalities:
311
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
312
+ # TODO: route to "video" once the video modality is supported
313
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
314
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
315
"image", keep_on_cpu=True
@@ -291,6 +335,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
335
self,
336
prompt_ids: list[int],
337
processed_data: "BatchFeature",
338
+ num_audios: int,
339
) -> dict[str, list[PlaceholderRange]]:
340
audio_token_id = self.info._get_audio_token_id()
341
prompt_tensor = torch.tensor(prompt_ids)
@@ -301,17 +346,21 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
346
347
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
348
transitions = padded.int().diff()
304
- starts = torch.where(transitions == 1)[0]
305
- ends = torch.where(transitions == -1)[0]
306
- lengths = ends - starts
349
+ offsets = torch.where(transitions == 1)[0]
350
+ lengths = torch.where(transitions == -1)[0] - offsets
351
308
- ranges = [
309
- PlaceholderRange(
310
- offset=s.item(),
311
- length=ln.item(),
312
- is_embed=torch.ones(ln.item(), dtype=torch.bool),
352
+ if len(offsets) != num_audios:
353
+ raise ValueError(
354
+ f"Found {len(offsets)} run(s) of the audio token in the prompt but "
355
+ f"{num_audios} audio item(s) were passed. The Transformers backend "
356
+ "locates audio placeholders by finding contiguous runs of the audio "
357
+ "token, so placeholders with no text between them cannot yet be told "
358
+ "apart. Separate them in the prompt to work around this."
359
)
314
- for s, ln in zip(starts, lengths)
360
+
361
+ ranges = [
362
+ PlaceholderRange(offset=offset.item(), length=length.item())
363
+ for offset, length in zip(offsets, lengths)
364
]
365
processed_data["num_audio_tokens"] = lengths
366
return {"audio": ranges}
@@ -324,6 +373,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
373
hf_processor_mm_kwargs: Mapping[str, object],
374
mm_token_type_ids: torch.Tensor | None,
375
) -> dict[str, list[PlaceholderRange]]:
376
+ # Placeholders can't be located without them, so give up rather than guess
377
if mm_token_type_ids is None:
378
return {}
379
@@ -383,10 +433,13 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
433
with timing_ctx.record("apply_hf_processor"):
434
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
435
if not isinstance(prompt, str):
386
- # the prompt is the tokenized ids which is not supported
387
- # by the hf_processor, which is why we would need to decode the ids
388
- # into string
436
+ # HF processors only accept text, and the decoded string already
437
+ # contains any special tokens, so don't let them be added again
438
prompt = hf_processor.decode(prompt)
439
+ tokenization_kwargs = {
440
+ **tokenization_kwargs,
441
+ "add_special_tokens": False,
442
+ }
443
444
# Bypass cached processor and always apply to the full set of mm inputs
445
# NOTE: we can't just set caching=False because base class method
@@ -412,9 +465,11 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
465
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
466
467
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
415
- if self.info._is_audio_model():
416
- mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
417
- if self.info._is_image_model():
468
+ if num_audios := mm_items.get_count("audio", strict=False):
469
+ mm_placeholders.update(
470
+ self._apply_audio(prompt_ids, processed_data, num_audios)
471
+ )
472
+ if mm_items.get_count("image", strict=False):
473
mm_placeholders.update(
474
self._apply_vision(
475
prompt_ids,