@tduncan / Nvidia-Optimization-Stack-VLLM / commits / b9b6306ebe

feat[vLLM × v5]: Add audio support for the Transformers backend (#39330)

Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Harshal Janjani committed Jul 25, 2026 at 15:20 UTC b9b6306ebe0f5fa5046553667192e56bb3934af0
8 files changed +680 -150
docs/models/supported_models.md
+3 -2
@@ -19,7 +19,7 @@ vLLM also supports model implementations that are available in Transformers. We
19
20 Currently, the Transformers modeling backend works for the following:
21
22 -- Modalities: embedding models, language models and vision-language models*
22 +- Modalities: embedding models, language models, vision-language models* and audio-language models
23 - Architectures: encoder-only, decoder-only, mixture-of-experts
24 - Attention types: full attention and/or sliding attention
25
@@ -606,7 +606,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor
606
607 | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
608 | ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
609 -| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
609 +| `Emu3ForConditionalGeneration` | Emu3 | T + I<sup>+</sup> | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
610 +| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A<sup>+</sup> | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ |
611
612 <sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.</br>
613 <sup>E</sup> Pre-computed embeddings can be inputted for this modality.</br>
tests/models/multimodal/generation/test_transformers_audio.py new
+133
@@ -0,0 +1,133 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from typing import Any
5 +
6 +import pytest
7 +from transformers import AutoModelForSeq2SeqLM
8 +
9 +from vllm.assets.audio import AudioAsset
10 +from vllm.envs import disable_envs_cache
11 +from vllm.lora.request import LoRARequest
12 +from vllm.multimodal.audio import AudioResampler
13 +
14 +from ....conftest import HfRunner, VllmRunner
15 +from ...utils import check_logprobs_close
16 +
17 +AUDIO_ASSET = AudioAsset("mary_had_lamb")
18 +
19 +AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
20 + "ibm-granite/granite-speech-3.3-2b": {
21 + "prompt": (
22 + "<|start_of_role|>system<|end_of_role|>"
23 + "You are a helpful AI assistant<|end_of_text|>\n"
24 + "<|start_of_role|>user<|end_of_role|>"
25 + "<|audio|>can you transcribe the speech into a written format?"
26 + "<|end_of_text|>\n"
27 + "<|start_of_role|>assistant<|end_of_role|>"
28 + ),
29 + "audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
30 + },
31 + "nvidia/audio-flamingo-3-hf": {
32 + "prompt": (
33 + "<|im_start|>system\n"
34 + "You are a helpful assistant.<|im_end|>\n"
35 + "<|im_start|>user\n"
36 + "<sound>Transcribe the input speech.<|im_end|>\n"
37 + "<|im_start|>assistant\n"
38 + ),
39 + "vllm_runner_kwargs": {
40 + "gpu_memory_utilization": 0.85,
41 + },
42 + },
43 + "microsoft/VibeVoice-ASR-HF": {
44 + "prompt": (
45 + "<|im_start|>system\n"
46 + "You are a helpful assistant that transcribes audio input "
47 + "into text output in JSON format.<|im_end|>\n"
48 + "<|im_start|>user\n"
49 + "<|object_ref_start|><|box_start|><|object_ref_end|>\n"
50 + "This is a 1.0 seconds audio, please transcribe it with "
51 + "these keys: Start time, End time, Speaker ID, Content"
52 + "<|im_end|>\n"
53 + "<|im_start|>assistant\n"
54 + ),
55 + "sampling_rate": 24000,
56 + "vllm_runner_kwargs": {
57 + "max_num_batched_tokens": 2048,
58 + "gpu_memory_utilization": 0.85,
59 + },
60 + },
61 + "zai-org/GLM-ASR-Nano-2512": {
62 + "prompt": (
63 + "<|user|>\n"
64 + "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
65 + "Please transcribe this audio into text"
66 + "<|assistant|>\n"
67 + ),
68 + },
69 +}
70 +
71 +
72 +@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
73 +def test_transformers_audio_generation(
74 + hf_runner: type[HfRunner],
75 + vllm_runner: type[VllmRunner],
76 + monkeypatch,
77 + model_id: str,
78 +):
79 + """Single-process workaround for V1 fork safety deadlock issue
80 + (vllm-project/vllm/issues/17676). Running multiple audio models together
81 + under pytest can cause (possibly flaky) hangs, so they are grouped under
82 + the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
83 + deadlock and allows worker processes to terminate cleanly, and release
84 + GPU memory between test runs until the issue is fixed."""
85 + # TODO: Remove monkeypatch once
86 + # https://github.com/vllm-project/vllm/issues/17676 is fixed.
87 + disable_envs_cache()
88 + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
89 +
90 + settings = AUDIO_MODEL_SETTINGS[model_id]
91 + audio_lora_path = settings.get("audio_lora_path")
92 +
93 + audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
94 + target_sr = settings.get("sampling_rate", orig_sr)
95 + if orig_sr != target_sr:
96 + audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr)
97 + audio = (audio, target_sr)
98 +
99 + with vllm_runner(
100 + model_id,
101 + model_impl="transformers",
102 + dtype="bfloat16",
103 + max_model_len=2048,
104 + enforce_eager=True,
105 + limit_mm_per_prompt={"audio": 1},
106 + enable_lora=audio_lora_path is not None,
107 + max_lora_rank=64,
108 + **settings.get("vllm_runner_kwargs", {}),
109 + ) as vllm_model:
110 + lora_request = (
111 + LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
112 + )
113 + vllm_outputs = vllm_model.generate_greedy_logprobs(
114 + [settings["prompt"]],
115 + 128,
116 + num_logprobs=10,
117 + audios=[audio],
118 + lora_request=lora_request,
119 + )
120 +
121 + with hf_runner(
122 + model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM
123 + ) as hf_model:
124 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
125 + [settings["prompt"]], 128, num_logprobs=10, audios=[audio]
126 + )
127 +
128 + check_logprobs_close(
129 + outputs_0_lst=hf_outputs,
130 + outputs_1_lst=vllm_outputs,
131 + name_0="hf",
132 + name_1="vllm",
133 + )
tests/models/multimodal/processing/test_common.py
+8
@@ -452,6 +452,14 @@ def test_processing_correctness(
452 "audio placeholders from processed audio lengths. Its vLLM "
453 "processor paths are covered by test_moss_audio.py."
454 )
455 + # TODO: Remove when transformers 5.15.0 is released, which contains
456 + # https://github.com/huggingface/transformers/pull/47483.
457 + if model_id == "microsoft/VibeVoice-ASR-HF":
458 + pytest.skip(
459 + "VibeVoice ASR requires audio as a positional argument and hence "
460 + "cannot pass the processing correctness test as is. Its generation "
461 + "is covered by test_transformers_audio.py."
462 + )
463 if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
464 pytest.skip(
465 "LLaVA-OneVision-2 video processing routes frames through custom "
tests/models/multimodal/processing/test_transformers_audio.py new
+138
@@ -0,0 +1,138 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import numpy as np
4 +import pytest
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.multimodal import MULTIMODAL_REGISTRY
8 +
9 +AUDIO_MODEL_SETTINGS = {
10 + "ibm-granite/granite-speech-3.3-2b": {
11 + "prompt": (
12 + "<|start_of_role|>system<|end_of_role|>"
13 + "You are a helpful AI assistant<|end_of_text|>\n"
14 + "<|start_of_role|>user<|end_of_role|>"
15 + "<|audio|>can you transcribe the speech into a written format?"
16 + "<|end_of_text|>\n"
17 + "<|start_of_role|>assistant<|end_of_role|>"
18 + ),
19 + },
20 + "nvidia/audio-flamingo-3-hf": {
21 + "prompt": (
22 + "<|im_start|>system\n"
23 + "You are a helpful assistant.<|im_end|>\n"
24 + "<|im_start|>user\n"
25 + "<sound>Transcribe the input speech.<|im_end|>\n"
26 + "<|im_start|>assistant\n"
27 + ),
28 + },
29 + "mistralai/Voxtral-Mini-3B-2507": {
30 + "prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
31 + },
32 + "microsoft/VibeVoice-ASR-HF": {
33 + "prompt": (
34 + "<|im_start|>system\n"
35 + "You are a helpful assistant that transcribes audio input "
36 + "into text output in JSON format.<|im_end|>\n"
37 + "<|im_start|>user\n"
38 + "<|object_ref_start|><|box_start|><|object_ref_end|>\n"
39 + "This is a 1.0 seconds audio, please transcribe it with "
40 + "these keys: Start time, End time, Speaker ID, Content"
41 + "<|im_end|>\n"
42 + "<|im_start|>assistant\n"
43 + ),
44 + },
45 + "zai-org/GLM-ASR-Nano-2512": {
46 + "prompt": (
47 + "<|user|>\n"
48 + "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
49 + "Please transcribe this audio into text"
50 + "<|assistant|>\n"
51 + ),
52 + },
53 +}
54 +
55 +
56 +@pytest.mark.parametrize(
57 + "model_id",
58 + [
59 + "ibm-granite/granite-speech-3.3-2b",
60 + "nvidia/audio-flamingo-3-hf",
61 + pytest.param(
62 + "mistralai/Voxtral-Mini-3B-2507",
63 + marks=pytest.mark.xfail(
64 + reason="MistralCommonBackend.encode does not produce the audio "
65 + "placeholder token (ID 24) from raw text. apply_chat_template "
66 + "yields token IDs with placeholders, but MultiModalProcessor."
67 + "apply() decodes the prompt back to text and re-tokenizes, at "
68 + "which point the placeholders are lost. Fix belongs in "
69 + "mistral_common or in the Voxtral-specific path.",
70 + strict=False,
71 + ),
72 + ),
73 + "microsoft/VibeVoice-ASR-HF",
74 + "zai-org/GLM-ASR-Nano-2512",
75 + ],
76 +)
77 +def test_audio_multimodal_processor(model_id):
78 + settings = AUDIO_MODEL_SETTINGS[model_id]
79 +
80 + model_config = ModelConfig(
81 + model=model_id,
82 + model_impl="transformers",
83 + )
84 +
85 + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
86 +
87 + audio = np.zeros(16000, dtype=np.float32)
88 + mm_data = {"audio": (audio, 16000)}
89 +
90 + result = mm_processor(
91 + prompt=settings["prompt"],
92 + mm_items=mm_processor.info.parse_mm_data(mm_data),
93 + hf_processor_mm_kwargs={},
94 + )
95 +
96 + assert "prompt_token_ids" in result
97 + assert len(result["prompt_token_ids"]) > 0
98 +
99 + mm_placeholders = result.get("mm_placeholders", {})
100 + assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}"
101 + assert len(mm_placeholders["audio"]) == 1
102 +
103 + placeholder = mm_placeholders["audio"][0]
104 + assert placeholder.length > 0
105 + assert placeholder.offset >= 0
106 +
107 + audio_items = result.get("mm_kwargs", {}).get("audio", [])
108 + assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
109 + item_keys = list(audio_items[0].keys())
110 + has_features = "input_features" in item_keys or "input_values" in item_keys
111 + assert has_features, (
112 + f"No audio features (input_features/input_values) in {item_keys} for {model_id}"
113 + )
114 +
115 +
116 +def test_audio_multiple_inputs():
117 + """Multiple audios per prompt are each detected as a separate placeholder
118 + and multi-modal item by the Transformers backend."""
119 + model_id = "ibm-granite/granite-speech-3.3-2b"
120 + model_config = ModelConfig(model=model_id, model_impl="transformers")
121 + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
122 +
123 + audio_token = mm_processor.info.get_hf_processor().audio_token
124 + # One token per audio; the processor expands each to its placeholder run.
125 + prompt = (
126 + "<|start_of_role|>user<|end_of_role|>"
127 + f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
128 + )
129 + audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
130 +
131 + result = mm_processor(
132 + prompt=prompt,
133 + mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
134 + hf_processor_mm_kwargs={},
135 + )
136 +
137 + assert len(result["mm_placeholders"]["audio"]) == 2
138 + assert len(result["mm_kwargs"]["audio"]) == 2
tests/models/multimodal/processing/test_transformers_image.py renamed
+23
@@ -54,3 +54,26 @@ def test_multimodal_processor(model_id):
54 str_processed_inputs["prompt_token_ids"]
55 == ids_processed_inputs["prompt_token_ids"]
56 )
57 +
58 +
59 +def test_image_multiple_inputs():
60 + """Multiple images per prompt are each detected as a separate placeholder
61 + and multi-modal item by the Transformers backend."""
62 + model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
63 + model_config = ModelConfig(model=model_id, model_impl="transformers")
64 + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
65 +
66 + image = ImageAsset("cherry_blossom").pil_image
67 + prompt = (
68 + "<|im_start|>user <image>\n and <image>\n"
69 + "What do these images show?<|im_end|><|im_start|>assistant\n"
70 + )
71 +
72 + result = mm_processor(
73 + prompt=prompt,
74 + mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
75 + hf_processor_mm_kwargs={},
76 + )
77 +
78 + assert len(result["mm_placeholders"]["image"]) == 2
79 + assert len(result["mm_kwargs"]["image"]) == 2
tests/models/registry.py
+4
@@ -1393,6 +1393,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1393 "fixie-ai/ultravox-v0_5-llama-3_2-1b",
1394 trust_remote_code=True,
1395 ),
1396 + "VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
1397 + "microsoft/VibeVoice-ASR-HF",
1398 + min_transformers_version="5.13.0",
1399 + ),
1400 "VoxtralForConditionalGeneration": _HfExamplesInfo(
1401 "mistralai/Voxtral-Mini-3B-2507",
1402 tokenizer_mode="mistral",
vllm/model_executor/models/registry.py
+4
@@ -664,6 +664,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
664 "transformers",
665 "TransformersMultiModalForCausalLM",
666 ),
667 + "VibeVoiceAsrForConditionalGeneration": (
668 + "transformers",
669 + "TransformersMultiModalForCausalLM",
670 + ),
671 }
672
673 _TRANSFORMERS_BACKEND_MODELS = {
vllm/model_executor/models/transformers/multimodal.py
+367 -148
@@ -17,7 +17,8 @@
17 """Transformers modeling backend mixin for multi-modal models."""
18
19 from collections.abc import Mapping
20 -from typing import TYPE_CHECKING
20 +from contextlib import nullcontext
21 +from typing import TYPE_CHECKING, Any
22
23 import torch
24 from transformers import AutoModel
@@ -27,6 +28,7 @@ from vllm.config.utils import getattr_iter
28 from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
29 from vllm.logger import init_logger
30 from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal
31 +from vllm.model_executor.models.module_mapping import MultiModelKeys
32 from vllm.multimodal import MultiModalKwargsItems
33 from vllm.multimodal.inputs import (
34 MultiModalFeatureSpec,
@@ -36,6 +38,7 @@ from vllm.multimodal.inputs import (
38 from vllm.multimodal.parse import (
39 ImageProcessorItems,
40 MultiModalDataItems,
41 + MultiModalDataParser,
42 )
43 from vllm.multimodal.processing import (
44 BaseDummyInputsBuilder,
@@ -59,11 +62,90 @@ _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
62
63
64 class MultiModalProcessingInfo(BaseProcessingInfo):
65 + def _get_audio_processor(self) -> Any:
66 + # TODO: drop feature_extractor branch once huggingface/transformers#44394 lands.
67 + return getattr_iter(
68 + self.get_hf_processor(), ("audio_processor", "feature_extractor")
69 + )
70 +
71 + def _is_audio_model(self) -> bool:
72 + return self._get_audio_processor() is not None
73 +
74 + def _is_image_model(self) -> bool:
75 + return hasattr(self.get_hf_processor(), "image_processor")
76 +
77 + def _is_video_model(self) -> bool:
78 + return hasattr(self.get_hf_processor(), "video_processor")
79 +
80 + def _get_audio_token_id(self) -> int:
81 + processor = self.get_hf_processor()
82 + if hasattr(processor, "audio_token_id"):
83 + return processor.audio_token_id
84 + config = self.get_hf_config()
85 + val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
86 + if val is not None:
87 + return val
88 + if hasattr(processor, "audio_token"):
89 + tokenizer = self.get_tokenizer()
90 + vocab = tokenizer.get_vocab()
91 + if processor.audio_token in vocab:
92 + return vocab[processor.audio_token]
93 + raise ValueError("Cannot find audio_token_id on processor or model config")
94 +
95 + def _get_audio_sampling_rate(self) -> float:
96 + sub = self._get_audio_processor()
97 + if sub is not None and hasattr(sub, "sampling_rate"):
98 + return sub.sampling_rate
99 + return 16000.0
100 +
101 + def get_data_parser(self) -> MultiModalDataParser:
102 + if self._is_audio_model():
103 + return MultiModalDataParser(
104 + target_sr=self._get_audio_sampling_rate(),
105 + expected_hidden_size=self._get_expected_hidden_size(),
106 + )
107 + return super().get_data_parser()
108 +
109 def get_supported_mm_limits(self):
63 - return {"image": None}
110 + limits = {}
111 + if self._is_audio_model():
112 + limits["audio"] = None
113 + if self._is_image_model():
114 + limits["image"] = None
115 + if not limits:
116 + raise ValueError(
117 + f"Unable to detect a supported modality on "
118 + f"{type(self.get_hf_processor()).__name__}. "
119 + "Checked `_is_audio_model` and `_is_image_model`."
120 + )
121 + return limits
122
123 def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
66 - return {"image": self.get_max_image_tokens()}
124 + result = {}
125 + if self._is_audio_model():
126 + result["audio"] = self.get_max_audio_tokens()
127 + if self._is_image_model():
128 + result["image"] = self.get_max_image_tokens()
129 + if not result:
130 + raise ValueError(
131 + f"Unable to detect a supported modality on "
132 + f"{type(self.get_hf_processor()).__name__}. "
133 + "Checked `_is_audio_model` and `_is_image_model`."
134 + )
135 + return result
136 +
137 + def get_max_audio_tokens(self) -> int:
138 + config = self.get_hf_config()
139 + audio_config_names = ("audio_config", "encoder_config")
140 + names = ("max_source_positions", "max_position_embeddings", "max_pos_emb")
141 + audio_config = getattr_iter(config, audio_config_names, default=config)
142 + val = getattr_iter(audio_config, names)
143 + if val is not None:
144 + return int(val)
145 + raise ValueError(
146 + f"Unable to get max input length from {type(audio_config).__name__}. "
147 + f"The following attribute names were checked: {names}."
148 + )
149
150 def get_max_image_tokens(self) -> int:
151 width, height = self.get_max_image_size()
@@ -82,14 +164,19 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
164
165 class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
166 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
85 - num_images = mm_counts.get("image", 0)
86 -
87 - processor = self.info.get_hf_processor()
88 - if "gemma3" in processor.__class__.__name__.lower():
89 - image_token = processor.boi_token
90 - else:
91 - image_token = getattr(processor, "image_token", "")
92 - return image_token * num_images
167 + text = ""
168 + if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
169 + processor = self.info.get_hf_processor()
170 + audio_token = getattr(processor, "audio_token", "")
171 + text += audio_token * num_audios
172 + if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
173 + processor = self.info.get_hf_processor()
174 + if "gemma3" in processor.__class__.__name__.lower():
175 + image_token = processor.boi_token
176 + else:
177 + image_token = getattr(processor, "image_token", "")
178 + text += image_token * num_images
179 + return text
180
181 def get_dummy_mm_data(
182 self,
@@ -97,20 +184,28 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
184 mm_counts: Mapping[str, int],
185 mm_options: Mapping[str, "BaseDummyOptions"],
186 ) -> MultiModalDataDict:
100 - num_images = mm_counts.get("image", 0)
101 -
102 - target_width, target_height = self.info.get_max_image_size()
103 -
104 - image_overrides = mm_options.get("image")
105 -
106 - return {
107 - "image": self._get_dummy_images(
187 + data: MultiModalDataDict = {}
188 + if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
189 + sampling_rate = self.info._get_audio_sampling_rate()
190 + sub = self.info._get_audio_processor()
191 + chunk_length = getattr(sub, "chunk_length", None) if sub else None
192 + if chunk_length is None:
193 + chunk_length = 30
194 + audio_len = int(chunk_length * sampling_rate)
195 + data["audio"] = self._get_dummy_audios(
196 + length=audio_len,
197 + num_audios=num_audios,
198 + overrides=mm_options.get("audio"),
199 + )
200 + if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
201 + target_width, target_height = self.info.get_max_image_size()
202 + data["image"] = self._get_dummy_images(
203 width=target_width,
204 height=target_height,
205 num_images=num_images,
111 - overrides=image_overrides,
112 - ),
113 - }
206 + overrides=mm_options.get("image"),
207 + )
208 + return data
209
210
211 class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
@@ -142,21 +237,39 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
237 ) -> Mapping[str, MultiModalFieldConfig]:
238 # HF Processors always return a mask but vLLM doesn't need it
239 hf_inputs.pop("attention_mask", None)
145 - num_image_patches = hf_inputs.get("num_image_patches")
146 - mm_fields = {
147 - key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
148 - for key in hf_inputs
149 - }
150 - mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
151 - "image", num_image_patches
152 - )
240
154 - # Keep these as batched, as they always have batch size as first dim
155 - mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
156 - mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
157 - mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
158 - "image", keep_on_cpu=True
159 - )
241 + mm_fields: dict[str, MultiModalFieldConfig] = {}
242 + if self.info._is_audio_model():
243 + num_audio_tokens = hf_inputs.get("num_audio_tokens")
244 + mm_fields.update(
245 + {
246 + key: MultiModalFieldConfig.flat_from_sizes(
247 + "audio", num_audio_tokens
248 + )
249 + for key in hf_inputs
250 + }
251 + )
252 + mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
253 + if self.info._is_image_model():
254 + num_image_patches = hf_inputs.get("num_image_patches")
255 + mm_fields.update(
256 + {
257 + key: MultiModalFieldConfig.flat_from_sizes(
258 + "image", num_image_patches
259 + )
260 + for key in hf_inputs
261 + }
262 + )
263 + mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
264 + "image", num_image_patches
265 + )
266 +
267 + # Keep these as batched, as they always have batch size as first dim
268 + mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
269 + mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
270 + mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
271 + "image", keep_on_cpu=True
272 + )
273 return mm_fields
274
275 def _get_hf_mm_data(
@@ -164,13 +277,93 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
277 mm_items: MultiModalDataItems,
278 ) -> tuple[Mapping[str, object], Mapping[str, object]]:
279 """
167 - In contrast to the base class, this method always adds
168 - `return_mm_token_type_ids` to the processor data
280 + In contrast to the base class, this method requests
281 + `return_mm_token_type_ids` and remaps the `audios` key to `audio` for
282 + audio models.
283 """
284 processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
285 + if self.info._is_audio_model() and "audios" in processor_data:
286 + processor_data["audio"] = processor_data.pop("audios")
287 processor_data["return_mm_token_type_ids"] = True
288 return processor_data, passthrough_data
289
290 + def _apply_audio(
291 + self,
292 + prompt_ids: list[int],
293 + processed_data: "BatchFeature",
294 + ) -> dict[str, list[PlaceholderRange]]:
295 + audio_token_id = self.info._get_audio_token_id()
296 + prompt_tensor = torch.tensor(prompt_ids)
297 + is_audio = prompt_tensor == audio_token_id
298 +
299 + if not is_audio.any():
300 + return {}
301 +
302 + padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
303 + transitions = padded.int().diff()
304 + starts = torch.where(transitions == 1)[0]
305 + ends = torch.where(transitions == -1)[0]
306 + lengths = ends - starts
307 +
308 + ranges = [
309 + PlaceholderRange(
310 + offset=s.item(),
311 + length=ln.item(),
312 + is_embed=torch.ones(ln.item(), dtype=torch.bool),
313 + )
314 + for s, ln in zip(starts, lengths)
315 + ]
316 + processed_data["num_audio_tokens"] = lengths
317 + return {"audio": ranges}
318 +
319 + def _apply_vision(
320 + self,
321 + prompt_ids: list[int],
322 + processed_data: "BatchFeature",
323 + mm_items: MultiModalDataItems,
324 + hf_processor_mm_kwargs: Mapping[str, object],
325 + mm_token_type_ids: torch.Tensor | None,
326 + ) -> dict[str, list[PlaceholderRange]]:
327 + if mm_token_type_ids is None:
328 + return {}
329 +
330 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
331 +
332 + # We can infer vLLM style placeholder from token type ids, if we split
333 + # it for each input `mm_data`.
334 + mm_positions = torch.where(mm_token_type_ids == 1)[1]
335 + images = mm_items.get_items("image", ImageProcessorItems)
336 + image_sizes = []
337 + for item_idx in range(len(images)):
338 + image_size = images.get_image_size(item_idx)
339 + image_sizes.append((image_size.height, image_size.width))
340 +
341 + mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
342 + image_sizes=image_sizes,
343 + **self.info.ctx.get_merged_mm_kwargs({}),
344 + )
345 +
346 + mm_placeholders: dict[str, list[PlaceholderRange]] = {}
347 + split_sizes = mm_tokens_per_modality["num_image_tokens"]
348 + if split_sizes:
349 + chunked_mm_positions = torch.split(mm_positions, split_sizes)
350 + mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
351 + chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
352 + ranges = [
353 + PlaceholderRange(
354 + offset=positions[0].item(),
355 + length=positions.shape[0],
356 + is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
357 + )
358 + for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
359 + ]
360 + mm_placeholders = {"image": ranges}
361 +
362 + processed_data["num_image_patches"] = torch.tensor(
363 + mm_tokens_per_modality["num_image_patches"]
364 + )
365 + return mm_placeholders
366 +
367 def apply(
368 self,
369 inputs: ProcessorInputs,
@@ -198,8 +391,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
391 # Bypass cached processor and always apply to the full set of mm inputs
392 # NOTE: we can't just set caching=False because base class method
393 # transforms outputs to `MultiModalKwargs` which is not going to
201 - # work for Transformers. We have a lot of logic tied to
202 - # `mm_tokens_per_modality` below
394 + # work for Transformers. The vision path has logic tied to
395 + # `mm_tokens_per_modality` in _apply_vision()
396 prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
397 prompt_text=prompt,
398 mm_items=mm_items,
@@ -207,52 +400,33 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
400 tokenization_kwargs=tokenization_kwargs,
401 )
402
403 + # Use overrides if provided; fallback to data-dependent hashing.
404 + with timing_ctx.record("get_mm_hashes"):
405 + mm_hashes = inputs.get_mm_hashes(self.info.model_id)
406 +
407 # For gemma3 we check `token_type_ids` as the key
408 mm_token_type_ids = processed_data.pop("token_type_ids", None)
409 mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
410
214 - # We can infer vLLM style placeholder from token type ids, if we split
215 - # it for each input `mm_data`.
216 - mm_positions = torch.where(mm_token_type_ids == 1)[1]
217 - images = mm_items.get_items("image", ImageProcessorItems)
218 - image_sizes = []
219 - for item_idx in range(len(images)):
220 - image_size = images.get_image_size(item_idx)
221 - image_sizes.append((image_size.height, image_size.width))
222 -
223 - mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
224 - image_sizes=image_sizes,
225 - **self.info.ctx.get_merged_mm_kwargs({}),
226 - )
227 -
228 - mm_placeholders = {}
229 - split_sizes = mm_tokens_per_modality["num_image_tokens"]
230 - if split_sizes:
231 - chunked_mm_positions = torch.split(mm_positions, split_sizes)
232 - mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
233 - chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
234 - ranges = [
235 - PlaceholderRange(
236 - offset=positions[0].item(),
237 - length=positions.shape[0],
238 - is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
411 + mm_placeholders: dict[str, list[PlaceholderRange]] = {}
412 + if self.info._is_audio_model():
413 + mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
414 + if self.info._is_image_model():
415 + mm_placeholders.update(
416 + self._apply_vision(
417 + prompt_ids,
418 + processed_data,
419 + mm_items,
420 + hf_processor_mm_kwargs,
421 + mm_token_type_ids,
422 )
240 - for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
241 - ]
242 - mm_placeholders = {"image": ranges}
423 + )
424
244 - processed_data["num_image_patches"] = torch.tensor(
245 - mm_tokens_per_modality["num_image_patches"]
246 - )
425 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
426 processed_data,
427 self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
428 )
429
252 - # Use overrides if provided; fallback to data-dependent hashing.
253 - with timing_ctx.record("get_mm_hashes"):
254 - mm_hashes = inputs.get_mm_hashes(self.info.model_id)
255 -
430 return mm_input(
431 prompt_token_ids=prompt_ids,
432 mm_kwargs=mm_kwargs,
@@ -364,7 +538,79 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
538
539 return LanguageModel(self)
540
367 - def embed_multimodal(self, **kwargs):
541 + def get_mm_mapping(self) -> MultiModelKeys:
542 + """
543 + Get the module prefix in multimodal models
544 + """
545 + for name in ("language_model", "text_model"):
546 + if getattr(self.model, name, None) is not None:
547 + return MultiModelKeys.from_string_field(language_model=f"model.{name}")
548 + raise ValueError(
549 + "Could not locate the language model submodule for LoRA support"
550 + )
551 +
552 + def _split_embeddings(
553 + self, embeddings: torch.Tensor, split_sizes: list[int]
554 + ) -> list[torch.Tensor]:
555 + total_expected = sum(split_sizes)
556 +
557 + # Flatten to 2D: [total_tokens, hidden_dim]
558 + if embeddings.ndim == 3:
559 + embeddings = embeddings.view(-1, embeddings.shape[-1])
560 +
561 + total_tokens = embeddings.shape[0]
562 + if total_tokens == total_expected:
563 + # Direct match: split_sizes are actual token counts
564 + token_split_sizes = split_sizes
565 + elif total_expected > 0 and total_tokens % total_expected == 0:
566 + # Uniform expansion: each item expands to N tokens
567 + tokens_per_item = total_tokens // total_expected
568 + token_split_sizes = [s * tokens_per_item for s in split_sizes]
569 + elif total_expected > 0:
570 + # Mismatch (profiling with dummy data) - pad/truncate
571 + if total_tokens == 0:
572 + raise ValueError(
573 + "Encoder returned empty embeddings. "
574 + f"Expected {total_expected} tokens from "
575 + f"split_sizes={split_sizes}"
576 + )
577 + if total_tokens < total_expected:
578 + repeat_factor = (total_expected + total_tokens - 1) // total_tokens
579 + embeddings = embeddings.repeat(repeat_factor, 1)
580 + embeddings = embeddings[:total_expected]
581 + token_split_sizes = split_sizes
582 + else:
583 + return []
584 +
585 + return list(torch.split(embeddings, token_split_sizes, dim=0))
586 +
587 + def _embed_audio(self, **kwargs) -> list[torch.Tensor] | None:
588 + self.check_version("5.13.0", "audio models support")
589 + input_features: torch.Tensor | None = kwargs.pop("input_features", None)
590 + if input_features is None:
591 + input_features = kwargs.pop("input_values", None)
592 + if input_features is None:
593 + return None
594 +
595 + num_audio_tokens = kwargs.pop("num_audio_tokens")
596 + kwargs.pop("token_type_ids", None)
597 + kwargs.pop("mm_token_type_ids", None)
598 +
599 + context = nullcontext()
600 + if current_platform.is_rocm():
601 + context = torch.nn.attention.sdpa_kernel(
602 + backends=[torch.nn.attention.SDPBackend.MATH]
603 + )
604 + with context:
605 + audio_output = self.model.get_audio_features(
606 + input_features, return_dict=True, **kwargs
607 + )
608 + audio_embeddings = audio_output.pooler_output
609 +
610 + split_sizes = num_audio_tokens.flatten().tolist()
611 + return self._split_embeddings(audio_embeddings, split_sizes)
612 +
613 + def _embed_vision(self, **kwargs) -> list[torch.Tensor] | torch.Tensor | None:
614 pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
615 image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
616 # Model might use `image_patches` instead of `pixel_values`
@@ -379,84 +625,57 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
625
626 num_image_patches = kwargs.pop("num_image_patches")
627
382 - if pixel_values is not None:
628 + context = nullcontext()
629 + if current_platform.is_rocm():
630 # ROCm: Force math SDP backend for vision encoder to avoid accuracy issues
631 # with flash_sdp and mem_efficient_sdp
385 - if current_platform.is_rocm():
386 - # TODO: [ROCm] Fix accuracy issues with flash backend
387 - logger.debug(
388 - "ROCm platform detected. Forcing math SDP backend "
389 - "for vision encoder. Currently ROCm platform has "
390 - "accuracy issues with `flash_sdp` and"
391 - "`mem_efficient_sdp` backends. See issue: "
392 - "https://github.com/vllm-project/vllm/issues/30167"
393 - )
394 - with torch.nn.attention.sdpa_kernel(
395 - backends=[torch.nn.attention.SDPBackend.MATH]
396 - ):
397 - vision_embeddings = self.model.get_image_features(
398 - pixel_values, **kwargs
399 - )
400 - else:
401 - vision_embeddings = self.model.get_image_features(
402 - pixel_values, **kwargs
403 - )
632 + # TODO: [ROCm] Fix accuracy issues with flash backend
633 + logger.debug(
634 + "ROCm platform detected. Forcing math SDP backend "
635 + "for vision encoder. Currently ROCm platform has "
636 + "accuracy issues with `flash_sdp` and"
637 + "`mem_efficient_sdp` backends. See issue: "
638 + "https://github.com/vllm-project/vllm/issues/30167"
639 + )
640 + context = torch.nn.attention.sdpa_kernel(
641 + backends=[torch.nn.attention.SDPBackend.MATH]
642 + )
643 + with context:
644 + vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
645
405 - # Transformers `v5`, `self.get_image_features` returns a tuple
406 - # containing the features and optionally attentions/hidden_states
407 - # After v5 is settled, we can enable qwen3-vl with several outputs
408 - # from `self.get_image_features`
409 - if isinstance(vision_embeddings, tuple):
410 - vision_embeddings = vision_embeddings[0]
411 - elif isinstance(vision_embeddings, dict):
412 - vision_embeddings = vision_embeddings.pooler_output
413 -
414 - if isinstance(vision_embeddings, torch.Tensor):
415 - split_sizes = num_image_patches.flatten().tolist()
416 - total_patches = sum(split_sizes)
417 -
418 - # Flatten to 2D: [total_tokens, hidden_dim]
419 - if vision_embeddings.ndim == 3:
420 - vision_embeddings = vision_embeddings.view(
421 - -1, vision_embeddings.shape[-1]
422 - )
646 + # Transformers `v5`, `self.get_image_features` returns a tuple
647 + # containing the features and optionally attentions/hidden_states
648 + # After v5 is settled, we can enable qwen3-vl with several outputs
649 + # from `self.get_image_features`
650 + if isinstance(vision_embeddings, tuple):
651 + vision_embeddings = vision_embeddings[0]
652 + elif isinstance(vision_embeddings, dict):
653 + vision_embeddings = vision_embeddings.pooler_output
654
424 - total_tokens = vision_embeddings.shape[0]
425 - if total_tokens == total_patches:
426 - # Direct match: num_image_patches are actual token counts
427 - # (e.g., Qwen2.5-VL style)
428 - token_split_sizes = split_sizes
429 - elif total_patches > 0 and total_tokens % total_patches == 0:
430 - # Uniform expansion: each patch expands to N tokens
431 - # (e.g., Idefics3 style)
432 - tokens_per_patch = total_tokens // total_patches
433 - token_split_sizes = [s * tokens_per_patch for s in split_sizes]
434 - elif total_patches > 0:
435 - # Mismatch (profiling with dummy data) - pad/truncate
436 - if total_tokens == 0:
437 - raise ValueError(
438 - "Vision encoder returned empty embeddings. "
439 - f"Expected {total_patches} patches from "
440 - f"num_image_patches={split_sizes}"
441 - )
442 - if total_tokens < total_patches:
443 - repeat_factor = (
444 - total_patches + total_tokens - 1
445 - ) // total_tokens
446 - vision_embeddings = vision_embeddings.repeat(repeat_factor, 1)
447 - vision_embeddings = vision_embeddings[:total_patches]
448 - token_split_sizes = split_sizes
449 - else:
450 - return []
655 + if isinstance(vision_embeddings, torch.Tensor):
656 + split_sizes = num_image_patches.flatten().tolist()
657 + return self._split_embeddings(vision_embeddings, split_sizes)
658
452 - return list(torch.split(vision_embeddings, token_split_sizes, dim=0))
659 + return vision_embeddings
660
454 - return vision_embeddings
455 - else:
456 - logger.debug(
457 - "No pixel values or image embeddings provided for multimodal embedding."
458 - )
459 - return None
661 + def embed_multimodal(self, **kwargs):
662 + embeddings: tuple[torch.Tensor, ...] = ()
663 + if "input_features" in kwargs or "input_values" in kwargs:
664 + audio_embeddings = self._embed_audio(**kwargs)
665 + if audio_embeddings is not None:
666 + embeddings += tuple(audio_embeddings)
667 + if (
668 + "pixel_values" in kwargs
669 + or "image_embeds" in kwargs
670 + or "image_patches" in kwargs
671 + ):
672 + vision_embeddings = self._embed_vision(**kwargs)
673 + if vision_embeddings is not None:
674 + if isinstance(vision_embeddings, torch.Tensor):
675 + embeddings += (vision_embeddings,)
676 + else:
677 + embeddings += tuple(vision_embeddings)
678 + return embeddings
679
680 def get_mrope_input_positions(
681 self,