@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eed2f463b2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 23:45 UTC

[VLM] Support HF format Phi-4-MM model (#17121)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jul 27, 2025 at 11:07 UTC eed2f463b26c0ca65c12be62505a04de99ead456
10 files changed +1847 -5
docs/models/supported_models.md
+1
@@ -614,6 +614,7 @@ Specified using `--task generate`.
614 | `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | | ✅︎ | ⚠️ |
615 | `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ | ✅︎ |
616 | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
617 +| `Phi4MultimodalForCausalLM` | Phi-4-multimodal (HF Transformers) | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct` (with revision `refs/pr/70`), etc. | ✅︎ | ✅︎ | ✅︎ |
618 | `PixtralForConditionalGeneration` | Mistral 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Pixtral-12B-2409`, etc. | | ✅︎ | ✅︎ |
619 | `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
620 | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ | ✅︎ |
examples/offline_inference/audio_language.py
+32
@@ -190,6 +190,37 @@ def run_phi4mm(question: str, audio_count: int) -> ModelRequestData:
190 )
191
192
193 +def run_phi4_multimodal(question: str, audio_count: int) -> ModelRequestData:
194 + """
195 + Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
196 + show how to process audio inputs.
197 + """
198 + model_path = snapshot_download(
199 + "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70"
200 + )
201 + # Since the vision-lora and speech-lora co-exist with the base model,
202 + # we have to manually specify the path of the lora weights.
203 + speech_lora_path = os.path.join(model_path, "speech-lora")
204 + placeholders = "<|audio|>" * audio_count
205 +
206 + prompts = f"<|user|>{placeholders}{question}<|end|><|assistant|>"
207 +
208 + engine_args = EngineArgs(
209 + model=model_path,
210 + max_model_len=12800,
211 + max_num_seqs=2,
212 + enable_lora=True,
213 + max_lora_rank=320,
214 + limit_mm_per_prompt={"audio": audio_count},
215 + )
216 +
217 + return ModelRequestData(
218 + engine_args=engine_args,
219 + prompt=prompts,
220 + lora_requests=[LoRARequest("speech", 1, speech_lora_path)],
221 + )
222 +
223 +
224 # Qwen2-Audio
225 def run_qwen2_audio(question: str, audio_count: int) -> ModelRequestData:
226 model_name = "Qwen/Qwen2-Audio-7B-Instruct"
@@ -303,6 +334,7 @@ model_example_map = {
334 "granite_speech": run_granite_speech,
335 "minicpmo": run_minicpmo,
336 "phi4_mm": run_phi4mm,
337 + "phi4_multimodal": run_phi4_multimodal,
338 "qwen2_audio": run_qwen2_audio,
339 "qwen2_5_omni": run_qwen2_5_omni,
340 "ultravox": run_ultravox,
examples/offline_inference/vision_language.py
+36
@@ -1097,6 +1097,41 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
1097 )
1098
1099
1100 +# HF format Phi-4-multimodal-instruct
1101 +def run_phi4_multimodal(questions: list[str], modality: str) -> ModelRequestData:
1102 + """
1103 + Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
1104 + show how to process image inputs.
1105 + """
1106 + assert modality == "image"
1107 + model_path = snapshot_download(
1108 + "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70"
1109 + )
1110 + # Since the vision-lora and speech-lora co-exist with the base model,
1111 + # we have to manually specify the path of the lora weights.
1112 + vision_lora_path = os.path.join(model_path, "vision-lora")
1113 + prompts = [
1114 + f"<|user|><|image|>{question}<|end|><|assistant|>" for question in questions
1115 + ]
1116 + engine_args = EngineArgs(
1117 + model=model_path,
1118 + max_model_len=5120,
1119 + max_num_seqs=2,
1120 + max_num_batched_tokens=12800,
1121 + enable_lora=True,
1122 + max_lora_rank=320,
1123 + # Note - mm_processor_kwargs can also be passed to generate/chat calls
1124 + mm_processor_kwargs={"dynamic_hd": 16},
1125 + limit_mm_per_prompt={"image": 1},
1126 + )
1127 +
1128 + return ModelRequestData(
1129 + engine_args=engine_args,
1130 + prompts=prompts,
1131 + lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
1132 + )
1133 +
1134 +
1135 # Pixtral HF-format
1136 def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
1137 assert modality == "image"
@@ -1356,6 +1391,7 @@ model_example_map = {
1391 "paligemma2": run_paligemma2,
1392 "phi3_v": run_phi3v,
1393 "phi4_mm": run_phi4mm,
1394 + "phi4_multimodal": run_phi4_multimodal,
1395 "pixtral_hf": run_pixtral_hf,
1396 "qwen_vl": run_qwen_vl,
1397 "qwen2_vl": run_qwen2_vl,
examples/offline_inference/vision_language_multi_image.py
+35
@@ -760,6 +760,40 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
760 )
761
762
763 +def load_phi4_multimodal(question: str, image_urls: list[str]) -> ModelRequestData:
764 + """
765 + Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
766 + show how to process multi images inputs.
767 + """
768 +
769 + model_path = snapshot_download(
770 + "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70"
771 + )
772 + # Since the vision-lora and speech-lora co-exist with the base model,
773 + # we have to manually specify the path of the lora weights.
774 + vision_lora_path = os.path.join(model_path, "vision-lora")
775 + engine_args = EngineArgs(
776 + model=model_path,
777 + max_model_len=4096,
778 + max_num_seqs=2,
779 + limit_mm_per_prompt={"image": len(image_urls)},
780 + enable_lora=True,
781 + max_lora_rank=320,
782 + # Note - mm_processor_kwargs can also be passed to generate/chat calls
783 + mm_processor_kwargs={"dynamic_hd": 4},
784 + )
785 +
786 + placeholders = "<|image|>" * len(image_urls)
787 + prompt = f"<|user|>{placeholders}{question}<|end|><|assistant|>"
788 +
789 + return ModelRequestData(
790 + engine_args=engine_args,
791 + prompt=prompt,
792 + image_data=[fetch_image(url) for url in image_urls],
793 + lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
794 + )
795 +
796 +
797 def load_qwen_vl_chat(question: str, image_urls: list[str]) -> ModelRequestData:
798 model_name = "Qwen/Qwen-VL-Chat"
799 engine_args = EngineArgs(
@@ -988,6 +1022,7 @@ model_example_map = {
1022 "ovis": load_ovis,
1023 "phi3_v": load_phi3v,
1024 "phi4_mm": load_phi4mm,
1025 + "phi4_multimodal": load_phi4_multimodal,
1026 "pixtral_hf": load_pixtral_hf,
1027 "qwen_vl_chat": load_qwen_vl_chat,
1028 "qwen2_vl": load_qwen2_vl,
tests/models/multimodal/generation/test_phi4_multimodal.py new
+252
@@ -0,0 +1,252 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import os
5 +from collections.abc import Sequence
6 +from typing import Optional
7 +
8 +import librosa
9 +import pytest
10 +from huggingface_hub import snapshot_download
11 +
12 +from vllm.assets.image import ImageAsset
13 +from vllm.lora.request import LoRARequest
14 +from vllm.multimodal.image import rescale_image_size
15 +from vllm.platforms import current_platform
16 +
17 +from ....conftest import (IMAGE_ASSETS, HfRunner, PromptAudioInput,
18 + PromptImageInput, VllmRunner)
19 +from ....utils import large_gpu_test
20 +from ...utils import check_logprobs_close
21 +
22 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
23 + "stop_sign":
24 + "<|user|>\n<|image|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
25 + "cherry_blossom":
26 + "<|user|>\n<|image|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
27 +})
28 +HF_MULTIIMAGE_IMAGE_PROMPT = "<|user|>\n<|image|>\n<|image|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
29 +
30 +model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct",
31 + revision="refs/pr/70")
32 +# Since the vision-lora and speech-lora co-exist with the base model,
33 +# we have to manually specify the path of the lora weights.
34 +vision_lora_path = os.path.join(model_path, "vision-lora")
35 +speech_question = os.path.join(model_path, "examples",
36 + "what_is_shown_in_this_image.wav")
37 +models = [model_path]
38 +
39 +target_dtype = "half"
40 +
41 +# ROCm Triton FA can run into shared memory issues with these models,
42 +# use other backends in the meantime
43 +# FIXME (mattwong, gshtrasb, hongxiayan)
44 +if current_platform.is_rocm():
45 + os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
46 +
47 +
48 +def run_test(
49 + hf_runner: type[HfRunner],
50 + vllm_runner: type[VllmRunner],
51 + inputs: Sequence[tuple[list[str], PromptImageInput,
52 + Optional[PromptAudioInput]]],
53 + model: str,
54 + *,
55 + max_model_len: int,
56 + dtype: str,
57 + max_tokens: int,
58 + num_logprobs: int,
59 + mm_limit: int,
60 + tensor_parallel_size: int,
61 + distributed_executor_backend: Optional[str] = None,
62 +):
63 + """Inference result should be the same between hf and vllm.
64 +
65 + All the image fixtures for the test are from IMAGE_ASSETS.
66 + For huggingface runner, we provide the PIL images as input.
67 + For vllm runner, we provide MultiModalDataDict objects
68 + and corresponding MultiModalConfig as input.
69 + Note, the text input is also adjusted to abide by vllm contract.
70 + The text output is sanitized to be able to compare with hf.
71 + """
72 + # NOTE: take care of the order. run vLLM first, and then run HF.
73 + # vLLM needs a fresh new process without cuda initialization.
74 + # if we run HF first, the cuda initialization will be done and it
75 + # will hurt multiprocessing backend with fork method (the default method).
76 + # max_model_len should be greater than image_feature_size
77 + with vllm_runner(
78 + model,
79 + task="generate",
80 + max_model_len=max_model_len,
81 + max_num_seqs=2,
82 + dtype=dtype,
83 + limit_mm_per_prompt={"image": mm_limit},
84 + tensor_parallel_size=tensor_parallel_size,
85 + distributed_executor_backend=distributed_executor_backend,
86 + enable_lora=True,
87 + max_lora_rank=320,
88 + gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
89 + enforce_eager=True,
90 + trust_remote_code=False,
91 + ) as vllm_model:
92 + lora_request = LoRARequest("vision", 1, vision_lora_path)
93 + vllm_outputs_per_case = [
94 + vllm_model.generate_greedy_logprobs(prompts,
95 + max_tokens,
96 + num_logprobs=num_logprobs,
97 + images=images,
98 + audios=audios,
99 + lora_request=lora_request)
100 + for prompts, images, audios in inputs
101 + ]
102 +
103 + with hf_runner(model, dtype=dtype) as hf_model:
104 + hf_model.model.load_adapter(
105 + vision_lora_path,
106 + adapter_name="vision",
107 + )
108 + hf_processor = hf_model.processor
109 + eos_token_id = hf_processor.tokenizer.eos_token_id
110 + hf_outputs_per_case = [
111 + hf_model.generate_greedy_logprobs_limit(prompts,
112 + max_tokens,
113 + num_logprobs=num_logprobs,
114 + images=images,
115 + audios=audios,
116 + eos_token_id=eos_token_id)
117 + for prompts, images, audios in inputs
118 + ]
119 +
120 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
121 + vllm_outputs_per_case):
122 + check_logprobs_close(
123 + outputs_0_lst=hf_outputs,
124 + outputs_1_lst=vllm_outputs,
125 + name_0="hf",
126 + name_1="vllm",
127 + )
128 +
129 +
130 +@pytest.mark.parametrize("model", models)
131 +@pytest.mark.parametrize(
132 + "size_factors",
133 + [
134 + # No image
135 + [],
136 + # Single-scale
137 + [1.0],
138 + # Single-scale, batched
139 + [1.0, 1.0, 1.0],
140 + # Multi-scale
141 + [0.25, 0.5, 1.0],
142 + ],
143 +)
144 +@pytest.mark.parametrize("dtype", [target_dtype])
145 +@pytest.mark.parametrize("max_model_len", [12800])
146 +@pytest.mark.parametrize("max_tokens", [128])
147 +@pytest.mark.parametrize("num_logprobs", [10])
148 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
149 + dtype: str, max_model_len: int, max_tokens: int,
150 + num_logprobs: int) -> None:
151 + images = [asset.pil_image for asset in image_assets]
152 +
153 + inputs_per_image = [(
154 + [prompt for _ in size_factors],
155 + [rescale_image_size(image, factor) for factor in size_factors],
156 + None,
157 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
158 +
159 + run_test(
160 + hf_runner,
161 + vllm_runner,
162 + inputs_per_image,
163 + model,
164 + dtype=dtype,
165 + max_model_len=max_model_len,
166 + max_tokens=max_tokens,
167 + num_logprobs=num_logprobs,
168 + mm_limit=1,
169 + tensor_parallel_size=1,
170 + )
171 +
172 +
173 +@large_gpu_test(min_gb=48)
174 +@pytest.mark.parametrize("model", models)
175 +@pytest.mark.parametrize(
176 + "size_factors",
177 + [
178 + # No image
179 + # [],
180 + # Single-scale
181 + [1.0],
182 + # Single-scale, batched
183 + [1.0, 1.0, 1.0],
184 + # Multi-scale
185 + [0.25, 0.5, 1.0],
186 + ],
187 +)
188 +@pytest.mark.parametrize("dtype", [target_dtype])
189 +@pytest.mark.parametrize("max_model_len", [25600])
190 +@pytest.mark.parametrize("max_tokens", [128])
191 +@pytest.mark.parametrize("num_logprobs", [10])
192 +def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
193 + size_factors, dtype: str, max_model_len: int,
194 + max_tokens: int, num_logprobs: int) -> None:
195 + images = [asset.pil_image for asset in image_assets]
196 +
197 + inputs_per_case = [
198 + (
199 + [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
200 + [[rescale_image_size(image, factor) for image in images]
201 + for factor in size_factors],
202 + None,
203 + ),
204 + ]
205 +
206 + run_test(
207 + hf_runner,
208 + vllm_runner,
209 + inputs_per_case,
210 + model,
211 + dtype=dtype,
212 + max_model_len=max_model_len,
213 + max_tokens=max_tokens,
214 + num_logprobs=num_logprobs,
215 + mm_limit=2,
216 + tensor_parallel_size=1,
217 + )
218 +
219 +
220 +@pytest.mark.parametrize("model", models)
221 +@pytest.mark.parametrize("dtype", [target_dtype])
222 +@pytest.mark.parametrize("max_model_len", [12800])
223 +@pytest.mark.parametrize("max_tokens", [128])
224 +@pytest.mark.parametrize("num_logprobs", [10])
225 +def test_vision_speech_models(hf_runner, vllm_runner, model, dtype: str,
226 + max_model_len: int, max_tokens: int,
227 + num_logprobs: int) -> None:
228 +
229 + # use the example speech question so that the model outputs are reasonable
230 + audio = librosa.load(speech_question, sr=16000)
231 + image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
232 +
233 + inputs_vision_speech = [
234 + (
235 + ["<|user|><|image|><|audio|><|end|><|assistant|>"],
236 + [image],
237 + [audio],
238 + ),
239 + ]
240 +
241 + run_test(
242 + hf_runner,
243 + vllm_runner,
244 + inputs_vision_speech,
245 + model,
246 + dtype=dtype,
247 + max_model_len=max_model_len,
248 + max_tokens=max_tokens,
249 + num_logprobs=num_logprobs,
250 + mm_limit=1,
251 + tensor_parallel_size=1,
252 + )
tests/models/multimodal/processing/test_common.py
+31 -3
@@ -41,12 +41,18 @@ def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
41
42
43 def _test_processing_correctness(
44 - model_id: str,
44 + model_id_or_arch: str,
45 hit_rate: float,
46 num_batches: int,
47 simplify_rate: float,
48 ):
49 - model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
49 + if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
50 + # Use model architecture to get the default model id
51 + model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
52 + model_id = model_info.default
53 + else:
54 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
55 + model_id = model_id_or_arch
56 model_info.check_available_online(on_fail="skip")
57 model_info.check_transformers_version(on_fail="skip")
58
@@ -58,7 +64,7 @@ def _test_processing_correctness(
64 trust_remote_code=model_info.trust_remote_code,
65 seed=0,
66 dtype="auto",
61 - revision=None,
67 + revision=model_info.revision,
68 hf_overrides=model_info.hf_overrides,
69 )
70
@@ -331,6 +337,28 @@ def test_processing_correctness(
337 )
338
339
340 +# Phi4MultimodalForCausalLM share same model repo with original format
341 +# Phi4MMForCausalLM, so we add it as a separate test case
342 +# Remove this test after conversion PR merged:
343 +# https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/70
344 +@pytest.mark.parametrize("model_arch", ["Phi4MultimodalForCausalLM"])
345 +@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
346 +@pytest.mark.parametrize("num_batches", [32])
347 +@pytest.mark.parametrize("simplify_rate", [1.0])
348 +def test_processing_correctness_phi4_multimodal(
349 + model_arch: str,
350 + hit_rate: float,
351 + num_batches: int,
352 + simplify_rate: float,
353 +):
354 + _test_processing_correctness(
355 + model_arch,
356 + hit_rate=hit_rate,
357 + num_batches=num_batches,
358 + simplify_rate=simplify_rate,
359 + )
360 +
361 +
362 def _assert_inputs_equal(
363 a: MultiModalInputs,
364 b: MultiModalInputs,
tests/models/registry.py
+2
@@ -433,6 +433,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
433 "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B"}), # noqa: E501
434 "Phi4MMForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct",
435 trust_remote_code=True),
436 + "Phi4MultimodalForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct", # noqa: E501
437 + revision="refs/pr/70"),
438 "PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
439 tokenizer_mode="mistral"),
440 "QwenVLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen-VL",
vllm/model_executor/models/phi4_multimodal.py new
+1455
@@ -0,0 +1,1455 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import math
4 +from collections.abc import Iterable, Mapping, Sequence
5 +from typing import Any, Literal, Optional, TypedDict, Union
6 +
7 +import numpy as np
8 +import torch
9 +import torch.nn as nn
10 +import torch.nn.functional as F
11 +from transformers import (BatchFeature, Phi4MultimodalAudioConfig,
12 + Phi4MultimodalConfig, Phi4MultimodalFeatureExtractor,
13 + Phi4MultimodalImageProcessorFast)
14 +from transformers import Phi4MultimodalProcessor as Phi4MMProcessor
15 +from transformers.models.phi4_multimodal.modeling_phi4_multimodal import (
16 + Phi4MultimodalAudioConvModule, Phi4MultimodalAudioNemoConvSubsampling,
17 + Phi4MultimodalAudioRelativeAttentionBias, adaptive_enc_mask, unfold_tensor)
18 +
19 +from vllm.config import VllmConfig
20 +from vllm.distributed import (divide, get_tensor_model_parallel_rank,
21 + get_tensor_model_parallel_world_size)
22 +from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
23 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24 + MergedColumnParallelLinear,
25 + QKVParallelLinear,
26 + RowParallelLinear)
27 +from vllm.model_executor.layers.quantization import QuantizationConfig
28 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 +from vllm.model_executor.models.module_mapping import MultiModelKeys
30 +from vllm.model_executor.sampling_metadata import SamplingMetadata
31 +from vllm.multimodal import MULTIMODAL_REGISTRY
32 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
33 + MultiModalKwargs, NestedTensors)
34 +from vllm.multimodal.parse import (AudioProcessorItems, ImageEmbeddingItems,
35 + ImageProcessorItems, ImageSize,
36 + MultiModalDataItems, MultiModalDataParser)
37 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
38 + BaseProcessingInfo, PromptReplacement,
39 + PromptUpdate)
40 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
41 +from vllm.sequence import IntermediateTensors
42 +from vllm.utils import is_list_of
43 +
44 +from .idefics2_vision_model import Idefics2VisionTransformer
45 +from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
46 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
47 + init_vllm_registered_model, maybe_prefix,
48 + merge_multimodal_embeddings)
49 +
50 +# <|endoftext10|> (see vocab.json in hf model)
51 +_IMAGE_PLACEHOLDER_TOKEN_ID = 200010
52 +# <|endoftext11|>
53 +_AUDIO_PLACEHOLDER_TOKEN_ID = 200011
54 +
55 +_AUDIO_MAX_SOUNDFILE_SIZE = 241_000
56 +
57 +
58 +def _get_padding_size(orig_width: int, orig_height: int, target_height: int,
59 + target_width: int):
60 + ratio_width = target_width / orig_width
61 + ratio_height = target_height / orig_height
62 +
63 + if ratio_width < ratio_height:
64 + padding_width = 0
65 + padding_height = target_height - int(orig_height * ratio_width)
66 + else:
67 + padding_width = target_width - int(orig_width * ratio_height)
68 + padding_height = 0
69 + return padding_height, padding_width
70 +
71 +
72 +class Phi4MMProjector(nn.Module):
73 +
74 + def __init__(self, input_size: int, hidden_size: int):
75 + super().__init__()
76 + self.up = ColumnParallelLinear(input_size, hidden_size)
77 + self.down = RowParallelLinear(hidden_size, hidden_size)
78 + self.act = get_act_fn("gelu")
79 +
80 + def forward(self, x: torch.Tensor) -> torch.Tensor:
81 + x, _ = self.up(x)
82 + x = self.act(x)
83 + x, _ = self.down(x)
84 + return x
85 +
86 +
87 +class Phi4MMImageEmbedding(nn.Module):
88 + """Image embedding."""
89 +
90 + def __init__(self, config: Phi4MultimodalConfig):
91 + super().__init__()
92 + self.config = config
93 + self.layer_idx = config.vision_config.feature_layer
94 + self.crop_size = config.vision_config.crop_size
95 + self.image_dim_out = config.vision_config.hidden_size
96 +
97 + n_patches = (config.vision_config.image_size //
98 + config.vision_config.patch_size)
99 + if n_patches % 2 != 0:
100 + self.img_processor_padding = nn.ReflectionPad2d((0, 1, 0, 1))
101 + n_patches += 1
102 + self.num_img_tokens = (n_patches // 2)**2
103 +
104 + num_hidden_layers = (config.vision_config.num_hidden_layers +
105 + self.layer_idx +
106 + 1 if self.layer_idx < 0 else self.layer_idx + 1)
107 + self.img_processor = Idefics2VisionTransformer(
108 + config.vision_config,
109 + require_post_norm=False,
110 + num_hidden_layers_override=num_hidden_layers)
111 + self.image_token_compression = nn.AvgPool2d(kernel_size=2, stride=2)
112 + self.img_projection = Phi4MMProjector(self.image_dim_out,
113 + config.hidden_size)
114 + self.global_img_feature_extensor = nn.Parameter(
115 + torch.zeros([1, 1, self.image_dim_out]))
116 + self.sub_img_feature_extensor = nn.Parameter(
117 + torch.zeros([1, 1, 1, self.image_dim_out]))
118 +
119 + def get_img_features(
120 + self,
121 + img_embeds: torch.FloatTensor,
122 + attention_mask: Optional[torch.Tensor] = None,
123 + ) -> torch.FloatTensor:
124 + img_feature = self.img_processor(img_embeds,
125 + patch_attention_mask=attention_mask)
126 +
127 + patch_feature = img_feature
128 + # reshape to 2D tensor
129 + width = int(math.sqrt(patch_feature.size(1)))
130 + patch_feature = patch_feature.view(-1, width, width,
131 + patch_feature.size(-1))
132 + # convert to NCHW
133 + patch_feature = patch_feature.permute(0, 3, 1, 2)
134 + if getattr(self, "img_processor_padding", None) is not None:
135 + patch_feature = self.img_processor_padding(patch_feature)
136 + patch_feature = self.image_token_compression(patch_feature)
137 + # convert to NHWC
138 + patch_feature = patch_feature.permute(0, 2, 3, 1)
139 + patch_feature = patch_feature.view(
140 + -1,
141 + patch_feature.size(1) * patch_feature.size(2),
142 + patch_feature.size(-1))
143 + return patch_feature
144 +
145 + def forward(
146 + self,
147 + image_pixel_values: torch.FloatTensor,
148 + image_sizes: Optional[torch.Tensor] = None,
149 + image_attention_mask: Optional[torch.Tensor] = None,
150 + ) -> torch.FloatTensor:
151 + image_pixel_values = image_pixel_values.to(
152 + self.img_processor.embeddings.patch_embedding.weight.dtype)
153 +
154 + target_device = self.img_projection.up.bias.device
155 + target_dtype = self.img_projection.up.bias.dtype
156 +
157 + batch_size = image_pixel_values.shape[0]
158 +
159 + img_features = self.get_img_features(
160 + image_pixel_values.flatten(0, 1),
161 + attention_mask=image_attention_mask.flatten(0, 1).to(
162 + dtype=bool, device=target_device),
163 + )
164 + base_feat_size = int(np.sqrt(img_features.shape[1]))
165 + img_features = img_features.view(batch_size, -1, base_feat_size**2,
166 + self.image_dim_out)
167 + image_sizes = image_sizes.view(-1, 2)
168 +
169 + output_imgs = []
170 + for idx in range(batch_size):
171 + height, width = image_sizes[idx]
172 + height_ratio = height // self.crop_size
173 + width_ratio = width // self.crop_size
174 + area_ratio = height_ratio * width_ratio
175 +
176 + global_img = img_features[idx, :1]
177 + global_img = global_img.reshape(1, base_feat_size, base_feat_size,
178 + self.image_dim_out).contiguous()
179 + temporary_extensor = self.sub_img_feature_extensor.repeat(
180 + 1, base_feat_size, 1, 1)
181 + global_img = torch.cat([global_img, temporary_extensor],
182 + dim=2).reshape(1, -1, self.image_dim_out)
183 +
184 + sub_img = img_features[idx, 1:]
185 + sub_img = sub_img[:area_ratio]
186 + sub_img = (sub_img.reshape(
187 + height_ratio, width_ratio, base_feat_size, base_feat_size,
188 + self.image_dim_out).transpose(1, 2).reshape(
189 + 1, height_ratio * base_feat_size,
190 + width_ratio * base_feat_size,
191 + self.image_dim_out).contiguous())
192 +
193 + if image_attention_mask is not None:
194 + reshaped_image_attention_mask = (
195 + image_attention_mask[idx, 1:area_ratio + 1,
196 + 0::2, 0::2].reshape(
197 + height_ratio, width_ratio,
198 + base_feat_size,
199 + base_feat_size).transpose(
200 + 1, 2).reshape(
201 + 1, height_ratio *
202 + base_feat_size,
203 + width_ratio *
204 + base_feat_size))
205 + useful_height = int(
206 + reshaped_image_attention_mask[0, :, 0].sum().item())
207 + useful_width = int(
208 + reshaped_image_attention_mask[0, 0, :].sum().item())
209 + sub_img = sub_img[:, :useful_height, :useful_width]
210 + temporary_extensor = self.sub_img_feature_extensor.repeat(
211 + 1, useful_height, 1, 1)
212 + else:
213 + temporary_extensor = self.sub_img_feature_extensor.repeat(
214 + 1, height_ratio * base_feat_size, 1, 1)
215 +
216 + sub_img = torch.cat([sub_img, temporary_extensor],
217 + dim=2).reshape(1, -1, self.image_dim_out)
218 +
219 + # Merge global and sub
220 + output_imgs.append(
221 + torch.cat(
222 + [sub_img, self.global_img_feature_extensor, global_img],
223 + dim=1))
224 +
225 + img_set_tensor = []
226 + for output_img in output_imgs:
227 + output_img = output_img.to(device=target_device,
228 + dtype=target_dtype)
229 + img_feature_proj = self.img_projection(output_img)
230 + img_set_tensor.append(img_feature_proj.flatten(0, 1))
231 +
232 + return img_set_tensor
233 +
234 +
235 +class Phi4MultimodalAudioMLP(nn.Module):
236 +
237 + def __init__(
238 + self,
239 + config: Phi4MultimodalAudioConfig,
240 + quant_config: Optional[QuantizationConfig] = None,
241 + prefix: str = "",
242 + ):
243 + super().__init__()
244 + self.layer_norm = nn.LayerNorm(config.hidden_size)
245 + self.act_fn = MulAndSilu()
246 + self.gate_up_proj = MergedColumnParallelLinear(
247 + config.hidden_size, [config.intermediate_size] * 2,
248 + bias=True,
249 + quant_config=quant_config,
250 + prefix=f"{prefix}.gate_up_proj")
251 + self.down_proj = RowParallelLinear(config.intermediate_size,
252 + config.hidden_size,
253 + bias=True,
254 + quant_config=quant_config,
255 + prefix=f"{prefix}.down_proj")
256 +
257 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
258 + hidden_states = self.layer_norm(hidden_states)
259 + hidden_states, _ = self.gate_up_proj(hidden_states)
260 + hidden_states = self.act_fn(hidden_states)
261 + hidden_states, _ = self.down_proj(hidden_states)
262 + return hidden_states
263 +
264 +
265 +class Phi4MultimodalAudioAttention(nn.Module):
266 +
267 + def __init__(
268 + self,
269 + config: Phi4MultimodalAudioConfig,
270 + quant_config: Optional[QuantizationConfig] = None,
271 + prefix: str = "",
272 + ):
273 + super().__init__()
274 + self.config = config
275 + self.embed_dim = config.hidden_size
276 + self.total_num_heads = config.num_attention_heads
277 + self.head_dim = self.embed_dim // self.total_num_heads
278 + if self.head_dim * self.total_num_heads != self.embed_dim:
279 + raise ValueError(
280 + "embed_dim must be divisible by num_heads "
281 + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
282 + f" {self.num_heads}).")
283 + self.scale = self.head_dim**-0.5
284 +
285 + self.qkv_proj = QKVParallelLinear(
286 + hidden_size=self.embed_dim,
287 + head_size=self.head_dim,
288 + total_num_heads=self.total_num_heads,
289 + quant_config=quant_config,
290 + prefix=f"{prefix}.qkv_proj",
291 + )
292 +
293 + self.o_proj = RowParallelLinear(
294 + input_size=self.embed_dim,
295 + output_size=self.embed_dim,
296 + quant_config=quant_config,
297 + prefix=f"{prefix}.out_proj",
298 + )
299 +
300 + self.tp_size = get_tensor_model_parallel_world_size()
301 + self.tp_rank = get_tensor_model_parallel_rank()
302 + self.num_heads = divide(self.total_num_heads, self.tp_size)
303 +
304 + def split_attn_mask(self, attention_mask: torch.Tensor) -> torch.Tensor:
305 + start_idx = self.num_heads * self.tp_rank
306 + end_idx = self.num_heads * (self.tp_rank + 1)
307 + return attention_mask[:, start_idx:end_idx]
308 +
309 + def forward(
310 + self,
311 + hidden_states: torch.Tensor,
312 + attention_mask: torch.Tensor,
313 + ) -> torch.Tensor:
314 + qkv_states, _ = self.qkv_proj(hidden_states)
315 + query, key, value = qkv_states.chunk(3, dim=-1)
316 +
317 + bsz, seq_len, _ = query.size()
318 + query = query.view(bsz, seq_len, self.num_heads, self.head_dim)
319 + key = key.view(bsz, seq_len, self.num_heads, self.head_dim)
320 + value = value.view(bsz, seq_len, self.num_heads, self.head_dim)
321 + query, key, value = (x.transpose(1, 2) for x in (query, key, value))
322 +
323 + attention_mask = self.split_attn_mask(attention_mask)
324 + out = F.scaled_dot_product_attention(
325 + query,
326 + key,
327 + value,
328 + scale=self.scale,
329 + attn_mask=attention_mask,
330 + )
331 + out = out.transpose(1, 2).reshape(bsz, seq_len, -1)
332 +
333 + attn_output, _ = self.o_proj(out)
334 +
335 + return attn_output
336 +
337 +
338 +class Phi4MultimodalAudioConformerEncoderLayer(nn.Module):
339 +
340 + def __init__(self, config: Phi4MultimodalAudioConfig):
341 + super().__init__()
342 +
343 + self.feed_forward_in = Phi4MultimodalAudioMLP(config)
344 + self.self_attn = Phi4MultimodalAudioAttention(config)
345 + self.conv = Phi4MultimodalAudioConvModule(config)
346 + self.feed_forward_out = Phi4MultimodalAudioMLP(config)
347 + self.layer_norm_att = nn.LayerNorm(config.hidden_size)
348 + self.layer_norm = nn.LayerNorm(config.hidden_size)
349 +
350 + def forward(
351 + self,
352 + hidden_states: torch.Tensor,
353 + attention_mask: torch.Tensor,
354 + ) -> torch.Tensor:
355 + residual = hidden_states + 0.5 * self.feed_forward_in(hidden_states)
356 + hidden_states = self.layer_norm_att(residual)
357 +
358 + hidden_states = residual + self.self_attn(hidden_states,
359 + attention_mask)
360 + hidden_states = hidden_states + self.conv(hidden_states)
361 + hidden_states = hidden_states + 0.5 * self.feed_forward_out(
362 + hidden_states)
363 +
364 + out = self.layer_norm(hidden_states)
365 +
366 + return out
367 +
368 +
369 +class Phi4MMAudioMeanVarianceNormLayer(nn.Module):
370 + """Mean/variance normalization layer.
371 +
372 + Will subtract mean and multiply input by inverted standard deviation.
373 + Typically used as a very first layer in a model.
374 +
375 + Args:
376 + input_size: int
377 + layer input size.
378 + """
379 +
380 + def __init__(self, config: Phi4MultimodalAudioConfig):
381 + super().__init__()
382 + self.global_mean = nn.Parameter(torch.zeros(config.input_size))
383 + self.global_invstd = nn.Parameter(torch.ones(config.input_size))
384 +
385 + def forward(self, input_: torch.Tensor) -> torch.Tensor:
386 + """MeanVarianceNormLayer Forward
387 +
388 + Args:
389 + input_: torch.Tensor
390 + input tensor.
391 + """
392 + return (input_ - self.global_mean) * self.global_invstd
393 +
394 +
395 +class Phi4MultimodalAudioModel(nn.Module):
396 +
397 + def __init__(self, config: Phi4MultimodalAudioConfig):
398 + super().__init__()
399 + self.config = config
400 +
401 + self.encoder_embedding = Phi4MMAudioMeanVarianceNormLayer(config)
402 + self.embed = Phi4MultimodalAudioNemoConvSubsampling(config)
403 + self.relative_attention_bias_layer = (
404 + Phi4MultimodalAudioRelativeAttentionBias(config))
405 + self.encoders = nn.ModuleList([
406 + Phi4MultimodalAudioConformerEncoderLayer(config)
407 + for _ in range(config.num_blocks)
408 + ])
409 +
410 + def _streaming_mask(
411 + self,
412 + seq_len: int,
413 + batch_size: int,
414 + chunk_size: int,
415 + left_chunk: int,
416 + ):
417 + # Create mask matrix for streaming
418 + # S stores start index. if chunksize is 18, s is [0,18,36,....]
419 + chunk_start_idx = np.arange(0, seq_len, chunk_size)
420 +
421 + enc_streaming_mask = (adaptive_enc_mask(
422 + seq_len, chunk_start_idx,
423 + left_window=left_chunk).unsqueeze(0).expand([batch_size, -1, -1]))
424 + return enc_streaming_mask
425 +
426 + def forward_embeddings(
427 + self,
428 + hidden_states: torch.Tensor,
429 + masks: torch.Tensor,
430 + ):
431 + """Forwarding the inputs through the top embedding layers"""
432 + seq_len = math.ceil(hidden_states.shape[1] /
433 + self.config.time_reduction)
434 + if seq_len <= 0:
435 + raise ValueError(
436 + f"Sequence length after time reduction is invalid: {seq_len}."
437 + "Your input feature is too short.")
438 +
439 + batch_size = hidden_states.shape[0]
440 +
441 + enc_streaming_mask = self._streaming_mask(seq_len, batch_size,
442 + self.config.chunk_size,
443 + self.config.left_chunk)
444 + enc_streaming_mask = enc_streaming_mask.to(hidden_states.device)
445 +
446 + hidden_states, masks = self.embed(hidden_states, masks)
447 +
448 + streaming_mask = enc_streaming_mask
449 + if streaming_mask is not None and masks is not None:
450 + hs_mask = masks & streaming_mask
451 + elif masks is not None:
452 + hs_mask = masks
453 + else:
454 + hs_mask = streaming_mask
455 +
456 + return hidden_states, hs_mask, masks
457 +
458 + def calculate_hs_mask(self, hidden_states: torch.Tensor,
459 + device: torch.device, mask: torch.Tensor):
460 + max_audio_length = hidden_states.shape[1]
461 + batch_size = hidden_states.shape[0]
462 + enc_streaming_mask = self._streaming_mask(max_audio_length, batch_size,
463 + self.config.chunk_size,
464 + self.config.left_chunk)
465 + enc_streaming_mask = enc_streaming_mask.to(device)
466 + if mask is None:
467 + return enc_streaming_mask
468 +
469 + feature_lens = mask.sum(1)
470 + padding_length = feature_lens
471 + pad_mask = torch.arange(0, max_audio_length, device=device).expand(
472 + padding_length.size(0), -1) < padding_length.unsqueeze(1)
473 + pad_mask = pad_mask.unsqueeze(1)
474 + pad_mask = pad_mask & enc_streaming_mask
475 + return pad_mask
476 +
477 + def forward(self,
478 + hidden_states: torch.Tensor,
479 + mask: Optional[torch.Tensor] = None):
480 + hidden_states = self.encoder_embedding(hidden_states)
481 + hidden_states, hs_mask, mask = self.forward_embeddings(
482 + hidden_states, mask)
483 +
484 + unfolded = False
485 + bs, seq_len, _ = hidden_states.shape
486 + max_seq_len = 500 # maximum position for absolute positional encoding
487 + if seq_len > max_seq_len:
488 + # audio sequence is longer than max_seq_len,
489 + # unfold it into chunks of max_seq_len
490 + unfolded = True
491 + # the unfold op will drop residual frames,
492 + # pad it to the multiple of max_seq_len
493 + if seq_len % max_seq_len > 0:
494 + chunk_pad_size = max_seq_len - (seq_len % max_seq_len)
495 + else:
496 + chunk_pad_size = 0
497 + if chunk_pad_size > 0:
498 + hidden_states_pad = F.pad(hidden_states,
499 + (0, 0, 0, chunk_pad_size),
500 + "constant", 0)
501 + hidden_states = hidden_states_pad.to(hidden_states.device)
502 +
503 + hidden_states = unfold_tensor(hidden_states, max_seq_len)
504 + masks_unfold = None
505 + if mask is not None:
506 + # revise hs_mask here because the previous calculated hs_mask
507 + # did not consider extra pad
508 + subsampled_pad_mask = mask.squeeze(
509 + 1) # [bz, subsampled_unmask_seq_len]
510 + extra_padded_subsamlped_pad_mask = F.pad(
511 + subsampled_pad_mask, (0, chunk_pad_size), "constant",
512 + False) # extra padding to the pad mask
513 + extra_padded_subsamlped_pad_mask = (
514 + extra_padded_subsamlped_pad_mask.unsqueeze(-1).float())
515 + masks_unfold = unfold_tensor(
516 + extra_padded_subsamlped_pad_mask, max_seq_len
517 + ) # unfold the pad mask like we did to the input tensor
518 + masks_unfold = masks_unfold.squeeze(
519 + -1).bool() # unfold op does not support bool tensor
520 + hs_mask = self.calculate_hs_mask(
521 + hidden_states, hidden_states.device, masks_unfold
522 + ) # calculate hs_mask based on the unfolded pad mask
523 +
524 + relative_attention_bias = self.relative_attention_bias_layer(
525 + hidden_states)
526 + attention_mask = hs_mask.unsqueeze(1) + relative_attention_bias
527 +
528 + for layer in self.encoders:
529 + hidden_states = layer(hidden_states, attention_mask)
530 +
531 + if unfolded:
532 + embed_dim = hidden_states.shape[-1]
533 + hidden_states = hidden_states.reshape(bs, -1, embed_dim)
534 + # if we ever padded before unfolding, we need to remove the padding
535 + if chunk_pad_size > 0:
536 + hidden_states = hidden_states[:, :-chunk_pad_size, :]
537 +
538 + return hidden_states
539 +
540 +
541 +class Phi4MMAudioEmbedding(nn.Module):
542 +
543 + def __init__(self, config: Phi4MultimodalConfig):
544 + super().__init__()
545 + self.config = config
546 + self.layer_idx = config.audio_config.feature_layer
547 +
548 + self.encoder = Phi4MultimodalAudioModel(config.audio_config)
549 +
550 + audio_config = config.audio_config
551 + proj_input_size = (audio_config.hidden_size *
552 + audio_config.downsample_rate)
553 + self.vision_speech_projection = Phi4MMProjector(
554 + proj_input_size, config.hidden_size)
555 + self.speech_projection = Phi4MMProjector(proj_input_size,
556 + config.hidden_size)
557 +
558 + def get_projection(
559 + self,
560 + audio_projection_mode: Literal["speech", "vision"],
561 + ) -> Phi4MMProjector:
562 + if audio_projection_mode == "speech":
563 + return self.speech_projection
564 + elif audio_projection_mode == "vision":
565 + return self.vision_speech_projection
566 +
567 + def forward(
568 + self,
569 + audio_input_features: torch.FloatTensor,
570 + audio_embed_sizes=None,
571 + audio_attention_mask=None,
572 + audio_projection_mode="speech",
573 + ) -> torch.FloatTensor:
574 +
575 + audio_projection = self.get_projection(audio_projection_mode)
576 +
577 + target_device = audio_projection.up.bias.device
578 + target_dtype = audio_projection.up.bias.dtype
579 +
580 + audio_input_features = audio_input_features.to(device=target_device,
581 + dtype=target_dtype)
582 +
583 + audio_encoder_hidden_states = self.encoder(audio_input_features,
584 + audio_attention_mask)
585 + audio_embeds = audio_projection(audio_encoder_hidden_states)
586 +
587 + return audio_embeds.flatten(0, 1)
588 +
589 + def load_weights(self, weights: Iterable[tuple[str,
590 + torch.Tensor]]) -> set[str]:
591 + stacked_params_mapping = [
592 + # (param_name, shard_name, shard_id)
593 + ("qkv_proj", "q_proj", "q"),
594 + ("qkv_proj", "k_proj", "k"),
595 + ("qkv_proj", "v_proj", "v"),
596 + ]
597 + params_dict = dict(self.named_parameters())
598 + loaded_params: set[str] = set()
599 +
600 + for name, loaded_weight in weights:
601 + for param_name, weight_name, shard_id in stacked_params_mapping:
602 + if weight_name not in name:
603 + continue
604 + name = name.replace(weight_name, param_name)
605 + param = params_dict[name]
606 + weight_loader = param.weight_loader
607 + weight_loader(param, loaded_weight, shard_id)
608 + break
609 + else:
610 + param = params_dict[name]
611 + weight_loader = getattr(param, "weight_loader",
612 + default_weight_loader)
613 + weight_loader(param, loaded_weight)
614 + loaded_params.add(name)
615 + return loaded_params
616 +
617 +
618 +class Phi4MMImagePixelInputs(TypedDict):
619 + type: Literal["pixel_values"]
620 + data: Union[torch.Tensor, list[torch.Tensor]]
621 + """
622 + Shape:
623 + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
624 +
625 + Note that `num_patches` may be different per batch and image,
626 + in which case the data is passed as a list instead of a batched tensor.
627 + """
628 +
629 + image_sizes: torch.Tensor
630 + """
631 + Shape: `(batch_size * num_images, 2)`
632 +
633 + This should be in `(height, width)` format.
634 + """
635 +
636 + num_img_tokens: list[int]
637 + """Shape: `(batch_size * num_images)`"""
638 +
639 + image_attention_mask: torch.Tensor
640 + """Shape: `(batch_size * num_images, H_mask, W_mask)`"""
641 +
642 +
643 +class Phi4MMImageEmbeddingInputs(TypedDict):
644 + type: Literal["image_embeds"]
645 + data: Union[torch.Tensor, list[torch.Tensor]]
646 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
647 +
648 + `hidden_size` must match the hidden size of language model backbone.
649 + """
650 +
651 +
652 +class Phi4MMAudioFeatureInputs(TypedDict):
653 + type: Literal["audio_features"]
654 + data: Union[torch.Tensor, list[torch.Tensor]]
655 + """Shape: `(batch_size * num_audios, 80, M)"""
656 +
657 +
658 +class Phi4MMAudioEmbeddingInputs(TypedDict):
659 + type: Literal["audio_embeds"]
660 + data: NestedTensors
661 + """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
662 +
663 +
664 +Phi4MMImageInput = Union[Phi4MMImagePixelInputs, Phi4MMImageEmbeddingInputs]
665 +Phi4MMAudioInputs = Union[Phi4MMAudioFeatureInputs, Phi4MMAudioEmbeddingInputs]
666 +
667 +
668 +def cat_with_pad(tensors, dim, padding_value=0):
669 + """
670 + cat along dim, while pad to max for all other dims
671 + """
672 + ndim = tensors[0].dim()
673 + assert all(
674 + t.dim() == ndim for t in
675 + tensors[1:]), "All tensors must have the same number of dimensions"
676 +
677 + out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)]
678 + out_size[dim] = sum(t.shape[dim] for t in tensors)
679 + output = tensors[0].new_full(out_size, padding_value)
680 +
681 + index = 0
682 + for t in tensors:
683 + # Create a slice list where every dimension except dim is full slice
684 + slices = [slice(0, t.shape[d]) for d in range(ndim)]
685 + # Update only the concat dimension slice
686 + slices[dim] = slice(index, index + t.shape[dim])
687 +
688 + output[slices] = t
689 + index += t.shape[dim]
690 +
691 + return output
692 +
693 +
694 +class Phi4MMProcessingInfo(BaseProcessingInfo):
695 +
696 + def get_hf_config(self) -> Phi4MultimodalConfig:
697 + return self.ctx.get_hf_config(Phi4MultimodalConfig)
698 +
699 + def get_hf_processor(
700 + self,
701 + *,
702 + dynamic_hd: Optional[int] = None,
703 + **kwargs: object,
704 + ) -> Phi4MMProcessor:
705 + if dynamic_hd is not None:
706 + kwargs["dynamic_hd"] = dynamic_hd
707 +
708 + return self.ctx.get_hf_processor(**kwargs)
709 +
710 + def get_feature_extractor(self) -> Phi4MultimodalFeatureExtractor:
711 + return self.get_hf_processor().audio_processor
712 +
713 + def get_image_processor(
714 + self,
715 + processor: Optional[Phi4MMProcessor] = None,
716 + ) -> Phi4MultimodalImageProcessorFast:
717 + if processor is None:
718 + processor = self.get_hf_processor()
719 + return processor.image_processor
720 +
721 + def get_dynamic_hd(
722 + self,
723 + processor: Optional[Phi4MMProcessor] = None,
724 + ) -> int:
725 + return self.get_image_processor(processor).dynamic_hd
726 +
727 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
728 + return {"audio": None, "image": None}
729 +
730 + def _find_target_aspect_ratio(
731 + self,
732 + orig_width: int,
733 + orig_height: int,
734 + image_size: int,
735 + max_num: int,
736 + min_num: int,
737 + ):
738 + w_crop_num = math.ceil(orig_width / float(image_size))
739 + h_crop_num = math.ceil(orig_height / float(image_size))
740 + if w_crop_num * h_crop_num > max_num:
741 + aspect_ratio = orig_width / orig_height
742 +
743 + # calculate the existing image aspect ratio
744 + target_ratios = set((i, j) for i in range(1, max_num + 1)
745 + for j in range(1, max_num + 1)
746 + if i * j <= max_num and i * j >= min_num)
747 + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
748 +
749 + # find the closest aspect ratio to the target
750 + image_processor = self.get_image_processor()
751 + target_aspect_ratio = image_processor.find_closest_aspect_ratio(
752 + aspect_ratio,
753 + target_ratios,
754 + orig_width,
755 + orig_height,
756 + )
757 +
758 + # calculate the target width and height
759 + target_width = image_size * target_aspect_ratio[0]
760 + target_height = image_size * target_aspect_ratio[1]
761 + else:
762 + target_width = image_size * w_crop_num
763 + target_height = image_size * h_crop_num
764 + target_aspect_ratio = (w_crop_num, h_crop_num)
765 + return target_aspect_ratio, target_height, target_width
766 +
767 + def _compute_num_image_tokens(
768 + self,
769 + orig_width: int,
770 + orig_height: int,
771 + dynamic_hd_size: int,
772 + vit_image_size: int,
773 + vit_patch_size: int,
774 + token_compression_factor: int = 2,
775 + ):
776 + """
777 + compute the number of tokens an image is expected to take up considering
778 + the image encoder architecture and exclude output features containing
779 + only padding pixels
780 +
781 + for siglip, vit_image_size=448, vit_patch_size=14, so output will be
782 + 32x32 feature map
783 + NOTE right now, Phi4MM uses hard-coded token_compression_factor=2
784 + """
785 + assert vit_image_size % vit_patch_size == 0, (
786 + "vit_image_size must be divisible by vit_patch_size")
787 + assert (vit_image_size // vit_patch_size %
788 + token_compression_factor == 0), (
789 + "vit_image_size // vit_patch_size must be divisible by "
790 + "token_compression_factor")
791 +
792 + target_aspect_ratio, target_height, target_width = (
793 + self._find_target_aspect_ratio(orig_width,
794 + orig_height,
795 + vit_image_size,
796 + dynamic_hd_size,
797 + min_num=1))
798 + assert target_aspect_ratio[0] * vit_image_size == target_width, (
799 + f"{target_aspect_ratio[0]} * {vit_image_size} != {target_width}")
800 + assert target_aspect_ratio[1] * vit_image_size == target_height, (
801 + f"{target_aspect_ratio[1]} * {vit_image_size} != {target_height}")
802 + assert (target_height % vit_image_size == 0
803 + and target_width % vit_image_size == 0)
804 +
805 + padding_height, padding_width = _get_padding_size(
806 + orig_width, orig_height, target_height, target_width)
807 + assert padding_width == 0 or padding_height == 0, \
808 + "padding_width or padding_height must be 0"
809 +
810 + target_feat_width = target_width // vit_patch_size
811 + target_feat_height = target_height // vit_patch_size
812 + if padding_width >= vit_patch_size:
813 + assert padding_height == 0, "padding_height not 0"
814 + non_pad_feat_width = target_feat_width - math.floor(
815 + padding_width / vit_patch_size)
816 + non_pad_feat_height = target_feat_height
817 + elif padding_height >= vit_patch_size:
818 + assert padding_width == 0, "padding_width not 0"
819 + non_pad_feat_height = target_feat_height - math.floor(
820 + padding_height / vit_patch_size)
821 + non_pad_feat_width = target_feat_width
822 + else:
823 + # small padding shorter than a vit patch
824 + non_pad_feat_width = target_feat_width
825 + non_pad_feat_height = target_feat_height
826 +
827 + feat_width = non_pad_feat_width // token_compression_factor
828 + feat_height = non_pad_feat_height // token_compression_factor
829 + # NOTE it's possible that the non-padding feature is not divisible
830 + if non_pad_feat_width % token_compression_factor != 0:
831 + feat_width += 1
832 + if non_pad_feat_height % token_compression_factor != 0:
833 + feat_height += 1
834 + num_hd_patch_tokens = feat_width * feat_height
835 + num_hd_newline_tokens = feat_height
836 + vit_feature_size = vit_image_size // vit_patch_size
837 + num_global_image_tokens = (vit_feature_size //
838 + token_compression_factor)**2
839 + num_sep_tokens = 1
840 + num_global_image_newline_tokens = \
841 + vit_feature_size // token_compression_factor
842 +
843 + return (num_global_image_tokens + num_sep_tokens +
844 + num_hd_patch_tokens + num_hd_newline_tokens +
845 + num_global_image_newline_tokens)
846 +
847 + def get_num_image_tokens(
848 + self,
849 + *,
850 + image_width: int,
851 + image_height: int,
852 + processor: Optional[Phi4MMProcessor] = None,
853 + ) -> int:
854 + hf_config = self.get_hf_config()
855 + vision_config = hf_config.vision_config
856 + vit_image_size = vision_config.image_size
857 + vit_patch_size = vision_config.patch_size
858 +
859 + dynamic_hd_size = self.get_dynamic_hd(processor=processor)
860 +
861 + # we use default `token_compression_factor=2`,
862 + # since it's not in HF vision config.
863 + image_num_tokens = self._compute_num_image_tokens(
864 + image_width,
865 + image_height,
866 + dynamic_hd_size=dynamic_hd_size,
867 + vit_image_size=vit_image_size,
868 + vit_patch_size=vit_patch_size,
869 + )
870 +
871 + return image_num_tokens
872 +
873 + def get_image_size_with_most_features(
874 + self,
875 + processor: Optional[Phi4MMProcessor] = None,
876 + ) -> ImageSize:
877 + vit_image_size = self.get_hf_config().vision_config.image_size
878 +
879 + max_side = vit_image_size * self.get_dynamic_hd(processor=processor)
880 + return ImageSize(height=max_side, width=vit_image_size)
881 +
882 + def get_audio_num_frames(self, audio_len: int, sr: float) -> int:
883 + """
884 + Compute the output size of the `extract_features` method.
885 +
886 + Args:
887 + audio_len (int): Length of the input waveform in samples.
888 + sr (float): Sampling rate of the waveform, either 16000 or 8000.
889 +
890 + Returns:
891 + tuple (int, int): Output size as (T, D), where:
892 + T: Number of time frames.
893 + D: Number of Mel filterbank bins (80).
894 + """
895 +
896 + # Resample to 16000 or 8000 if needed
897 + if sr > 16000:
898 + audio_len //= sr // 16000
899 + elif 8000 <= sr < 16000:
900 + # We'll resample to 16K from 8K
901 + audio_len *= 2
902 + elif sr < 8000:
903 + raise RuntimeError(f"Unsupported sample rate {sr}")
904 +
905 + # Spectrogram parameters for 16 kHz
906 + win_length = 400 # Frame length in samples
907 + hop_length = 160 # Frame shift in samples
908 +
909 + # Calculate number of frames (T)
910 + num_frames = (audio_len - win_length) // hop_length + 1
911 + if num_frames < 1:
912 + raise ValueError("Waveform too short for given parameters.")
913 +
914 + # Return time frames (T)
915 + return num_frames
916 +
917 + def _compute_audio_embed_size(self, audio_frames: int) -> int:
918 + """
919 + Compute the size of audio embeddings from the number of audio frames.
920 + """
921 + # `_compute_audio_embed_size` in audio_processor use torch for
922 + # computation, therefore we re-implement it to use pythonic
923 + # numeric computation to avoid extra tensor conversion.
924 + audio_processor = self.get_feature_extractor()
925 + audio_compression_rate = audio_processor.audio_compression_rate
926 + audio_downsample_rate = audio_processor.audio_downsample_rate
927 +
928 + integer = audio_frames // audio_compression_rate
929 + remainder = audio_frames % audio_compression_rate
930 + result = integer + int(remainder > 0)
931 +
932 + integer = result // audio_downsample_rate
933 + remainder = result % audio_downsample_rate
934 + result = integer + int(remainder > 0) # qformer compression
935 +
936 + return result
937 +
938 +
939 +class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
940 +
941 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
942 + num_audios = mm_counts.get("audio", 0)
943 + num_images = mm_counts.get("image", 0)
944 +
945 + tokenizer = self.info.get_tokenizer()
946 + image_tokens: str = tokenizer.image_token * num_images
947 + audio_tokens: str = tokenizer.audio_token * num_audios
948 +
949 + return image_tokens + audio_tokens
950 +
951 + def get_dummy_mm_data(
952 + self,
953 + seq_len: int,
954 + mm_counts: Mapping[str, int],
955 + ) -> MultiModalDataDict:
956 + num_audios = mm_counts.get("audio", 0)
957 + num_images = mm_counts.get("image", 0)
958 +
959 + target_width, target_height = \
960 + self.info.get_image_size_with_most_features()
961 +
962 + mm_data = {
963 + "image":
964 + self._get_dummy_images(width=target_width,
965 + height=target_height,
966 + num_images=num_images),
967 + "audio":
968 + self._get_dummy_audios(length=_AUDIO_MAX_SOUNDFILE_SIZE,
969 + num_audios=num_audios),
970 + }
971 +
972 + return mm_data
973 +
974 +
975 +class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
976 +
977 + def _get_data_parser(self) -> MultiModalDataParser:
978 + feature_extractor = self.info.get_feature_extractor()
979 + return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
980 +
981 + def _call_hf_processor(
982 + self,
983 + prompt: str,
984 + mm_data: Mapping[str, object],
985 + mm_kwargs: Mapping[str, object],
986 + tok_kwargs: Mapping[str, object],
987 + ) -> BatchFeature:
988 + if not mm_data:
989 + prompt_ids = self.info.get_tokenizer().encode(prompt)
990 + prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
991 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
992 +
993 + audio_data = mm_data.pop("audios", [])
994 + if audio_data:
995 + mm_data['audio'] = audio_data
996 +
997 + processed_outputs = super()._call_hf_processor(prompt, mm_data,
998 + mm_kwargs, tok_kwargs)
999 +
1000 + if "image_pixel_values" in processed_outputs:
1001 + num_img_tokens = [
1002 + self.info.get_num_image_tokens(image_width=img_size[0],
1003 + image_height=img_size[1])
1004 + for img_size in processed_outputs["image_sizes"]
1005 + ]
1006 + processed_outputs["num_img_tokens"] = num_img_tokens
1007 +
1008 + if audio_data:
1009 + audio_features = processed_outputs['audio_input_features']
1010 + sr = self.info.get_feature_extractor().sampling_rate
1011 + feature_sizes = [
1012 + self.info.get_audio_num_frames(len(audio), sr)
1013 + for audio in audio_data
1014 + ]
1015 + processed_outputs['audio_input_features'] = [
1016 + audio_features[idx, :size]
1017 + for idx, size in enumerate(feature_sizes)
1018 + ]
1019 +
1020 + return processed_outputs
1021 +
1022 + def _get_mm_fields_config(
1023 + self,
1024 + hf_inputs: BatchFeature,
1025 + hf_processor_mm_kwargs: Mapping[str, object],
1026 + ) -> Mapping[str, MultiModalFieldConfig]:
1027 + return dict(
1028 + image_pixel_values=MultiModalFieldConfig.batched("image"),
1029 + image_attention_mask=MultiModalFieldConfig.batched("image"),
1030 + image_sizes=MultiModalFieldConfig.batched("image"),
1031 + num_img_tokens=MultiModalFieldConfig.batched("image"),
1032 + audio_input_features=MultiModalFieldConfig.batched("audio"),
1033 + )
1034 +
1035 + def _get_prompt_updates(
1036 + self,
1037 + mm_items: MultiModalDataItems,
1038 + hf_processor_mm_kwargs: Mapping[str, Any],
1039 + out_mm_kwargs: MultiModalKwargs,
1040 + ) -> Sequence[PromptUpdate]:
1041 + tokenizer = self.info.get_tokenizer()
1042 + image_token_id = tokenizer.vocab[tokenizer.image_token]
1043 + audio_token_id = tokenizer.vocab[tokenizer.audio_token]
1044 +
1045 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1046 + audio_processor = self.info.get_feature_extractor()
1047 +
1048 + def get_image_replacement_phi4mm(item_idx: int):
1049 + images = mm_items.get_items(
1050 + "image", (ImageEmbeddingItems, ImageProcessorItems))
1051 +
1052 + if isinstance(images, ImageEmbeddingItems):
1053 + num_image_tokens = images.get_feature_size(item_idx)
1054 + else:
1055 + image_size = images.get_image_size(item_idx)
1056 + num_image_tokens = self.info.get_num_image_tokens(
1057 + image_width=image_size.width,
1058 + image_height=image_size.height,
1059 + processor=hf_processor,
1060 + )
1061 +
1062 + image_tokens = [image_token_id] * num_image_tokens
1063 +
1064 + return image_tokens
1065 +
1066 + def get_audio_replacement_phi4mm(item_idx: int):
1067 + audios = mm_items.get_items("audio", AudioProcessorItems)
1068 + # TODO(Isotr0py): support embedding inputs
1069 + audio_len = audios.get_audio_length(item_idx)
1070 + audio_frames = self.info.get_audio_num_frames(
1071 + audio_len, audio_processor.sampling_rate)
1072 + audio_embed_size = self.info._compute_audio_embed_size(
1073 + audio_frames)
1074 +
1075 + audio_tokens = [audio_token_id] * audio_embed_size
1076 +
1077 + return audio_tokens
1078 +
1079 + return [
1080 + PromptReplacement(
1081 + modality="audio",
1082 + target=[audio_token_id],
1083 + replacement=get_audio_replacement_phi4mm,
1084 + ),
1085 + PromptReplacement(
1086 + modality="image",
1087 + target=[image_token_id],
1088 + replacement=get_image_replacement_phi4mm,
1089 + ),
1090 + ]
1091 +
1092 +
1093 +@MULTIMODAL_REGISTRY.register_processor(
1094 + Phi4MMMultiModalProcessor,
1095 + info=Phi4MMProcessingInfo,
1096 + dummy_inputs=Phi4MMDummyInputsBuilder,
1097 +)
1098 +class Phi4MultimodalForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1099 + """
1100 + Implements the Phi-4-multimodal-instruct model in vLLM.
1101 + """
1102 + packed_modules_mapping = {
1103 + "qkv_proj": [
1104 + "qkv_proj",
1105 + ],
1106 + "gate_up_proj": [
1107 + "gate_up_proj",
1108 + ],
1109 + }
1110 +
1111 + hf_to_vllm_mapper = WeightsMapper(
1112 + orig_to_new_prefix={
1113 + # Multimodal embedding
1114 + "model.embed_tokens_extend.": "",
1115 + # LLM backbone
1116 + "model.": "language_model.model.",
1117 + },
1118 + orig_to_new_substr={
1119 + # projection
1120 + ".img_projection_": ".img_projection.",
1121 + ".up_proj_for_speech.": ".speech_projection.up.",
1122 + ".up_proj_for_vision_speech.": ".vision_speech_projection.up.",
1123 + ".down_proj_for_speech.": ".speech_projection.down.",
1124 + ".down_proj_for_vision_speech.": ".vision_speech_projection.down.",
1125 + },
1126 + )
1127 +
1128 + @classmethod
1129 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
1130 + if modality.startswith("image"):
1131 + return "<|image|>"
1132 + if modality.startswith("audio"):
1133 + return "<|audio|>"
1134 +
1135 + raise ValueError("Only image or audio modality is supported")
1136 +
1137 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1138 + super().__init__()
1139 + config = vllm_config.model_config.hf_config
1140 + multimodal_config = vllm_config.model_config.multimodal_config
1141 + self.config = config
1142 + self.multimodal_config = multimodal_config
1143 +
1144 + # TODO: Optionally initializes these for supporting input embeddings.
1145 + self.image_embed = Phi4MMImageEmbedding(
1146 + config,
1147 + # prefix=maybe_prefix(prefix, "image_embed"),
1148 + )
1149 + self.audio_embed = Phi4MMAudioEmbedding(
1150 + config,
1151 + # prefix=maybe_prefix(prefix, "audio_embed"),
1152 + )
1153 +
1154 + self.language_model = init_vllm_registered_model(
1155 + vllm_config=vllm_config,
1156 + prefix=maybe_prefix(prefix, "language_model"),
1157 + architectures=["Phi3ForCausalLM"],
1158 + )
1159 +
1160 + self.make_empty_intermediate_tensors = (
1161 + self.language_model.make_empty_intermediate_tensors)
1162 +
1163 + def _parse_and_validate_audio_input(
1164 + self, **kwargs: object) -> Optional[Phi4MMAudioInputs]:
1165 + """
1166 + Parse and validate the audio input to the model. This handles both
1167 + audio features and audio embeddings, but only the former is used for
1168 + now.
1169 +
1170 + Args:
1171 + kwargs (object): Keyword arguments.
1172 +
1173 + Returns:
1174 + Optional[Phi4MMAudioInputs]: Parsed and validated audio inputs.
1175 + """
1176 + audio_features = kwargs.pop("audio_input_features", None)
1177 + audio_embeds = kwargs.pop("audio_embeds", None)
1178 +
1179 + if audio_features is None and audio_embeds is None:
1180 + return None
1181 +
1182 + if audio_features is not None:
1183 + if not isinstance(audio_features, (torch.Tensor, list)):
1184 + raise ValueError("Incorrect type of audio features. "
1185 + f"Got type: {type(audio_features)}")
1186 +
1187 + return Phi4MMAudioFeatureInputs(type="audio_features",
1188 + data=flatten_bn(audio_features))
1189 +
1190 + if audio_embeds is not None:
1191 + if not isinstance(audio_embeds, (torch.Tensor, list)):
1192 + raise ValueError("Incorrect type of audio embeds. "
1193 + f"Got type: {type(audio_embeds)}")
1194 +
1195 + return Phi4MMAudioEmbeddingInputs(type="audio_embeds",
1196 + data=audio_embeds)
1197 +
1198 + raise AssertionError("This line should be unreachable.")
1199 +
1200 + def _process_audio_input(self, audio_input: Phi4MMAudioInputs,
1201 + audio_projection_mode: str) -> NestedTensors:
1202 + """
1203 + Create the audio embeddings from the audio input, where the audio input
1204 + is pairs of audio features and audio embed lengths. The audio input is
1205 + created by `input_mapper_for_phi4mm_audio`.
1206 +
1207 + Args:
1208 + audio_input (Phi4MMAudioInputs): Audio input.
1209 +
1210 + Returns:
1211 + NestedTensors: Audio embeddings
1212 + """
1213 + if audio_input["type"] == "audio_embeds":
1214 + return audio_input["data"]
1215 +
1216 + audio_features = audio_input["data"]
1217 + # (e.g. multiple examples) and the second dim is the multi-audio dim
1218 + # (e.g. multiple audios in the same example)
1219 +
1220 + dtype = next(self.audio_embed.parameters()).dtype
1221 + audio_embeds = [
1222 + self.audio_embed(
1223 + features.unsqueeze(0).to(dtype),
1224 + audio_projection_mode=audio_projection_mode,
1225 + ) for features in audio_features
1226 + ]
1227 + return audio_embeds
1228 +
1229 + def _parse_and_validate_image_input(
1230 + self, **kwargs: object) -> Optional[Phi4MMImagePixelInputs]:
1231 + image_pixel_values: NestedTensors = kwargs.get("image_pixel_values")
1232 + if image_pixel_values is None:
1233 + return None
1234 +
1235 + image_sizes = kwargs.get("image_sizes")
1236 + image_attention_mask = kwargs.get("image_attention_mask")
1237 + num_img_tokens = kwargs.get("num_img_tokens")
1238 + assert image_sizes is not None and image_attention_mask is not None\
1239 + and num_img_tokens is not None, "Missing image inputs"
1240 +
1241 + if is_list_of(image_pixel_values, torch.Tensor):
1242 + assert all(p.dim() == 5
1243 + for p in image_pixel_values), "Incorrect image inputs"
1244 + # list len is batch_size.
1245 + # each tensor has dimension: num_img_per_example, num_hd_patches,
1246 + # channels, height, width.
1247 + # need to pad along num_hd_patches.
1248 + # mask size num_img_per_prompt, num_hd_patches, feat_h, heat_w.
1249 + image_pixel_values = cat_with_pad(image_pixel_values, dim=0)
1250 + elif isinstance(image_pixel_values, torch.Tensor):
1251 + # dimension: batch_size, num_img_per_example, num_hd_patches,
1252 + # channels, height, width.
1253 + # we flatten first 2 dims to make it a single large batch for
1254 + # SigLIP Encoder.
1255 + assert image_pixel_values.dim() == 6, "Incorrect image inputs"
1256 + image_pixel_values = image_pixel_values.flatten(0, 1)
1257 + else:
1258 + raise ValueError("Incorrect image_pixel_values inputs")
1259 +
1260 + if isinstance(image_attention_mask, list):
1261 + image_attention_mask = cat_with_pad(image_attention_mask, dim=0)
1262 + elif isinstance(image_attention_mask, torch.Tensor):
1263 + image_attention_mask = image_attention_mask.flatten(0, 1)
1264 + else:
1265 + raise ValueError("Incorrect image_attention_mask inputs")
1266 +
1267 + if isinstance(image_sizes, list):
1268 + image_sizes = torch.cat(image_sizes, dim=0)
1269 + elif isinstance(image_sizes, torch.Tensor):
1270 + image_sizes = image_sizes.flatten(0, 1)
1271 + else:
1272 + raise ValueError("Incorrect image_attention_mask inputs")
1273 +
1274 + if isinstance(num_img_tokens, list):
1275 + num_img_tokens = [
1276 + n for num_tensor in num_img_tokens
1277 + for n in num_tensor.tolist()
1278 + ]
1279 + elif isinstance(num_img_tokens, torch.Tensor):
1280 + num_img_tokens = num_img_tokens.flatten(0, 1).tolist()
1281 + else:
1282 + raise ValueError("Incorrect image_attention_mask inputs")
1283 +
1284 + return Phi4MMImagePixelInputs(
1285 + type="pixel_values",
1286 + data=image_pixel_values,
1287 + image_sizes=image_sizes,
1288 + image_attention_mask=image_attention_mask,
1289 + num_img_tokens=num_img_tokens,
1290 + )
1291 +
1292 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1293 + modalities = {}
1294 +
1295 + # Preserve the order of modalities if there are multiple of them
1296 + # from the order of kwargs.
1297 + for input_key in kwargs:
1298 + if input_key in ("image_pixel_values",
1299 + "image_embeds") and "images" not in modalities:
1300 + modalities["images"] = self._parse_and_validate_image_input(
1301 + **kwargs)
1302 + if input_key in ("audio_input_features",
1303 + "audio_embeds") and "audios" not in modalities:
1304 + modalities["audios"] = self._parse_and_validate_audio_input(
1305 + **kwargs)
1306 +
1307 + return modalities
1308 +
1309 + def _process_image_input(
1310 + self, image_input: Phi4MMImagePixelInputs) -> list[torch.Tensor]:
1311 + if image_input["type"] == "image_embeds":
1312 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)
1313 + else:
1314 + dtype = next(self.image_embed.parameters()).dtype
1315 + pixel_values = image_input['data'].to(dtype)
1316 + image_sizes = image_input['image_sizes']
1317 + image_attention_mask = image_input['image_attention_mask']
1318 + image_embeds = self.image_embed(pixel_values, image_sizes,
1319 + image_attention_mask)
1320 + return image_embeds
1321 +
1322 + def get_multimodal_embeddings(
1323 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
1324 +
1325 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1326 + if not modalities:
1327 + return None
1328 +
1329 + # The result multimodal_embeddings is tuple of tensors, with each
1330 + # tensor correspoending to a multimodal data item (image or video).
1331 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1332 +
1333 + # NOTE: It is important to iterate over the keys in this dictionary
1334 + # to preserve the order of the modalities.
1335 + audio_projection_mode = 'speech'
1336 + for modality in modalities:
1337 + # make sure process images first
1338 + if modality == "images":
1339 + audio_projection_mode = "vision"
1340 + image_input = modalities["images"]
1341 + vision_embeddings = self._process_image_input(image_input)
1342 + multimodal_embeddings += tuple(vision_embeddings)
1343 + if modality == "audios":
1344 + audio_input = modalities["audios"]
1345 + audio_embeddings = self._process_audio_input(
1346 + audio_input, audio_projection_mode=audio_projection_mode)
1347 + multimodal_embeddings += tuple(audio_embeddings)
1348 +
1349 + return multimodal_embeddings
1350 +
1351 + def get_input_embeddings(
1352 + self,
1353 + input_ids: torch.Tensor,
1354 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
1355 + ) -> torch.Tensor:
1356 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1357 + if multimodal_embeddings is not None:
1358 + inputs_embeds = merge_multimodal_embeddings(
1359 + input_ids, inputs_embeds, multimodal_embeddings,
1360 + [_IMAGE_PLACEHOLDER_TOKEN_ID, _AUDIO_PLACEHOLDER_TOKEN_ID])
1361 + return inputs_embeds
1362 +
1363 + def get_input_embeddings_v0(
1364 + self,
1365 + input_ids: torch.Tensor,
1366 + image_input: Optional[Phi4MMImagePixelInputs] = None,
1367 + audio_input: Optional[Phi4MMAudioFeatureInputs] = None,
1368 + ) -> torch.Tensor:
1369 + audio_projection_mode = 'speech'
1370 + inputs_embeds = self.get_input_embeddings(input_ids)
1371 + if image_input is not None:
1372 + image_embeds = self._process_image_input(image_input)
1373 + inputs_embeds = merge_multimodal_embeddings(
1374 + input_ids,
1375 + inputs_embeds,
1376 + image_embeds,
1377 + placeholder_token_id=_IMAGE_PLACEHOLDER_TOKEN_ID,
1378 + )
1379 + audio_projection_mode = 'vision'
1380 +
1381 + if audio_input is not None:
1382 + audio_embeds = self._process_audio_input(
1383 + audio_input, audio_projection_mode=audio_projection_mode)
1384 + inputs_embeds = merge_multimodal_embeddings(
1385 + input_ids,
1386 + inputs_embeds,
1387 + audio_embeds,
1388 + placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN_ID,
1389 + )
1390 + return inputs_embeds
1391 +
1392 + def forward(
1393 + self,
1394 + input_ids: torch.Tensor,
1395 + positions: torch.Tensor,
1396 + intermediate_tensors: Optional[IntermediateTensors] = None,
1397 + inputs_embeds: Optional[torch.Tensor] = None,
1398 + **kwargs: object,
1399 + ) -> torch.Tensor:
1400 + if intermediate_tensors is not None:
1401 + inputs_embeds = None
1402 +
1403 + # NOTE: In v1, inputs_embeds is always generated at model runner from
1404 + # `get_multimodal_embeddings` and `get_input_embeddings`, this
1405 + # condition is only for v0 compatibility.
1406 + elif inputs_embeds is None:
1407 + image_input = self._parse_and_validate_image_input(**kwargs)
1408 + audio_input = self._parse_and_validate_audio_input(**kwargs)
1409 +
1410 + if image_input is None and audio_input is None:
1411 + inputs_embeds = None
1412 + else:
1413 + inputs_embeds = self.get_input_embeddings_v0(
1414 + input_ids,
1415 + image_input=image_input,
1416 + audio_input=audio_input)
1417 + input_ids = None
1418 +
1419 + hidden_states = self.language_model(
1420 + input_ids,
1421 + positions,
1422 + intermediate_tensors,
1423 + inputs_embeds=inputs_embeds,
1424 + )
1425 +
1426 + return hidden_states
1427 +
1428 + def compute_logits(
1429 + self,
1430 + hidden_states: torch.Tensor,
1431 + sampling_metadata: SamplingMetadata,
1432 + ) -> Optional[torch.Tensor]:
1433 + return self.language_model.compute_logits(hidden_states,
1434 + sampling_metadata)
1435 +
1436 + def load_weights(self, weights: Iterable[tuple[str,
1437 + torch.Tensor]]) -> set[str]:
1438 + loader = AutoWeightsLoader(self)
1439 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1440 +
1441 + def get_mm_mapping(self) -> MultiModelKeys:
1442 + """
1443 + Get the module prefix in multimodal models
1444 + """
1445 + return MultiModelKeys.from_string_field(
1446 + language_model="language_model.",
1447 + connector=[
1448 + "img_projection", "vision_speech_projection",
1449 + "speech_projection"
1450 + ],
1451 + tower_model=["image_embed", "audio_embed"],
1452 + )
1453 +
1454 + def get_language_model(self) -> torch.nn.Module:
1455 + return self.language_model
vllm/model_executor/models/registry.py
+2 -1
@@ -223,6 +223,8 @@ _MULTIMODAL_MODELS = {
223 "Ovis": ("ovis", "Ovis"),
224 "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
225 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
226 + "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
227 + "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"), # noqa: E501
228 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
229 "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501
230 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
@@ -231,7 +233,6 @@ _MULTIMODAL_MODELS = {
233 "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
234 "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
235 "UltravoxModel": ("ultravox", "UltravoxModel"),
234 - "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
236 "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501
237 "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"), # noqa: E501
238 "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"), # noqa: E501
vllm/transformers_utils/tokenizer.py
+1 -1
@@ -295,7 +295,7 @@ def cached_tokenizer_from_config(
295 return cached_get_tokenizer(
296 model_config.tokenizer,
297 tokenizer_mode=model_config.tokenizer_mode,
298 - tokenizer_revision=model_config.tokenizer_revision,
298 + revision=model_config.tokenizer_revision,
299 trust_remote_code=model_config.trust_remote_code,
300 **kwargs,
301 )