@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9998ea5b57
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 01:00 UTC

Delete HF version of Phi 4 MM (#30049)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Dec 4, 2025 at 13:44 UTC 9998ea5b576972a508c854227f57829dd4bca940
6 files changed +1 -1756
docs/models/supported_models.md
-1
@@ -711,7 +711,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
711 | `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | | ✅︎ |
712 | `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
713 | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
714 -| `Phi4MultimodalForCausalLM` | Phi-4-multimodal (HF Transformers) | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct` (with revision `refs/pr/70`), etc. | ✅︎ | ✅︎ |
714 | `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ |
715 | `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
716 | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
tests/models/multimodal/generation/test_phi4_multimodal.py deleted
-281
@@ -1,281 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import os
5 -from collections.abc import Sequence
6 -
7 -import librosa
8 -import pytest
9 -from huggingface_hub import snapshot_download
10 -
11 -from vllm.assets.image import ImageAsset
12 -from vllm.lora.request import LoRARequest
13 -from vllm.multimodal.image import rescale_image_size
14 -
15 -from ....conftest import (
16 - IMAGE_ASSETS,
17 - HfRunner,
18 - PromptAudioInput,
19 - PromptImageInput,
20 - VllmRunner,
21 -)
22 -from ....utils import large_gpu_test
23 -from ...utils import check_logprobs_close
24 -
25 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
26 - {
27 - "stop_sign": "<|user|>\n<|image|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
28 - "cherry_blossom": "<|user|>\n<|image|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
29 - }
30 -)
31 -HF_MULTIIMAGE_IMAGE_PROMPT = (
32 - "<|user|>\n<|image|>\n<|image|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
33 -)
34 -
35 -model_path = snapshot_download(
36 - "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70"
37 -)
38 -# Since the vision-lora and speech-lora co-exist with the base model,
39 -# we have to manually specify the path of the lora weights.
40 -vision_lora_path = os.path.join(model_path, "vision-lora")
41 -speech_question = os.path.join(
42 - model_path, "examples", "what_is_shown_in_this_image.wav"
43 -)
44 -models = [model_path]
45 -
46 -target_dtype = "half"
47 -
48 -
49 -def run_test(
50 - hf_runner: type[HfRunner],
51 - vllm_runner: type[VllmRunner],
52 - inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]],
53 - model: str,
54 - *,
55 - max_model_len: int,
56 - dtype: str,
57 - max_tokens: int,
58 - num_logprobs: int,
59 - mm_limit: int,
60 - tensor_parallel_size: int,
61 - distributed_executor_backend: str | None = None,
62 -):
63 - """Inference result should be the same between hf and vllm.
64 -
65 - All the image fixtures for the test are from IMAGE_ASSETS.
66 - For huggingface runner, we provide the PIL images as input.
67 - For vllm runner, we provide MultiModalDataDict objects
68 - and corresponding MultiModalConfig as input.
69 - Note, the text input is also adjusted to abide by vllm contract.
70 - The text output is sanitized to be able to compare with hf.
71 - """
72 - # NOTE: take care of the order. run vLLM first, and then run HF.
73 - # vLLM needs a fresh new process without cuda initialization.
74 - # if we run HF first, the cuda initialization will be done and it
75 - # will hurt multiprocessing backend with fork method (the default method).
76 - # max_model_len should be greater than image_feature_size
77 - with vllm_runner(
78 - model,
79 - task="generate",
80 - max_model_len=max_model_len,
81 - max_num_seqs=2,
82 - dtype=dtype,
83 - limit_mm_per_prompt={"image": mm_limit},
84 - tensor_parallel_size=tensor_parallel_size,
85 - distributed_executor_backend=distributed_executor_backend,
86 - enable_lora=True,
87 - max_lora_rank=320,
88 - gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
89 - enforce_eager=True,
90 - trust_remote_code=False,
91 - ) as vllm_model:
92 - lora_request = LoRARequest("vision", 1, vision_lora_path)
93 - vllm_outputs_per_case = [
94 - vllm_model.generate_greedy_logprobs(
95 - prompts,
96 - max_tokens,
97 - num_logprobs=num_logprobs,
98 - images=images,
99 - audios=audios,
100 - lora_request=lora_request,
101 - )
102 - for prompts, images, audios in inputs
103 - ]
104 -
105 - with hf_runner(model, dtype=dtype) as hf_model:
106 - hf_model.model.load_adapter(
107 - vision_lora_path,
108 - adapter_name="vision",
109 - )
110 - hf_processor = hf_model.processor
111 - eos_token_id = hf_processor.tokenizer.eos_token_id
112 - hf_outputs_per_case = [
113 - hf_model.generate_greedy_logprobs_limit(
114 - prompts,
115 - max_tokens,
116 - num_logprobs=num_logprobs,
117 - images=images,
118 - audios=audios,
119 - eos_token_id=eos_token_id,
120 - )
121 - for prompts, images, audios in inputs
122 - ]
123 -
124 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
125 - check_logprobs_close(
126 - outputs_0_lst=hf_outputs,
127 - outputs_1_lst=vllm_outputs,
128 - name_0="hf",
129 - name_1="vllm",
130 - )
131 -
132 -
133 -@pytest.mark.parametrize("model", models)
134 -@pytest.mark.parametrize(
135 - "size_factors",
136 - [
137 - # No image
138 - [],
139 - # Single-scale
140 - [1.0],
141 - # Single-scale, batched
142 - [1.0, 1.0, 1.0],
143 - # Multi-scale
144 - [0.25, 0.5, 1.0],
145 - ],
146 -)
147 -@pytest.mark.parametrize("dtype", [target_dtype])
148 -@pytest.mark.parametrize("max_model_len", [12800])
149 -@pytest.mark.parametrize("max_tokens", [128])
150 -@pytest.mark.parametrize("num_logprobs", [10])
151 -def test_models(
152 - hf_runner,
153 - vllm_runner,
154 - image_assets,
155 - model,
156 - size_factors,
157 - dtype: str,
158 - max_model_len: int,
159 - max_tokens: int,
160 - num_logprobs: int,
161 -) -> None:
162 - images = [asset.pil_image for asset in image_assets]
163 -
164 - inputs_per_image = [
165 - (
166 - [prompt for _ in size_factors],
167 - [rescale_image_size(image, factor) for factor in size_factors],
168 - None,
169 - )
170 - for image, prompt in zip(images, HF_IMAGE_PROMPTS)
171 - ]
172 -
173 - run_test(
174 - hf_runner,
175 - vllm_runner,
176 - inputs_per_image,
177 - model,
178 - dtype=dtype,
179 - max_model_len=max_model_len,
180 - max_tokens=max_tokens,
181 - num_logprobs=num_logprobs,
182 - mm_limit=1,
183 - tensor_parallel_size=1,
184 - )
185 -
186 -
187 -@large_gpu_test(min_gb=48)
188 -@pytest.mark.parametrize("model", models)
189 -@pytest.mark.parametrize(
190 - "size_factors",
191 - [
192 - # No image
193 - # [],
194 - # Single-scale
195 - [1.0],
196 - # Single-scale, batched
197 - [1.0, 1.0, 1.0],
198 - # Multi-scale
199 - [0.25, 0.5, 1.0],
200 - ],
201 -)
202 -@pytest.mark.parametrize("dtype", [target_dtype])
203 -@pytest.mark.parametrize("max_model_len", [25600])
204 -@pytest.mark.parametrize("max_tokens", [128])
205 -@pytest.mark.parametrize("num_logprobs", [10])
206 -def test_multi_images_models(
207 - hf_runner,
208 - vllm_runner,
209 - image_assets,
210 - model,
211 - size_factors,
212 - dtype: str,
213 - max_model_len: int,
214 - max_tokens: int,
215 - num_logprobs: int,
216 -) -> None:
217 - images = [asset.pil_image for asset in image_assets]
218 -
219 - inputs_per_case = [
220 - (
221 - [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
222 - [
223 - [rescale_image_size(image, factor) for image in images]
224 - for factor in size_factors
225 - ],
226 - None,
227 - ),
228 - ]
229 -
230 - run_test(
231 - hf_runner,
232 - vllm_runner,
233 - inputs_per_case,
234 - model,
235 - dtype=dtype,
236 - max_model_len=max_model_len,
237 - max_tokens=max_tokens,
238 - num_logprobs=num_logprobs,
239 - mm_limit=2,
240 - tensor_parallel_size=1,
241 - )
242 -
243 -
244 -@pytest.mark.parametrize("model", models)
245 -@pytest.mark.parametrize("dtype", [target_dtype])
246 -@pytest.mark.parametrize("max_model_len", [12800])
247 -@pytest.mark.parametrize("max_tokens", [128])
248 -@pytest.mark.parametrize("num_logprobs", [10])
249 -def test_vision_speech_models(
250 - hf_runner,
251 - vllm_runner,
252 - model,
253 - dtype: str,
254 - max_model_len: int,
255 - max_tokens: int,
256 - num_logprobs: int,
257 -) -> None:
258 - # use the example speech question so that the model outputs are reasonable
259 - audio = librosa.load(speech_question, sr=16000)
260 - image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
261 -
262 - inputs_vision_speech = [
263 - (
264 - ["<|user|><|image|><|audio|><|end|><|assistant|>"],
265 - [image],
266 - [audio],
267 - ),
268 - ]
269 -
270 - run_test(
271 - hf_runner,
272 - vllm_runner,
273 - inputs_vision_speech,
274 - model,
275 - dtype=dtype,
276 - max_model_len=max_model_len,
277 - max_tokens=max_tokens,
278 - num_logprobs=num_logprobs,
279 - mm_limit=1,
280 - tensor_parallel_size=1,
281 - )
tests/models/multimodal/processing/test_common.py
-22
@@ -396,28 +396,6 @@ def test_processing_correctness(
396 )
397
398
399 -# Phi4MultimodalForCausalLM share same model repo with original format
400 -# Phi4MMForCausalLM, so we add it as a separate test case
401 -# Remove this test after conversion PR merged:
402 -# https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/70
403 -@pytest.mark.parametrize("model_arch", ["Phi4MultimodalForCausalLM"])
404 -@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
405 -@pytest.mark.parametrize("num_batches", [32])
406 -@pytest.mark.parametrize("simplify_rate", [1.0])
407 -def test_processing_correctness_phi4_multimodal(
408 - model_arch: str,
409 - hit_rate: float,
410 - num_batches: int,
411 - simplify_rate: float,
412 -):
413 - _test_processing_correctness(
414 - model_arch,
415 - hit_rate=hit_rate,
416 - num_batches=num_batches,
417 - simplify_rate=simplify_rate,
418 - )
419 -
420 -
399 def _assert_inputs_equal(
400 a: MultiModalInputs,
401 b: MultiModalInputs,
tests/models/registry.py
-4
@@ -771,10 +771,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
771 "Phi4MMForCausalLM": _HfExamplesInfo(
772 "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
773 ),
774 - "Phi4MultimodalForCausalLM": _HfExamplesInfo(
775 - "microsoft/Phi-4-multimodal-instruct",
776 - revision="refs/pr/70",
777 - ),
774 "PixtralForConditionalGeneration": _HfExamplesInfo(
775 "mistralai/Pixtral-12B-2409",
776 extras={
vllm/model_executor/models/phi4_multimodal.py deleted
-1447
@@ -1,1447 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import math
4 -from collections.abc import Iterable, Mapping, Sequence
5 -from typing import Annotated, Any, Literal, TypeAlias
6 -
7 -import numpy as np
8 -import torch
9 -import torch.nn as nn
10 -import torch.nn.functional as F
11 -from transformers import (
12 - BatchFeature,
13 - Phi4MultimodalAudioConfig,
14 - Phi4MultimodalConfig,
15 - Phi4MultimodalFeatureExtractor,
16 - Phi4MultimodalImageProcessorFast,
17 -)
18 -from transformers import Phi4MultimodalProcessor as Phi4MMProcessor
19 -from transformers.models.phi4_multimodal.modeling_phi4_multimodal import (
20 - Phi4MultimodalAudioConvModule,
21 - Phi4MultimodalAudioNemoConvSubsampling,
22 - Phi4MultimodalAudioRelativeAttentionBias,
23 - adaptive_enc_mask,
24 - unfold_tensor,
25 -)
26 -
27 -from vllm.config import VllmConfig
28 -from vllm.config.multimodal import BaseDummyOptions
29 -from vllm.distributed import (
30 - divide,
31 - get_tensor_model_parallel_rank,
32 - get_tensor_model_parallel_world_size,
33 -)
34 -from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
35 -from vllm.model_executor.layers.linear import (
36 - ColumnParallelLinear,
37 - MergedColumnParallelLinear,
38 - QKVParallelLinear,
39 - RowParallelLinear,
40 -)
41 -from vllm.model_executor.layers.quantization import QuantizationConfig
42 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 -from vllm.model_executor.models.module_mapping import MultiModelKeys
44 -from vllm.multimodal import MULTIMODAL_REGISTRY
45 -from vllm.multimodal.inputs import (
46 - MultiModalDataDict,
47 - MultiModalFieldConfig,
48 - MultiModalKwargsItems,
49 - NestedTensors,
50 -)
51 -from vllm.multimodal.parse import (
52 - AudioProcessorItems,
53 - ImageEmbeddingItems,
54 - ImageProcessorItems,
55 - ImageSize,
56 - MultiModalDataItems,
57 - MultiModalDataParser,
58 -)
59 -from vllm.multimodal.processing import (
60 - BaseMultiModalProcessor,
61 - BaseProcessingInfo,
62 - PromptReplacement,
63 - PromptUpdate,
64 -)
65 -from vllm.multimodal.profiling import BaseDummyInputsBuilder
66 -from vllm.sequence import IntermediateTensors
67 -from vllm.utils.tensor_schema import TensorSchema, TensorShape
68 -
69 -from .idefics2_vision_model import Idefics2VisionTransformer
70 -from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
71 -from .utils import (
72 - AutoWeightsLoader,
73 - WeightsMapper,
74 - init_vllm_registered_model,
75 - maybe_prefix,
76 -)
77 -
78 -_AUDIO_MAX_SOUNDFILE_SIZE = 241_000
79 -
80 -
81 -def _get_padding_size(
82 - orig_width: int, orig_height: int, target_height: int, target_width: int
83 -):
84 - ratio_width = target_width / orig_width
85 - ratio_height = target_height / orig_height
86 -
87 - if ratio_width < ratio_height:
88 - padding_width = 0
89 - padding_height = target_height - int(orig_height * ratio_width)
90 - else:
91 - padding_width = target_width - int(orig_width * ratio_height)
92 - padding_height = 0
93 - return padding_height, padding_width
94 -
95 -
96 -class Phi4MMProjector(nn.Module):
97 - def __init__(self, input_size: int, hidden_size: int):
98 - super().__init__()
99 - self.up = ColumnParallelLinear(input_size, hidden_size)
100 - self.down = RowParallelLinear(hidden_size, hidden_size)
101 - self.act = get_act_fn("gelu")
102 -
103 - def forward(self, x: torch.Tensor) -> torch.Tensor:
104 - x, _ = self.up(x)
105 - x = self.act(x)
106 - x, _ = self.down(x)
107 - return x
108 -
109 -
110 -class Phi4MMImageEmbedding(nn.Module):
111 - """Image embedding."""
112 -
113 - def __init__(self, config: Phi4MultimodalConfig):
114 - super().__init__()
115 - self.config = config
116 - self.layer_idx = config.vision_config.feature_layer
117 - self.crop_size = config.vision_config.crop_size
118 - self.image_dim_out = config.vision_config.hidden_size
119 -
120 - n_patches = config.vision_config.image_size // config.vision_config.patch_size
121 - if n_patches % 2 != 0:
122 - self.img_processor_padding = nn.ReflectionPad2d((0, 1, 0, 1))
123 - n_patches += 1
124 - self.num_img_tokens = (n_patches // 2) ** 2
125 -
126 - num_hidden_layers = (
127 - config.vision_config.num_hidden_layers + self.layer_idx + 1
128 - if self.layer_idx < 0
129 - else self.layer_idx + 1
130 - )
131 - self.img_processor = Idefics2VisionTransformer(
132 - config.vision_config,
133 - require_post_norm=False,
134 - num_hidden_layers_override=num_hidden_layers,
135 - )
136 - self.image_token_compression = nn.AvgPool2d(kernel_size=2, stride=2)
137 - self.img_projection = Phi4MMProjector(self.image_dim_out, config.hidden_size)
138 - self.global_img_feature_extensor = nn.Parameter(
139 - torch.zeros([1, 1, self.image_dim_out])
140 - )
141 - self.sub_img_feature_extensor = nn.Parameter(
142 - torch.zeros([1, 1, 1, self.image_dim_out])
143 - )
144 -
145 - def get_img_features(
146 - self,
147 - img_embeds: torch.FloatTensor,
148 - attention_mask: torch.Tensor | None = None,
149 - ) -> torch.FloatTensor:
150 - img_feature = self.img_processor(
151 - img_embeds, patch_attention_mask=attention_mask
152 - )
153 -
154 - patch_feature = img_feature
155 - # reshape to 2D tensor
156 - width = int(math.sqrt(patch_feature.size(1)))
157 - patch_feature = patch_feature.view(-1, width, width, patch_feature.size(-1))
158 - # convert to NCHW
159 - patch_feature = patch_feature.permute(0, 3, 1, 2)
160 - if getattr(self, "img_processor_padding", None) is not None:
161 - patch_feature = self.img_processor_padding(patch_feature)
162 - patch_feature = self.image_token_compression(patch_feature)
163 - # convert to NHWC
164 - patch_feature = patch_feature.permute(0, 2, 3, 1)
165 - patch_feature = patch_feature.view(
166 - -1, patch_feature.size(1) * patch_feature.size(2), patch_feature.size(-1)
167 - )
168 - return patch_feature
169 -
170 - def forward(
171 - self,
172 - image_pixel_values: torch.FloatTensor,
173 - image_sizes: torch.Tensor | None = None,
174 - image_attention_mask: torch.Tensor | None = None,
175 - ) -> torch.FloatTensor:
176 - image_pixel_values = image_pixel_values.to(
177 - self.img_processor.embeddings.patch_embedding.weight.dtype
178 - )
179 -
180 - target_device = self.img_projection.up.bias.device
181 - target_dtype = self.img_projection.up.bias.dtype
182 -
183 - batch_size = image_pixel_values.shape[0]
184 -
185 - img_features = self.get_img_features(
186 - image_pixel_values.flatten(0, 1),
187 - attention_mask=image_attention_mask.flatten(0, 1).to(
188 - dtype=bool, device=target_device
189 - ),
190 - )
191 - base_feat_size = int(np.sqrt(img_features.shape[1]))
192 - img_features = img_features.view(
193 - batch_size, -1, base_feat_size**2, self.image_dim_out
194 - )
195 - image_sizes = image_sizes.view(-1, 2)
196 -
197 - output_imgs = []
198 - for idx in range(batch_size):
199 - height, width = image_sizes[idx]
200 - height_ratio = height // self.crop_size
201 - width_ratio = width // self.crop_size
202 - area_ratio = height_ratio * width_ratio
203 -
204 - global_img = img_features[idx, :1]
205 - global_img = global_img.reshape(
206 - 1, base_feat_size, base_feat_size, self.image_dim_out
207 - ).contiguous()
208 - temporary_extensor = self.sub_img_feature_extensor.repeat(
209 - 1, base_feat_size, 1, 1
210 - )
211 - global_img = torch.cat([global_img, temporary_extensor], dim=2).reshape(
212 - 1, -1, self.image_dim_out
213 - )
214 -
215 - sub_img = img_features[idx, 1:]
216 - sub_img = sub_img[:area_ratio]
217 - sub_img = (
218 - sub_img.reshape(
219 - height_ratio,
220 - width_ratio,
221 - base_feat_size,
222 - base_feat_size,
223 - self.image_dim_out,
224 - )
225 - .transpose(1, 2)
226 - .reshape(
227 - 1,
228 - height_ratio * base_feat_size,
229 - width_ratio * base_feat_size,
230 - self.image_dim_out,
231 - )
232 - .contiguous()
233 - )
234 -
235 - if image_attention_mask is not None:
236 - reshaped_image_attention_mask = (
237 - image_attention_mask[idx, 1 : area_ratio + 1, 0::2, 0::2]
238 - .reshape(height_ratio, width_ratio, base_feat_size, base_feat_size)
239 - .transpose(1, 2)
240 - .reshape(
241 - 1, height_ratio * base_feat_size, width_ratio * base_feat_size
242 - )
243 - )
244 - useful_height = int(reshaped_image_attention_mask[0, :, 0].sum().item())
245 - useful_width = int(reshaped_image_attention_mask[0, 0, :].sum().item())
246 - sub_img = sub_img[:, :useful_height, :useful_width]
247 - temporary_extensor = self.sub_img_feature_extensor.repeat(
248 - 1, useful_height, 1, 1
249 - )
250 - else:
251 - temporary_extensor = self.sub_img_feature_extensor.repeat(
252 - 1, height_ratio * base_feat_size, 1, 1
253 - )
254 -
255 - sub_img = torch.cat([sub_img, temporary_extensor], dim=2).reshape(
256 - 1, -1, self.image_dim_out
257 - )
258 -
259 - # Merge global and sub
260 - output_imgs.append(
261 - torch.cat(
262 - [sub_img, self.global_img_feature_extensor, global_img], dim=1
263 - )
264 - )
265 -
266 - img_set_tensor = []
267 - for output_img in output_imgs:
268 - output_img = output_img.to(device=target_device, dtype=target_dtype)
269 - img_feature_proj = self.img_projection(output_img)
270 - img_set_tensor.append(img_feature_proj.flatten(0, 1))
271 -
272 - return img_set_tensor
273 -
274 -
275 -class Phi4MultimodalAudioMLP(nn.Module):
276 - def __init__(
277 - self,
278 - config: Phi4MultimodalAudioConfig,
279 - quant_config: QuantizationConfig | None = None,
280 - prefix: str = "",
281 - ):
282 - super().__init__()
283 - self.layer_norm = nn.LayerNorm(config.hidden_size)
284 - self.act_fn = MulAndSilu()
285 - self.gate_up_proj = MergedColumnParallelLinear(
286 - config.hidden_size,
287 - [config.intermediate_size] * 2,
288 - bias=True,
289 - quant_config=quant_config,
290 - prefix=f"{prefix}.gate_up_proj",
291 - )
292 - self.down_proj = RowParallelLinear(
293 - config.intermediate_size,
294 - config.hidden_size,
295 - bias=True,
296 - quant_config=quant_config,
297 - prefix=f"{prefix}.down_proj",
298 - )
299 -
300 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
301 - hidden_states = self.layer_norm(hidden_states)
302 - hidden_states, _ = self.gate_up_proj(hidden_states)
303 - hidden_states = self.act_fn(hidden_states)
304 - hidden_states, _ = self.down_proj(hidden_states)
305 - return hidden_states
306 -
307 -
308 -class Phi4MultimodalAudioAttention(nn.Module):
309 - def __init__(
310 - self,
311 - config: Phi4MultimodalAudioConfig,
312 - quant_config: QuantizationConfig | None = None,
313 - prefix: str = "",
314 - ):
315 - super().__init__()
316 - self.config = config
317 - self.embed_dim = config.hidden_size
318 - self.total_num_heads = config.num_attention_heads
319 - self.head_dim = self.embed_dim // self.total_num_heads
320 - if self.head_dim * self.total_num_heads != self.embed_dim:
321 - raise ValueError(
322 - "embed_dim must be divisible by num_heads "
323 - f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
324 - f" {self.num_heads})."
325 - )
326 - self.scale = self.head_dim**-0.5
327 -
328 - self.qkv_proj = QKVParallelLinear(
329 - hidden_size=self.embed_dim,
330 - head_size=self.head_dim,
331 - total_num_heads=self.total_num_heads,
332 - quant_config=quant_config,
333 - prefix=f"{prefix}.qkv_proj",
334 - )
335 -
336 - self.o_proj = RowParallelLinear(
337 - input_size=self.embed_dim,
338 - output_size=self.embed_dim,
339 - quant_config=quant_config,
340 - prefix=f"{prefix}.out_proj",
341 - )
342 -
343 - self.tp_size = get_tensor_model_parallel_world_size()
344 - self.tp_rank = get_tensor_model_parallel_rank()
345 - self.num_heads = divide(self.total_num_heads, self.tp_size)
346 -
347 - def split_attn_mask(self, attention_mask: torch.Tensor) -> torch.Tensor:
348 - start_idx = self.num_heads * self.tp_rank
349 - end_idx = self.num_heads * (self.tp_rank + 1)
350 - return attention_mask[:, start_idx:end_idx]
351 -
352 - def forward(
353 - self,
354 - hidden_states: torch.Tensor,
355 - attention_mask: torch.Tensor,
356 - ) -> torch.Tensor:
357 - qkv_states, _ = self.qkv_proj(hidden_states)
358 - query, key, value = qkv_states.chunk(3, dim=-1)
359 -
360 - bsz, seq_len, _ = query.size()
361 - query = query.view(bsz, seq_len, self.num_heads, self.head_dim)
362 - key = key.view(bsz, seq_len, self.num_heads, self.head_dim)
363 - value = value.view(bsz, seq_len, self.num_heads, self.head_dim)
364 - query, key, value = (x.transpose(1, 2) for x in (query, key, value))
365 -
366 - attention_mask = self.split_attn_mask(attention_mask)
367 - out = F.scaled_dot_product_attention(
368 - query,
369 - key,
370 - value,
371 - scale=self.scale,
372 - attn_mask=attention_mask,
373 - )
374 - out = out.transpose(1, 2).reshape(bsz, seq_len, -1)
375 -
376 - attn_output, _ = self.o_proj(out)
377 -
378 - return attn_output
379 -
380 -
381 -class Phi4MultimodalAudioConformerEncoderLayer(nn.Module):
382 - def __init__(self, config: Phi4MultimodalAudioConfig):
383 - super().__init__()
384 -
385 - self.feed_forward_in = Phi4MultimodalAudioMLP(config)
386 - self.self_attn = Phi4MultimodalAudioAttention(config)
387 - self.conv = Phi4MultimodalAudioConvModule(config)
388 - self.feed_forward_out = Phi4MultimodalAudioMLP(config)
389 - self.layer_norm_att = nn.LayerNorm(config.hidden_size)
390 - self.layer_norm = nn.LayerNorm(config.hidden_size)
391 -
392 - def forward(
393 - self,
394 - hidden_states: torch.Tensor,
395 - attention_mask: torch.Tensor,
396 - ) -> torch.Tensor:
397 - residual = hidden_states + 0.5 * self.feed_forward_in(hidden_states)
398 - hidden_states = self.layer_norm_att(residual)
399 -
400 - hidden_states = residual + self.self_attn(hidden_states, attention_mask)
401 - hidden_states = hidden_states + self.conv(hidden_states)
402 - hidden_states = hidden_states + 0.5 * self.feed_forward_out(hidden_states)
403 -
404 - out = self.layer_norm(hidden_states)
405 -
406 - return out
407 -
408 -
409 -class Phi4MMAudioMeanVarianceNormLayer(nn.Module):
410 - """Mean/variance normalization layer.
411 -
412 - Will subtract mean and multiply input by inverted standard deviation.
413 - Typically used as a very first layer in a model.
414 -
415 - Args:
416 - config: [Phi4MultimodalAudioConfig](https://huggingface.co/docs/transformers/model_doc/phi4_multimodal#transformers.Phi4MultimodalAudioConfig)
417 - object containing model parameters.
418 - """
419 -
420 - def __init__(self, config: Phi4MultimodalAudioConfig):
421 - super().__init__()
422 - self.global_mean = nn.Parameter(torch.zeros(config.input_size))
423 - self.global_invstd = nn.Parameter(torch.ones(config.input_size))
424 -
425 - def forward(self, input_: torch.Tensor) -> torch.Tensor:
426 - """MeanVarianceNormLayer Forward
427 -
428 - Args:
429 - input_: torch.Tensor
430 - input tensor.
431 - """
432 - return (input_ - self.global_mean) * self.global_invstd
433 -
434 -
435 -class Phi4MultimodalAudioModel(nn.Module):
436 - def __init__(self, config: Phi4MultimodalAudioConfig):
437 - super().__init__()
438 - self.config = config
439 -
440 - self.encoder_embedding = Phi4MMAudioMeanVarianceNormLayer(config)
441 - self.embed = Phi4MultimodalAudioNemoConvSubsampling(config)
442 - self.relative_attention_bias_layer = Phi4MultimodalAudioRelativeAttentionBias(
443 - config
444 - )
445 - self.encoders = nn.ModuleList(
446 - [
447 - Phi4MultimodalAudioConformerEncoderLayer(config)
448 - for _ in range(config.num_blocks)
449 - ]
450 - )
451 -
452 - def _streaming_mask(
453 - self,
454 - seq_len: int,
455 - batch_size: int,
456 - chunk_size: int,
457 - left_chunk: int,
458 - ):
459 - # Create mask matrix for streaming
460 - # S stores start index. if chunksize is 18, s is [0,18,36,....]
461 - chunk_start_idx = np.arange(0, seq_len, chunk_size)
462 -
463 - enc_streaming_mask = (
464 - adaptive_enc_mask(seq_len, chunk_start_idx, left_window=left_chunk)
465 - .unsqueeze(0)
466 - .expand([batch_size, -1, -1])
467 - )
468 - return enc_streaming_mask
469 -
470 - def forward_embeddings(
471 - self,
472 - hidden_states: torch.Tensor,
473 - masks: torch.Tensor,
474 - ):
475 - """Forwarding the inputs through the top embedding layers"""
476 - seq_len = math.ceil(hidden_states.shape[1] / self.config.time_reduction)
477 - if seq_len <= 0:
478 - raise ValueError(
479 - f"Sequence length after time reduction is invalid: {seq_len}."
480 - "Your input feature is too short."
481 - )
482 -
483 - batch_size = hidden_states.shape[0]
484 -
485 - enc_streaming_mask = self._streaming_mask(
486 - seq_len, batch_size, self.config.chunk_size, self.config.left_chunk
487 - )
488 - enc_streaming_mask = enc_streaming_mask.to(hidden_states.device)
489 -
490 - hidden_states, masks = self.embed(hidden_states, masks)
491 -
492 - streaming_mask = enc_streaming_mask
493 - if streaming_mask is not None and masks is not None:
494 - hs_mask = masks & streaming_mask
495 - elif masks is not None:
496 - hs_mask = masks
497 - else:
498 - hs_mask = streaming_mask
499 -
500 - return hidden_states, hs_mask, masks
501 -
502 - def calculate_hs_mask(
503 - self, hidden_states: torch.Tensor, device: torch.device, mask: torch.Tensor
504 - ):
505 - max_audio_length = hidden_states.shape[1]
506 - batch_size = hidden_states.shape[0]
507 - enc_streaming_mask = self._streaming_mask(
508 - max_audio_length, batch_size, self.config.chunk_size, self.config.left_chunk
509 - )
510 - enc_streaming_mask = enc_streaming_mask.to(device)
511 - if mask is None:
512 - return enc_streaming_mask
513 -
514 - feature_lens = mask.sum(1)
515 - padding_length = feature_lens
516 - pad_mask = torch.arange(0, max_audio_length, device=device).expand(
517 - padding_length.size(0), -1
518 - ) < padding_length.unsqueeze(1)
519 - pad_mask = pad_mask.unsqueeze(1)
520 - pad_mask = pad_mask & enc_streaming_mask
521 - return pad_mask
522 -
523 - def forward(self, hidden_states: torch.Tensor, mask: torch.Tensor | None = None):
524 - hidden_states = self.encoder_embedding(hidden_states)
525 - hidden_states, hs_mask, mask = self.forward_embeddings(hidden_states, mask)
526 -
527 - unfolded = False
528 - bs, seq_len, _ = hidden_states.shape
529 - max_seq_len = 500 # maximum position for absolute positional encoding
530 - if seq_len > max_seq_len:
531 - # audio sequence is longer than max_seq_len,
532 - # unfold it into chunks of max_seq_len
533 - unfolded = True
534 - # the unfold op will drop residual frames,
535 - # pad it to the multiple of max_seq_len
536 - if seq_len % max_seq_len > 0:
537 - chunk_pad_size = max_seq_len - (seq_len % max_seq_len)
538 - else:
539 - chunk_pad_size = 0
540 - if chunk_pad_size > 0:
541 - hidden_states_pad = F.pad(
542 - hidden_states, (0, 0, 0, chunk_pad_size), "constant", 0
543 - )
544 - hidden_states = hidden_states_pad.to(hidden_states.device)
545 -
546 - hidden_states = unfold_tensor(hidden_states, max_seq_len)
547 - masks_unfold = None
548 - if mask is not None:
549 - # revise hs_mask here because the previous calculated hs_mask
550 - # did not consider extra pad
551 - subsampled_pad_mask = mask.squeeze(1) # [bz, subsampled_unmask_seq_len]
552 - extra_padded_subsamlped_pad_mask = F.pad(
553 - subsampled_pad_mask, (0, chunk_pad_size), "constant", False
554 - ) # extra padding to the pad mask
555 - extra_padded_subsamlped_pad_mask = (
556 - extra_padded_subsamlped_pad_mask.unsqueeze(-1).float()
557 - )
558 - masks_unfold = unfold_tensor(
559 - extra_padded_subsamlped_pad_mask, max_seq_len
560 - ) # unfold the pad mask like we did to the input tensor
561 - masks_unfold = masks_unfold.squeeze(
562 - -1
563 - ).bool() # unfold op does not support bool tensor
564 - hs_mask = self.calculate_hs_mask(
565 - hidden_states, hidden_states.device, masks_unfold
566 - ) # calculate hs_mask based on the unfolded pad mask
567 -
568 - relative_attention_bias = self.relative_attention_bias_layer(hidden_states)
569 - attention_mask = hs_mask.unsqueeze(1) + relative_attention_bias
570 -
571 - for layer in self.encoders:
572 - hidden_states = layer(hidden_states, attention_mask)
573 -
574 - if unfolded:
575 - embed_dim = hidden_states.shape[-1]
576 - hidden_states = hidden_states.reshape(bs, -1, embed_dim)
577 - # if we ever padded before unfolding, we need to remove the padding
578 - if chunk_pad_size > 0:
579 - hidden_states = hidden_states[:, :-chunk_pad_size, :]
580 -
581 - return hidden_states
582 -
583 -
584 -class Phi4MMAudioEmbedding(nn.Module):
585 - def __init__(self, config: Phi4MultimodalConfig):
586 - super().__init__()
587 - self.config = config
588 - self.layer_idx = config.audio_config.feature_layer
589 -
590 - self.encoder = Phi4MultimodalAudioModel(config.audio_config)
591 -
592 - audio_config = config.audio_config
593 - proj_input_size = audio_config.hidden_size * audio_config.downsample_rate
594 - self.vision_speech_projection = Phi4MMProjector(
595 - proj_input_size, config.hidden_size
596 - )
597 - self.speech_projection = Phi4MMProjector(proj_input_size, config.hidden_size)
598 -
599 - def get_projection(
600 - self,
601 - audio_projection_mode: Literal["speech", "vision"],
602 - ) -> Phi4MMProjector:
603 - if audio_projection_mode == "speech":
604 - return self.speech_projection
605 - elif audio_projection_mode == "vision":
606 - return self.vision_speech_projection
607 -
608 - def forward(
609 - self,
610 - audio_input_features: torch.FloatTensor,
611 - audio_embed_sizes=None,
612 - audio_attention_mask=None,
613 - audio_projection_mode="speech",
614 - ) -> torch.FloatTensor:
615 - audio_projection = self.get_projection(audio_projection_mode)
616 -
617 - target_device = audio_projection.up.bias.device
618 - target_dtype = audio_projection.up.bias.dtype
619 -
620 - audio_input_features = audio_input_features.to(
621 - device=target_device, dtype=target_dtype
622 - )
623 -
624 - audio_encoder_hidden_states = self.encoder(
625 - audio_input_features, audio_attention_mask
626 - )
627 - audio_embeds = audio_projection(audio_encoder_hidden_states)
628 -
629 - return audio_embeds.flatten(0, 1)
630 -
631 - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
632 - stacked_params_mapping = [
633 - # (param_name, shard_name, shard_id)
634 - ("qkv_proj", "q_proj", "q"),
635 - ("qkv_proj", "k_proj", "k"),
636 - ("qkv_proj", "v_proj", "v"),
637 - ]
638 - params_dict = dict(self.named_parameters())
639 - loaded_params: set[str] = set()
640 -
641 - for name, loaded_weight in weights:
642 - for param_name, weight_name, shard_id in stacked_params_mapping:
643 - if weight_name not in name:
644 - continue
645 - name = name.replace(weight_name, param_name)
646 - param = params_dict[name]
647 - weight_loader = param.weight_loader
648 - weight_loader(param, loaded_weight, shard_id)
649 - break
650 - else:
651 - param = params_dict[name]
652 - weight_loader = getattr(param, "weight_loader", default_weight_loader)
653 - weight_loader(param, loaded_weight)
654 - loaded_params.add(name)
655 - return loaded_params
656 -
657 -
658 -class Phi4MMImagePixelInputs(TensorSchema):
659 - """
660 - Dimensions:
661 - - bn: Batch size * number of images
662 - - p: Number of patches (1 + num_patches)
663 - - c: Number of channels (3)
664 - - h: Height of each image patch
665 - - w: Width of each image patch
666 - - nc: Number of crops
667 - - H_mask: Height of attention mask
668 - - W_mask: Width of attention mask
669 - """
670 -
671 - type: Literal["pixel_values"]
672 -
673 - pixel_values: Annotated[
674 - torch.Tensor | list[torch.Tensor],
675 - TensorShape(
676 - "bn", "p", 3, "h", "w", dynamic_dims={"p"}
677 - ), # may be different per batch and image
678 - ]
679 -
680 - image_sizes: Annotated[
681 - torch.Tensor,
682 - TensorShape("bn", 2), # (height, width)
683 - ]
684 -
685 - num_img_tokens: Annotated[
686 - list[int],
687 - TensorShape("bn"),
688 - ]
689 -
690 - image_attention_mask: Annotated[
691 - torch.Tensor,
692 - TensorShape("bn", "nc", 32, 32), # H_mask, W_mask
693 - ]
694 -
695 -
696 -class Phi4MMImageEmbeddingInputs(TensorSchema):
697 - """
698 - Dimensions:
699 - - bn: Batch size * number of images
700 - - f: Image feature size
701 - - h: Hidden size (must match language model backbone)
702 - """
703 -
704 - type: Literal["image_embeds"]
705 -
706 - data: Annotated[
707 - torch.Tensor | list[torch.Tensor],
708 - TensorShape("bn", "f", "h"),
709 - ]
710 -
711 -
712 -class Phi4MMAudioFeatureInputs(TensorSchema):
713 - """
714 - Dimensions:
715 - - bn: Batch size * number of audios
716 - - f: Number of Mel filterbank bins (80)
717 - - t: Time frames (M)
718 - """
719 -
720 - type: Literal["audio_features"]
721 -
722 - audio_features: Annotated[
723 - torch.Tensor | list[torch.Tensor],
724 - TensorShape("bn", "t", 80, dynamic_dims={"t"}),
725 - ]
726 -
727 -
728 -class Phi4MMAudioEmbeddingInputs(TensorSchema):
729 - """
730 - Dimensions:
731 - - b: Batch size
732 - - n: Number of audios
733 - - f: Audio feature size
734 - - h: Hidden size (must match language model backbone)
735 - """
736 -
737 - type: Literal["audio_embeds"]
738 -
739 - data: Annotated[
740 - NestedTensors,
741 - TensorShape("b", "n", "f", "h"),
742 - ]
743 -
744 -
745 -Phi4MMImageInput: TypeAlias = Phi4MMImagePixelInputs | Phi4MMImageEmbeddingInputs
746 -Phi4MMAudioInputs: TypeAlias = Phi4MMAudioFeatureInputs | Phi4MMAudioEmbeddingInputs
747 -
748 -
749 -def cat_with_pad(tensors, dim, padding_value=0):
750 - """
751 - cat along dim, while pad to max for all other dims
752 - """
753 - ndim = tensors[0].dim()
754 - assert all(t.dim() == ndim for t in tensors[1:]), (
755 - "All tensors must have the same number of dimensions"
756 - )
757 -
758 - out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)]
759 - out_size[dim] = sum(t.shape[dim] for t in tensors)
760 - output = tensors[0].new_full(out_size, padding_value)
761 -
762 - index = 0
763 - for t in tensors:
764 - # Create a slice list where every dimension except dim is full slice
765 - slices = [slice(0, t.shape[d]) for d in range(ndim)]
766 - # Update only the concat dimension slice
767 - slices[dim] = slice(index, index + t.shape[dim])
768 -
769 - output[slices] = t
770 - index += t.shape[dim]
771 -
772 - return output
773 -
774 -
775 -class Phi4MMProcessingInfo(BaseProcessingInfo):
776 - def get_hf_config(self) -> Phi4MultimodalConfig:
777 - return self.ctx.get_hf_config(Phi4MultimodalConfig)
778 -
779 - def get_hf_processor(self, **kwargs: object) -> Phi4MMProcessor:
780 - return self.ctx.get_hf_processor(Phi4MMProcessor, **kwargs)
781 -
782 - def get_feature_extractor(self, **kwargs: object) -> Phi4MultimodalFeatureExtractor:
783 - return self.get_hf_processor(**kwargs).audio_processor
784 -
785 - def get_image_processor(
786 - self,
787 - processor: Phi4MMProcessor | None = None,
788 - ) -> Phi4MultimodalImageProcessorFast:
789 - if processor is None:
790 - processor = self.get_hf_processor()
791 - return processor.image_processor
792 -
793 - def get_dynamic_hd(
794 - self,
795 - processor: Phi4MMProcessor | None = None,
796 - ) -> int:
797 - return self.get_image_processor(processor).dynamic_hd
798 -
799 - def get_supported_mm_limits(self) -> Mapping[str, int | None]:
800 - return {"audio": None, "image": None}
801 -
802 - def _find_target_aspect_ratio(
803 - self,
804 - orig_width: int,
805 - orig_height: int,
806 - image_size: int,
807 - max_num: int,
808 - min_num: int,
809 - ):
810 - w_crop_num = math.ceil(orig_width / float(image_size))
811 - h_crop_num = math.ceil(orig_height / float(image_size))
812 - if w_crop_num * h_crop_num > max_num:
813 - aspect_ratio = orig_width / orig_height
814 -
815 - # calculate the existing image aspect ratio
816 - target_ratios = set(
817 - (i, j)
818 - for i in range(1, max_num + 1)
819 - for j in range(1, max_num + 1)
820 - if i * j <= max_num and i * j >= min_num
821 - )
822 - target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
823 -
824 - # find the closest aspect ratio to the target
825 - image_processor = self.get_image_processor()
826 - target_aspect_ratio = image_processor.find_closest_aspect_ratio(
827 - aspect_ratio,
828 - target_ratios,
829 - orig_width,
830 - orig_height,
831 - image_size,
832 - )
833 -
834 - # calculate the target width and height
835 - target_width = image_size * target_aspect_ratio[0]
836 - target_height = image_size * target_aspect_ratio[1]
837 - else:
838 - target_width = image_size * w_crop_num
839 - target_height = image_size * h_crop_num
840 - target_aspect_ratio = (w_crop_num, h_crop_num)
841 - return target_aspect_ratio, target_height, target_width
842 -
843 - def _compute_num_image_tokens(
844 - self,
845 - orig_width: int,
846 - orig_height: int,
847 - dynamic_hd_size: int,
848 - vit_image_size: int,
849 - vit_patch_size: int,
850 - token_compression_factor: int = 2,
851 - ):
852 - """
853 - compute the number of tokens an image is expected to take up considering
854 - the image encoder architecture and exclude output features containing
855 - only padding pixels
856 -
857 - for siglip, vit_image_size=448, vit_patch_size=14, so output will be
858 - 32x32 feature map
859 - NOTE right now, Phi4MM uses hard-coded token_compression_factor=2
860 - """
861 - assert vit_image_size % vit_patch_size == 0, (
862 - "vit_image_size must be divisible by vit_patch_size"
863 - )
864 - assert vit_image_size // vit_patch_size % token_compression_factor == 0, (
865 - "vit_image_size // vit_patch_size must be divisible by "
866 - "token_compression_factor"
867 - )
868 -
869 - target_aspect_ratio, target_height, target_width = (
870 - self._find_target_aspect_ratio(
871 - orig_width, orig_height, vit_image_size, dynamic_hd_size, min_num=1
872 - )
873 - )
874 - assert target_aspect_ratio[0] * vit_image_size == target_width, (
875 - f"{target_aspect_ratio[0]} * {vit_image_size} != {target_width}"
876 - )
877 - assert target_aspect_ratio[1] * vit_image_size == target_height, (
878 - f"{target_aspect_ratio[1]} * {vit_image_size} != {target_height}"
879 - )
880 - assert (
881 - target_height % vit_image_size == 0 and target_width % vit_image_size == 0
882 - )
883 -
884 - padding_height, padding_width = _get_padding_size(
885 - orig_width, orig_height, target_height, target_width
886 - )
887 - assert padding_width == 0 or padding_height == 0, (
888 - "padding_width or padding_height must be 0"
889 - )
890 -
891 - target_feat_width = target_width // vit_patch_size
892 - target_feat_height = target_height // vit_patch_size
893 - if padding_width >= vit_patch_size:
894 - assert padding_height == 0, "padding_height not 0"
895 - non_pad_feat_width = target_feat_width - math.floor(
896 - padding_width / vit_patch_size
897 - )
898 - non_pad_feat_height = target_feat_height
899 - elif padding_height >= vit_patch_size:
900 - assert padding_width == 0, "padding_width not 0"
901 - non_pad_feat_height = target_feat_height - math.floor(
902 - padding_height / vit_patch_size
903 - )
904 - non_pad_feat_width = target_feat_width
905 - else:
906 - # small padding shorter than a vit patch
907 - non_pad_feat_width = target_feat_width
908 - non_pad_feat_height = target_feat_height
909 -
910 - feat_width = non_pad_feat_width // token_compression_factor
911 - feat_height = non_pad_feat_height // token_compression_factor
912 - # NOTE it's possible that the non-padding feature is not divisible
913 - if non_pad_feat_width % token_compression_factor != 0:
914 - feat_width += 1
915 - if non_pad_feat_height % token_compression_factor != 0:
916 - feat_height += 1
917 - num_hd_patch_tokens = feat_width * feat_height
918 - num_hd_newline_tokens = feat_height
919 - vit_feature_size = vit_image_size // vit_patch_size
920 - num_global_image_tokens = (vit_feature_size // token_compression_factor) ** 2
921 - num_sep_tokens = 1
922 - num_global_image_newline_tokens = vit_feature_size // token_compression_factor
923 -
924 - return (
925 - num_global_image_tokens
926 - + num_sep_tokens
927 - + num_hd_patch_tokens
928 - + num_hd_newline_tokens
929 - + num_global_image_newline_tokens
930 - )
931 -
932 - def get_num_image_tokens(
933 - self,
934 - *,
935 - image_width: int,
936 - image_height: int,
937 - processor: Phi4MMProcessor | None = None,
938 - ) -> int:
939 - hf_config = self.get_hf_config()
940 - vision_config = hf_config.vision_config
941 - vit_image_size = vision_config.image_size
942 - vit_patch_size = vision_config.patch_size
943 -
944 - dynamic_hd_size = self.get_dynamic_hd(processor=processor)
945 -
946 - # we use default `token_compression_factor=2`,
947 - # since it's not in HF vision config.
948 - image_num_tokens = self._compute_num_image_tokens(
949 - image_width,
950 - image_height,
951 - dynamic_hd_size=dynamic_hd_size,
952 - vit_image_size=vit_image_size,
953 - vit_patch_size=vit_patch_size,
954 - )
955 -
956 - return image_num_tokens
957 -
958 - def get_image_size_with_most_features(
959 - self,
960 - processor: Phi4MMProcessor | None = None,
961 - ) -> ImageSize:
962 - vit_image_size = self.get_hf_config().vision_config.image_size
963 -
964 - max_side = vit_image_size * self.get_dynamic_hd(processor=processor)
965 - return ImageSize(height=max_side, width=vit_image_size)
966 -
967 - def get_audio_num_frames(self, audio_len: int, sr: float) -> int:
968 - """
969 - Compute the output size of the `extract_features` method.
970 -
971 - Args:
972 - audio_len (int): Length of the input waveform in samples.
973 - sr (float): Sampling rate of the waveform, either 16000 or 8000.
974 -
975 - Returns:
976 - tuple (int, int): Output size as (T, D), where:
977 - T: Number of time frames.
978 - D: Number of Mel filterbank bins (80).
979 - """
980 -
981 - # Resample to 16000 or 8000 if needed
982 - if sr > 16000:
983 - audio_len //= sr // 16000
984 - elif 8000 <= sr < 16000:
985 - # We'll resample to 16K from 8K
986 - audio_len *= 2
987 - elif sr < 8000:
988 - raise RuntimeError(f"Unsupported sample rate {sr}")
989 -
990 - # Spectrogram parameters for 16 kHz
991 - win_length = 400 # Frame length in samples
992 - hop_length = 160 # Frame shift in samples
993 -
994 - # Calculate number of frames (T)
995 - num_frames = (audio_len - win_length) // hop_length + 1
996 - if num_frames < 1:
997 - raise ValueError("Waveform too short for given parameters.")
998 -
999 - # Return time frames (T)
1000 - return num_frames
1001 -
1002 - def _compute_audio_embed_size(self, audio_frames: int) -> int:
1003 - """
1004 - Compute the size of audio embeddings from the number of audio frames.
1005 - """
1006 - # `_compute_audio_embed_size` in audio_processor use torch for
1007 - # computation, therefore we re-implement it to use pythonic
1008 - # numeric computation to avoid extra tensor conversion.
1009 - audio_processor = self.get_feature_extractor()
1010 - audio_compression_rate = audio_processor.audio_compression_rate
1011 - audio_downsample_rate = audio_processor.audio_downsample_rate
1012 -
1013 - integer = audio_frames // audio_compression_rate
1014 - remainder = audio_frames % audio_compression_rate
1015 - result = integer + int(remainder > 0)
1016 -
1017 - integer = result // audio_downsample_rate
1018 - remainder = result % audio_downsample_rate
1019 - result = integer + int(remainder > 0) # qformer compression
1020 -
1021 - return result
1022 -
1023 -
1024 -class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
1025 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
1026 - num_audios = mm_counts.get("audio", 0)
1027 - num_images = mm_counts.get("image", 0)
1028 -
1029 - tokenizer = self.info.get_tokenizer()
1030 - image_tokens: str = tokenizer.image_token * num_images
1031 - audio_tokens: str = tokenizer.audio_token * num_audios
1032 -
1033 - return image_tokens + audio_tokens
1034 -
1035 - def get_dummy_mm_data(
1036 - self,
1037 - seq_len: int,
1038 - mm_counts: Mapping[str, int],
1039 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1040 - ) -> MultiModalDataDict:
1041 - num_audios = mm_counts.get("audio", 0)
1042 - num_images = mm_counts.get("image", 0)
1043 -
1044 - target_width, target_height = self.info.get_image_size_with_most_features()
1045 -
1046 - image_overrides = mm_options.get("image") if mm_options else None
1047 - audio_overrides = mm_options.get("audio") if mm_options else None
1048 -
1049 - mm_data = {
1050 - "image": self._get_dummy_images(
1051 - width=target_width,
1052 - height=target_height,
1053 - num_images=num_images,
1054 - overrides=image_overrides,
1055 - ),
1056 - "audio": self._get_dummy_audios(
1057 - length=_AUDIO_MAX_SOUNDFILE_SIZE,
1058 - num_audios=num_audios,
1059 - overrides=audio_overrides,
1060 - ),
1061 - }
1062 -
1063 - return mm_data
1064 -
1065 -
1066 -class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1067 - def _get_data_parser(self) -> MultiModalDataParser:
1068 - feature_extractor = self.info.get_feature_extractor()
1069 - return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
1070 -
1071 - def _call_hf_processor(
1072 - self,
1073 - prompt: str,
1074 - mm_data: Mapping[str, object],
1075 - mm_kwargs: Mapping[str, object],
1076 - tok_kwargs: Mapping[str, object],
1077 - ) -> BatchFeature:
1078 - if not mm_data:
1079 - prompt_ids = self.info.get_tokenizer().encode(prompt)
1080 - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
1081 - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
1082 -
1083 - audio_data = mm_data.pop("audios", [])
1084 - if audio_data:
1085 - mm_data["audio"] = audio_data
1086 -
1087 - processed_outputs = super()._call_hf_processor(
1088 - prompt, mm_data, mm_kwargs, tok_kwargs
1089 - )
1090 -
1091 - if "image_pixel_values" in processed_outputs:
1092 - num_img_tokens = [
1093 - self.info.get_num_image_tokens(
1094 - image_width=img_size[0], image_height=img_size[1]
1095 - )
1096 - for img_size in processed_outputs["image_sizes"]
1097 - ]
1098 - processed_outputs["num_img_tokens"] = num_img_tokens
1099 -
1100 - if audio_data:
1101 - audio_features = processed_outputs["audio_input_features"]
1102 - sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
1103 - feature_sizes = [
1104 - self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
1105 - ]
1106 - processed_outputs["audio_input_features"] = [
1107 - audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
1108 - ]
1109 -
1110 - return processed_outputs
1111 -
1112 - def _get_mm_fields_config(
1113 - self,
1114 - hf_inputs: BatchFeature,
1115 - hf_processor_mm_kwargs: Mapping[str, object],
1116 - ) -> Mapping[str, MultiModalFieldConfig]:
1117 - return dict(
1118 - image_pixel_values=MultiModalFieldConfig.batched("image"),
1119 - image_attention_mask=MultiModalFieldConfig.batched("image"),
1120 - image_sizes=MultiModalFieldConfig.batched("image"),
1121 - num_img_tokens=MultiModalFieldConfig.batched("image"),
1122 - audio_input_features=MultiModalFieldConfig.batched("audio"),
1123 - )
1124 -
1125 - def _get_prompt_updates(
1126 - self,
1127 - mm_items: MultiModalDataItems,
1128 - hf_processor_mm_kwargs: Mapping[str, Any],
1129 - out_mm_kwargs: MultiModalKwargsItems,
1130 - ) -> Sequence[PromptUpdate]:
1131 - tokenizer = self.info.get_tokenizer()
1132 - image_token_id: int = tokenizer.vocab[tokenizer.image_token]
1133 - audio_token_id: int = tokenizer.vocab[tokenizer.audio_token]
1134 -
1135 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1136 - audio_processor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
1137 -
1138 - def get_image_replacement_phi4mm(item_idx: int):
1139 - images = mm_items.get_items(
1140 - "image", (ImageEmbeddingItems, ImageProcessorItems)
1141 - )
1142 -
1143 - if isinstance(images, ImageEmbeddingItems):
1144 - num_image_tokens = images.get_feature_size(item_idx)
1145 - else:
1146 - image_size = images.get_image_size(item_idx)
1147 - num_image_tokens = self.info.get_num_image_tokens(
1148 - image_width=image_size.width,
1149 - image_height=image_size.height,
1150 - processor=hf_processor,
1151 - )
1152 -
1153 - return [image_token_id] * num_image_tokens
1154 -
1155 - def get_audio_replacement_phi4mm(item_idx: int):
1156 - audios = mm_items.get_items("audio", AudioProcessorItems)
1157 - # TODO(Isotr0py): support embedding inputs
1158 - audio_len = audios.get_audio_length(item_idx)
1159 - audio_frames = self.info.get_audio_num_frames(
1160 - audio_len, audio_processor.sampling_rate
1161 - )
1162 - audio_embed_size = self.info._compute_audio_embed_size(audio_frames)
1163 -
1164 - return [audio_token_id] * audio_embed_size
1165 -
1166 - return [
1167 - PromptReplacement(
1168 - modality="audio",
1169 - target=[audio_token_id],
1170 - replacement=get_audio_replacement_phi4mm,
1171 - ),
1172 - PromptReplacement(
1173 - modality="image",
1174 - target=[image_token_id],
1175 - replacement=get_image_replacement_phi4mm,
1176 - ),
1177 - ]
1178 -
1179 -
1180 -@MULTIMODAL_REGISTRY.register_processor(
1181 - Phi4MMMultiModalProcessor,
1182 - info=Phi4MMProcessingInfo,
1183 - dummy_inputs=Phi4MMDummyInputsBuilder,
1184 -)
1185 -class Phi4MultimodalForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1186 - """
1187 - Implements the Phi-4-multimodal-instruct model in vLLM.
1188 - """
1189 -
1190 - merge_by_field_config = True
1191 -
1192 - packed_modules_mapping = {
1193 - "qkv_proj": [
1194 - "qkv_proj",
1195 - ],
1196 - "gate_up_proj": [
1197 - "gate_up_proj",
1198 - ],
1199 - }
1200 -
1201 - hf_to_vllm_mapper = WeightsMapper(
1202 - orig_to_new_prefix={
1203 - # Multimodal embedding
1204 - "model.embed_tokens_extend.": "",
1205 - # LLM backbone
1206 - "model.": "language_model.model.",
1207 - },
1208 - orig_to_new_substr={
1209 - # projection
1210 - ".img_projection_": ".img_projection.",
1211 - ".up_proj_for_speech.": ".speech_projection.up.",
1212 - ".up_proj_for_vision_speech.": ".vision_speech_projection.up.",
1213 - ".down_proj_for_speech.": ".speech_projection.down.",
1214 - ".down_proj_for_vision_speech.": ".vision_speech_projection.down.",
1215 - },
1216 - )
1217 -
1218 - @classmethod
1219 - def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1220 - if modality.startswith("image"):
1221 - return "<|image|>"
1222 - if modality.startswith("audio"):
1223 - return "<|audio|>"
1224 -
1225 - raise ValueError("Only image or audio modality is supported")
1226 -
1227 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1228 - super().__init__()
1229 - config = vllm_config.model_config.hf_config
1230 - multimodal_config = vllm_config.model_config.multimodal_config
1231 - self.config = config
1232 - self.multimodal_config = multimodal_config
1233 -
1234 - # TODO: Optionally initializes these for supporting input embeddings.
1235 - self.image_embed = Phi4MMImageEmbedding(
1236 - config,
1237 - # prefix=maybe_prefix(prefix, "image_embed"),
1238 - )
1239 - self.audio_embed = Phi4MMAudioEmbedding(
1240 - config,
1241 - # prefix=maybe_prefix(prefix, "audio_embed"),
1242 - )
1243 -
1244 - self.language_model = init_vllm_registered_model(
1245 - vllm_config=vllm_config,
1246 - prefix=maybe_prefix(prefix, "language_model"),
1247 - architectures=["Phi3ForCausalLM"],
1248 - )
1249 -
1250 - self.make_empty_intermediate_tensors = (
1251 - self.language_model.make_empty_intermediate_tensors
1252 - )
1253 -
1254 - def _parse_and_validate_audio_input(
1255 - self, **kwargs: object
1256 - ) -> Phi4MMAudioInputs | None:
1257 - """
1258 - Parse and validate the audio input to the model. This handles both
1259 - audio features and audio embeddings, but only the former is used for
1260 - now.
1261 -
1262 - Args:
1263 - kwargs (object): Keyword arguments.
1264 -
1265 - Returns:
1266 - Optional[Phi4MMAudioInputs]: Parsed and validated audio inputs.
1267 - """
1268 - audio_features = kwargs.pop("audio_input_features", None)
1269 - audio_embeds = kwargs.pop("audio_embeds", None)
1270 -
1271 - if audio_features is None and audio_embeds is None:
1272 - return None
1273 -
1274 - if audio_features is not None:
1275 - return Phi4MMAudioFeatureInputs(
1276 - type="audio_features",
1277 - audio_features=audio_features,
1278 - )
1279 -
1280 - if audio_embeds is not None:
1281 - return Phi4MMAudioEmbeddingInputs(type="audio_embeds", data=audio_embeds)
1282 -
1283 - raise AssertionError("This line should be unreachable.")
1284 -
1285 - def _process_audio_input(
1286 - self, audio_input: Phi4MMAudioInputs, audio_projection_mode: str
1287 - ) -> NestedTensors:
1288 - """
1289 - Create the audio embeddings from the audio input, where the audio input
1290 - is pairs of audio features and audio embed lengths. The audio input is
1291 - created by `input_mapper_for_phi4mm_audio`.
1292 -
1293 - Args:
1294 - audio_input (Phi4MMAudioInputs): Audio input.
1295 -
1296 - Returns:
1297 - NestedTensors: Audio embeddings
1298 - """
1299 - if audio_input["type"] == "audio_embeds":
1300 - return audio_input["data"]
1301 -
1302 - audio_features = audio_input["audio_features"]
1303 - # (e.g. multiple examples) and the second dim is the multi-audio dim
1304 - # (e.g. multiple audios in the same example)
1305 -
1306 - dtype = next(self.audio_embed.parameters()).dtype
1307 - audio_embeds = [
1308 - self.audio_embed(
1309 - features.unsqueeze(0).to(dtype),
1310 - audio_projection_mode=audio_projection_mode,
1311 - )
1312 - for features in audio_features
1313 - ]
1314 - return audio_embeds
1315 -
1316 - def _parse_and_validate_image_input(
1317 - self, **kwargs: object
1318 - ) -> Phi4MMImagePixelInputs | None:
1319 - pixel_values = kwargs.get("image_pixel_values")
1320 - if pixel_values is None:
1321 - return None
1322 -
1323 - image_sizes = kwargs.get("image_sizes")
1324 - image_attention_mask = kwargs.get("image_attention_mask")
1325 - num_img_tokens = kwargs.get("num_img_tokens")
1326 - assert (
1327 - image_sizes is not None
1328 - and image_attention_mask is not None
1329 - and num_img_tokens is not None
1330 - ), "Missing image inputs"
1331 -
1332 - return Phi4MMImagePixelInputs(
1333 - type="pixel_values",
1334 - pixel_values=pixel_values,
1335 - image_sizes=image_sizes,
1336 - image_attention_mask=image_attention_mask,
1337 - num_img_tokens=num_img_tokens,
1338 - )
1339 -
1340 - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1341 - modalities = {}
1342 -
1343 - # Preserve the order of modalities if there are multiple of them
1344 - # from the order of kwargs.
1345 - for input_key in kwargs:
1346 - if (
1347 - input_key in ("image_pixel_values", "image_embeds")
1348 - and "images" not in modalities
1349 - ):
1350 - modalities["images"] = self._parse_and_validate_image_input(**kwargs)
1351 - if (
1352 - input_key in ("audio_input_features", "audio_embeds")
1353 - and "audios" not in modalities
1354 - ):
1355 - modalities["audios"] = self._parse_and_validate_audio_input(**kwargs)
1356 -
1357 - return modalities
1358 -
1359 - def _process_image_input(
1360 - self, image_input: Phi4MMImagePixelInputs
1361 - ) -> list[torch.Tensor]:
1362 - if image_input["type"] == "image_embeds":
1363 - image_embeds = image_input["image_embeds"].type(self.visual.dtype)
1364 - else:
1365 - dtype = next(self.image_embed.parameters()).dtype
1366 - pixel_values = image_input["pixel_values"].to(dtype)
1367 - image_sizes = image_input["image_sizes"]
1368 - image_attention_mask = image_input["image_attention_mask"]
1369 - image_embeds = self.image_embed(
1370 - pixel_values, image_sizes, image_attention_mask
1371 - )
1372 - return image_embeds
1373 -
1374 - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1375 - modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1376 - if not modalities:
1377 - return []
1378 -
1379 - # The result multimodal_embeddings is tuple of tensors, with each
1380 - # tensor corresponding to a multimodal data item (image or video).
1381 - multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1382 -
1383 - # NOTE: It is important to iterate over the keys in this dictionary
1384 - # to preserve the order of the modalities.
1385 - audio_projection_mode = "speech"
1386 - for modality in modalities:
1387 - # make sure process images first
1388 - if modality == "images":
1389 - audio_projection_mode = "vision"
1390 - image_input = modalities["images"]
1391 - image_embeddings = self._process_image_input(image_input)
1392 - multimodal_embeddings += tuple(image_embeddings)
1393 - if modality == "audios":
1394 - audio_input = modalities["audios"]
1395 - audio_embeddings = self._process_audio_input(
1396 - audio_input, audio_projection_mode=audio_projection_mode
1397 - )
1398 - multimodal_embeddings += tuple(audio_embeddings)
1399 -
1400 - return multimodal_embeddings
1401 -
1402 - def forward(
1403 - self,
1404 - input_ids: torch.Tensor,
1405 - positions: torch.Tensor,
1406 - intermediate_tensors: IntermediateTensors | None = None,
1407 - inputs_embeds: torch.Tensor | None = None,
1408 - **kwargs: object,
1409 - ) -> torch.Tensor:
1410 - if intermediate_tensors is not None:
1411 - inputs_embeds = None
1412 -
1413 - hidden_states = self.language_model(
1414 - input_ids,
1415 - positions,
1416 - intermediate_tensors,
1417 - inputs_embeds=inputs_embeds,
1418 - )
1419 -
1420 - return hidden_states
1421 -
1422 - def compute_logits(
1423 - self,
1424 - hidden_states: torch.Tensor,
1425 - ) -> torch.Tensor | None:
1426 - return self.language_model.compute_logits(hidden_states)
1427 -
1428 - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1429 - loader = AutoWeightsLoader(self)
1430 - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1431 -
1432 - def get_mm_mapping(self) -> MultiModelKeys:
1433 - """
1434 - Get the module prefix in multimodal models
1435 - """
1436 - return MultiModelKeys.from_string_field(
1437 - language_model="language_model.",
1438 - connector=[
1439 - "img_projection",
1440 - "vision_speech_projection",
1441 - "speech_projection",
1442 - ],
1443 - tower_model=["image_embed", "audio_embed"],
1444 - )
1445 -
1446 - def get_language_model(self) -> torch.nn.Module:
1447 - return self.language_model
vllm/model_executor/models/registry.py
+1 -1
@@ -374,7 +374,6 @@ _MULTIMODAL_MODELS = {
374 ),
375 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
376 "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
377 - "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"), # noqa: E501
377 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
378 "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501
379 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
@@ -507,6 +506,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
506 "MotifForCausalLM": "0.10.2",
507 "Phi3SmallForCausalLM": "0.9.2",
508 "Phi4FlashForCausalLM": "0.10.2",
509 + "Phi4MultimodalForCausalLM": "0.12.0",
510 # encoder-decoder models except whisper
511 # have been removed for V0 deprecation.
512 "BartModel": "0.10.2",