@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ad7c0335f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 14:45 UTC

[Model] Add Phi4ForCausalLMV for microsoft/Phi-4-reasoning-vision-15B (#38306)

Signed-off-by: Varun Sundar Rabindranath <vsundarr@redhat.com> Co-authored-by: Varun Sundar Rabindranath <vsundarr@redhat.com>

Varun Sundar Rabindranath committed Apr 3, 2026 at 00:14 UTC 2ad7c0335f3a3a7e726d0431bb2c26401c9a6b40
5 files changed +622 -1
.buildkite/test_areas/models_distributed.yaml
+2 -1
@@ -18,5 +18,6 @@ steps:
18 # Avoid importing model tests that cause CUDA reinitialization error
19 - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
20 - pytest models/language -v -s -m 'distributed(num_gpus=2)'
21 - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py
21 + - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
22 + - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
23 - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
tests/models/multimodal/generation/test_phi4siglip.py new
+187
@@ -0,0 +1,187 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Sequence
5 +
6 +import pytest
7 +import regex as re
8 +from transformers import AutoModelForCausalLM, AutoTokenizer
9 +
10 +from vllm.logprobs import SampleLogprobs
11 +from vllm.multimodal.image import rescale_image_size
12 +
13 +from ....conftest import (
14 + IMAGE_ASSETS,
15 + HfRunner,
16 + PromptImageInput,
17 + VllmRunner,
18 +)
19 +from ....utils import multi_gpu_test
20 +from ...utils import check_logprobs_close
21 +
22 +MODEL_ID = "microsoft/Phi-4-reasoning-vision-15B"
23 +
24 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
25 + {
26 + "stop_sign": "<|user|>\n<image>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
27 + "cherry_blossom": "<|user|>\n<image>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
28 + }
29 +)
30 +HF_MULTIIMAGE_IMAGE_PROMPT = (
31 + "<|user|>\n<image>\n<image>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
32 +)
33 +
34 +DTYPE = "half"
35 +MAX_TOKENS = 128
36 +NUM_LOGPROBS = 10
37 +
38 +
39 +def vllm_to_hf_output(
40 + vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
41 +):
42 + """Sanitize vllm output to be comparable with hf output."""
43 + _, output_str, out_logprobs = vllm_output
44 +
45 + output_str_without_image = re.sub(r"(<image>)+", "", output_str)
46 + if output_str_without_image and output_str_without_image[0] == " ":
47 + output_str_without_image = output_str_without_image[1:]
48 +
49 + hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
50 +
51 + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
52 + hf_output_ids = tokenizer.encode(output_str_without_image)
53 + if hf_output_ids and hf_output_ids[0] == tokenizer.bos_token_id:
54 + hf_output_ids = hf_output_ids[1:]
55 +
56 + return hf_output_ids, hf_output_str, out_logprobs
57 +
58 +
59 +def _build_single_image_inputs(
60 + image_assets,
61 +) -> list[tuple[list[str], PromptImageInput]]:
62 + """Build single-image inputs for all size_factors at once."""
63 + images = [asset.pil_image for asset in image_assets]
64 + all_inputs: list[tuple[list[str], PromptImageInput]] = []
65 + for size_factors in [[1.0], [0.25, 0.5, 1.0]]:
66 + for image, prompt in zip(images, HF_IMAGE_PROMPTS):
67 + all_inputs.append(
68 + (
69 + [prompt for _ in size_factors],
70 + [rescale_image_size(image, f) for f in size_factors],
71 + )
72 + )
73 + return all_inputs
74 +
75 +
76 +def _build_multi_image_inputs(
77 + image_assets,
78 +) -> list[tuple[list[str], PromptImageInput]]:
79 + """Build multi-image inputs for all size_factors at once."""
80 + images = [asset.pil_image for asset in image_assets]
81 + all_inputs: list[tuple[list[str], PromptImageInput]] = []
82 + for size_factors in [[0.5], [0.15, 0.30]]:
83 + all_inputs.append(
84 + (
85 + [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
86 + [
87 + [rescale_image_size(image, factor) for image in images]
88 + for factor in size_factors
89 + ],
90 + )
91 + )
92 + return all_inputs
93 +
94 +
95 +def _run_and_compare(
96 + hf_runner: type[HfRunner],
97 + vllm_runner: type[VllmRunner],
98 + all_inputs: Sequence[tuple[list[str], PromptImageInput]],
99 + model: str,
100 + max_model_len: int,
101 + max_num_seqs: int,
102 + mm_limit: int,
103 + gpu_memory_utilization: float,
104 +):
105 + """Load each runner once, run all inputs, then compare."""
106 + # NOTE: run vLLM first, then HF. vLLM needs a fresh process without
107 + # cuda initialization; running HF first would break the multiprocessing
108 + # backend with fork method.
109 + with vllm_runner(
110 + model,
111 + runner="generate",
112 + max_model_len=max_model_len,
113 + max_num_seqs=max_num_seqs,
114 + gpu_memory_utilization=gpu_memory_utilization,
115 + dtype=DTYPE,
116 + limit_mm_per_prompt={"image": mm_limit},
117 + tensor_parallel_size=2,
118 + trust_remote_code=True,
119 + enforce_eager=True,
120 + ) as vllm_model:
121 + vllm_outputs_per_case = [
122 + vllm_model.generate_greedy_logprobs(
123 + prompts,
124 + MAX_TOKENS,
125 + num_logprobs=NUM_LOGPROBS,
126 + images=images,
127 + )
128 + for prompts, images in all_inputs
129 + ]
130 +
131 + hf_model_kwargs = {"_attn_implementation": "sdpa", "device_map": "auto"}
132 + with hf_runner(
133 + model,
134 + dtype=DTYPE,
135 + model_kwargs=hf_model_kwargs,
136 + auto_cls=AutoModelForCausalLM,
137 + trust_remote_code=True,
138 + ) as hf_model:
139 + hf_outputs_per_case = [
140 + hf_model.generate_greedy_logprobs_limit(
141 + prompts,
142 + MAX_TOKENS,
143 + num_logprobs=NUM_LOGPROBS,
144 + images=images,
145 + )
146 + for prompts, images in all_inputs
147 + ]
148 +
149 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
150 + check_logprobs_close(
151 + outputs_0_lst=hf_outputs,
152 + outputs_1_lst=vllm_outputs,
153 + name_0="hf",
154 + name_1="vllm",
155 + )
156 +
157 +
158 +@multi_gpu_test(num_gpus=2)
159 +@pytest.mark.parametrize("model", [MODEL_ID])
160 +def test_models(hf_runner, vllm_runner, image_assets, model) -> None:
161 + all_inputs = _build_single_image_inputs(image_assets)
162 + _run_and_compare(
163 + hf_runner,
164 + vllm_runner,
165 + all_inputs,
166 + model,
167 + max_model_len=8192,
168 + max_num_seqs=2,
169 + mm_limit=1,
170 + gpu_memory_utilization=0.80,
171 + )
172 +
173 +
174 +@multi_gpu_test(num_gpus=2)
175 +@pytest.mark.parametrize("model", [MODEL_ID])
176 +def test_multi_images_models(hf_runner, vllm_runner, image_assets, model) -> None:
177 + all_inputs = _build_multi_image_inputs(image_assets)
178 + _run_and_compare(
179 + hf_runner,
180 + vllm_runner,
181 + all_inputs,
182 + model,
183 + max_model_len=8192,
184 + max_num_seqs=2,
185 + mm_limit=2,
186 + gpu_memory_utilization=0.80,
187 + )
tests/models/registry.py
+3
@@ -1049,6 +1049,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1049 }, # noqa: E501
1050 extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
1051 ),
1052 + "Phi4ForCausalLMV": _HfExamplesInfo(
1053 + "microsoft/Phi-4-reasoning-vision-15B", trust_remote_code=True
1054 + ),
1055 "Phi4MMForCausalLM": _HfExamplesInfo(
1056 "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
1057 ),
vllm/model_executor/models/phi4siglip.py new
+429
@@ -0,0 +1,429 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""vLLM support for microsoft/Phi-4-reasoning-vision-15B.
4 +
5 +Architecture: Siglip2 vision tower + MLP projector + Phi3 language model.
6 +"""
7 +
8 +import math
9 +from collections.abc import Iterable, Mapping, Sequence
10 +from typing import Annotated, Any, Literal
11 +
12 +import torch
13 +import torch.nn as nn
14 +from transformers import BatchFeature, PretrainedConfig, Siglip2VisionConfig
15 +
16 +from vllm.config import VllmConfig
17 +from vllm.config.multimodal import BaseDummyOptions
18 +from vllm.inputs import MultiModalDataDict
19 +from vllm.logger import init_logger
20 +from vllm.multimodal import MULTIMODAL_REGISTRY
21 +from vllm.multimodal.inputs import (
22 + MultiModalFieldConfig,
23 + MultiModalKwargsItems,
24 +)
25 +from vllm.multimodal.parse import (
26 + ImageSize,
27 + MultiModalDataItems,
28 +)
29 +from vllm.multimodal.processing import (
30 + BaseDummyInputsBuilder,
31 + PromptReplacement,
32 + PromptUpdate,
33 +)
34 +from vllm.multimodal.processing.processor import (
35 + BaseMultiModalProcessor,
36 + BaseProcessingInfo,
37 +)
38 +from vllm.sequence import IntermediateTensors
39 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
40 +
41 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
42 +from .lfm2_siglip2 import Siglip2Model
43 +from .llava import LlavaMultiModalProjector
44 +from .utils import (
45 + AutoWeightsLoader,
46 + WeightsMapper,
47 + init_vllm_registered_model,
48 + maybe_prefix,
49 +)
50 +
51 +logger = init_logger(__name__)
52 +
53 +IMAGE_TOKEN_INDEX = -200
54 +DEFAULT_IMAGE_TOKEN = "<image>"
55 +
56 +# The HF processor replaces "<image>" with IMAGE_TOKEN_INDEX (-200) in input_ids.
57 +# Negative token IDs cause OverflowError during decoding, so we remap to a real
58 +# in-vocabulary token. The Phi-4-reasoning-vision tokenizer ships with reserved
59 +# dummy tokens (<|dummy_0|> … <|dummy_83|>); we reuse the first one as the
60 +# image placeholder. This mirrors how Phi-3-vision uses its dedicated <|image|>
61 +# token (ID 32044).
62 +_IMAGE_TOKEN_ID = 100256 # <|dummy_0|> in the Phi-4 tokenizer
63 +
64 +
65 +# ---------------------------------------------------------------------------
66 +# Processing
67 +# ---------------------------------------------------------------------------
68 +
69 +
70 +class Phi4SiglipProcessingInfo(BaseProcessingInfo):
71 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
72 + return {"image": None}
73 +
74 + def _get_vision_config(self) -> dict:
75 + return self.get_hf_config().vision_config # type: ignore[attr-defined]
76 +
77 + def _get_patch_size(self) -> int:
78 + vc = self._get_vision_config()
79 + if isinstance(vc, dict):
80 + return vc.get("patch_size", 16)
81 + return getattr(vc, "patch_size", 16)
82 +
83 + def _get_max_num_patches(self) -> int:
84 + return getattr(self.get_hf_config(), "max_num_patches", 3600)
85 +
86 + def _get_min_num_patches(self) -> int:
87 + return getattr(self.get_hf_config(), "min_num_patches", 256)
88 +
89 + def get_num_image_tokens(
90 + self,
91 + *,
92 + image_width: int,
93 + image_height: int,
94 + ) -> int:
95 + patch_size = self._get_patch_size()
96 + min_patches = self._get_min_num_patches()
97 + max_patches = self._get_max_num_patches()
98 +
99 + num_patches_h = image_height // patch_size
100 + num_patches_w = image_width // patch_size
101 + num_patches = max(num_patches_h * num_patches_w, 1)
102 + num_patches = max(min(num_patches, max_patches), min_patches)
103 + return num_patches
104 +
105 + def get_image_size_with_most_features(self) -> ImageSize:
106 + patch_size = self._get_patch_size()
107 + max_patches = self._get_max_num_patches()
108 + side = int(math.sqrt(max_patches)) * patch_size
109 + return ImageSize(width=side, height=side)
110 +
111 + def get_mm_max_tokens_per_item(
112 + self, seq_len: int, mm_counts: Mapping[str, int]
113 + ) -> Mapping[str, int]:
114 + return {"image": self._get_max_num_patches()}
115 +
116 +
117 +class Phi4SiglipDummyInputsBuilder(
118 + BaseDummyInputsBuilder[Phi4SiglipProcessingInfo],
119 +):
120 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
121 + num_images = mm_counts.get("image", 0)
122 + return DEFAULT_IMAGE_TOKEN * num_images
123 +
124 + def get_dummy_mm_data(
125 + self,
126 + seq_len: int,
127 + mm_counts: Mapping[str, int],
128 + mm_options: Mapping[str, BaseDummyOptions],
129 + ) -> MultiModalDataDict:
130 + num_images = mm_counts.get("image", 0)
131 + size = self.info.get_image_size_with_most_features()
132 + return {
133 + "image": self._get_dummy_images(
134 + width=size.width,
135 + height=size.height,
136 + num_images=num_images,
137 + overrides=mm_options.get("image"),
138 + ),
139 + }
140 +
141 +
142 +class Phi4SiglipMultiModalProcessor(
143 + BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
144 +):
145 + def _call_hf_processor(
146 + self,
147 + prompt: str,
148 + mm_data: Mapping[str, object],
149 + mm_kwargs: Mapping[str, object],
150 + tok_kwargs: Mapping[str, object],
151 + ) -> BatchFeature:
152 + processed = super()._call_hf_processor(
153 + prompt=prompt,
154 + mm_data=mm_data,
155 + mm_kwargs=mm_kwargs,
156 + tok_kwargs=tok_kwargs,
157 + )
158 +
159 + # The HF processor's tokenizer_image_token() replaces the "<image>"
160 + # string with IMAGE_TOKEN_INDEX (-200) in input_ids. This breaks
161 + # vLLM's prompt-replacement pipeline which needs to find "<image>"
162 + # as normal sub-tokens. Re-tokenize with the plain tokenizer so
163 + # that "<image>" stays as sub-tokens and can be located by
164 + # PromptReplacement.
165 + # NOTE: tokenizer.__call__() (not .encode()) must be used so that
166 + # added/special tokens like <|user|>, <|end|> are kept as single IDs.
167 + tokenizer = self.info.get_tokenizer()
168 + new_ids = tokenizer(prompt).input_ids
169 + processed["input_ids"] = torch.tensor([new_ids])
170 +
171 + return processed
172 +
173 + def _hf_processor_applies_updates(
174 + self,
175 + prompt_text: str,
176 + mm_items: MultiModalDataItems,
177 + hf_processor_mm_kwargs: Mapping[str, object],
178 + tokenization_kwargs: Mapping[str, object],
179 + ) -> bool:
180 + # The HF processor replaces "<image>" with a single -200 placeholder
181 + # but does NOT expand it into N vision-encoder tokens. Since we also
182 + # re-tokenize the prompt (see _call_hf_processor), prompt updates are
183 + # never applied by the HF processor — vLLM handles the expansion via
184 + # _apply_prompt_updates.
185 + return False
186 +
187 + def _get_mm_fields_config(
188 + self,
189 + hf_inputs: BatchFeature,
190 + hf_processor_mm_kwargs: Mapping[str, object],
191 + ) -> Mapping[str, MultiModalFieldConfig]:
192 + return dict(
193 + pixel_values=MultiModalFieldConfig.batched("image"),
194 + pixel_attention_mask=MultiModalFieldConfig.batched("image"),
195 + spatial_shapes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
196 + )
197 +
198 + def _get_prompt_updates(
199 + self,
200 + mm_items: MultiModalDataItems,
201 + hf_processor_mm_kwargs: Mapping[str, Any],
202 + out_mm_kwargs: MultiModalKwargsItems,
203 + ) -> Sequence[PromptUpdate]:
204 + def get_replacement(item_idx: int):
205 + # Read the actual patch grid from the NaFlex processor's
206 + # spatial_shapes output (same pattern as LFM2-VL). This avoids
207 + # predicting from raw image dimensions, which can diverge from
208 + # the NaFlex resize/tile logic.
209 + out_item = out_mm_kwargs["image"][item_idx]
210 + spatial_shapes = out_item["spatial_shapes"].data
211 + assert isinstance(spatial_shapes, torch.Tensor)
212 + num_tokens = int(spatial_shapes.prod().item())
213 + return [_IMAGE_TOKEN_ID] * num_tokens
214 +
215 + return [
216 + PromptReplacement(
217 + modality="image",
218 + target=DEFAULT_IMAGE_TOKEN,
219 + replacement=get_replacement,
220 + ),
221 + ]
222 +
223 +
224 +# ---------------------------------------------------------------------------
225 +# Input schemas
226 +# ---------------------------------------------------------------------------
227 +
228 +
229 +class Phi4SiglipImagePixelInputs(TensorSchema):
230 + """
231 + Dimensions:
232 + - bn: Batch size * number of images
233 + - d: Max number of patches (padded across images in the batch)
234 + - fd: Features per patch (patch_size * patch_size * channels)
235 + """
236 +
237 + type: Literal["pixel_values"] = "pixel_values"
238 + pixel_values: Annotated[torch.Tensor, TensorShape("bn", "d", "fd")]
239 + pixel_attention_mask: Annotated[torch.Tensor, TensorShape("bn", "d")]
240 + spatial_shapes: Annotated[torch.Tensor, TensorShape("bn", 2)]
241 +
242 +
243 +# ---------------------------------------------------------------------------
244 +# Model
245 +# ---------------------------------------------------------------------------
246 +
247 +
248 +@MULTIMODAL_REGISTRY.register_processor(
249 + Phi4SiglipMultiModalProcessor,
250 + info=Phi4SiglipProcessingInfo,
251 + dummy_inputs=Phi4SiglipDummyInputsBuilder,
252 +)
253 +class Phi4ForCausalLMV(nn.Module, SupportsMultiModal, SupportsPP):
254 + hf_to_vllm_mapper = WeightsMapper(
255 + orig_to_new_prefix={
256 + "model.vision_tower.vision_tower.vision_model.head.": None,
257 + "model.vision_tower.vision_tower.": "vision_tower.",
258 + "model.mm_projector.0.": "multi_modal_projector.linear_1.",
259 + "model.mm_projector.2.": "multi_modal_projector.linear_2.",
260 + "lm_head.": "language_model.lm_head.",
261 + "model.": "language_model.model.",
262 + },
263 + )
264 +
265 + @classmethod
266 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
267 + if modality.startswith("image"):
268 + return DEFAULT_IMAGE_TOKEN
269 + raise ValueError("Only image modality is supported")
270 +
271 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
272 + super().__init__()
273 +
274 + config: PretrainedConfig = vllm_config.model_config.hf_config
275 + quant_config = vllm_config.quant_config
276 + self.config = config
277 +
278 + vision_config_dict: dict = getattr(config, "vision_config", {})
279 + if isinstance(vision_config_dict, dict):
280 + if "patch_size" not in vision_config_dict:
281 + vision_config_dict["patch_size"] = 16
282 + siglip2_config = Siglip2VisionConfig(**vision_config_dict)
283 + else:
284 + siglip2_config = vision_config_dict
285 +
286 + vision_hidden_size: int = config.mm_hidden_size # type: ignore[attr-defined]
287 + text_hidden_size: int = config.hidden_size # type: ignore[attr-defined]
288 +
289 + with self._mark_tower_model(vllm_config, "image"):
290 + layer_idx = -2
291 + num_hidden_layers = siglip2_config.num_hidden_layers + layer_idx + 1
292 +
293 + self.vision_tower = Siglip2Model(
294 + siglip2_config,
295 + quant_config=quant_config,
296 + num_hidden_layers_override=num_hidden_layers,
297 + require_post_norm=False,
298 + prefix=maybe_prefix(prefix, "vision_tower"),
299 + )
300 + self.multi_modal_projector = LlavaMultiModalProjector(
301 + vision_hidden_size=vision_hidden_size,
302 + text_hidden_size=text_hidden_size,
303 + projector_hidden_act="gelu",
304 + multimodal_projector_bias=True,
305 + quant_config=quant_config,
306 + prefix=maybe_prefix(prefix, "multi_modal_projector"),
307 + )
308 +
309 + with self._mark_language_model(vllm_config):
310 + self.language_model = init_vllm_registered_model(
311 + vllm_config=vllm_config,
312 + hf_config=config,
313 + prefix=maybe_prefix(prefix, "language_model"),
314 + architectures=["Phi3ForCausalLM"],
315 + )
316 +
317 + self.make_empty_intermediate_tensors = (
318 + self.language_model.make_empty_intermediate_tensors
319 + )
320 +
321 + self.configure_mm_token_handling(
322 + vocab_size=config.vocab_size, # type: ignore[attr-defined]
323 + mm_token_ids=[_IMAGE_TOKEN_ID],
324 + )
325 +
326 + def _packed_from_padded(
327 + self,
328 + pixel_values: torch.Tensor,
329 + pixel_attention_mask: torch.Tensor,
330 + spatial_shapes: torch.Tensor,
331 + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
332 + """Convert padded NaFlex tensors to packed format for Siglip2Model."""
333 + valid_counts = pixel_attention_mask.sum(dim=1).to(torch.int32)
334 + pixel_values_packed = pixel_values[pixel_attention_mask.bool()]
335 + cu_seqlens = torch.zeros(
336 + len(valid_counts) + 1,
337 + dtype=torch.int32,
338 + device=pixel_values.device,
339 + )
340 + cu_seqlens[1:] = valid_counts.cumsum(0)
341 + max_seqlen = valid_counts.max()
342 + return (
343 + pixel_values_packed,
344 + spatial_shapes,
345 + cu_seqlens,
346 + max_seqlen,
347 + )
348 +
349 + def _parse_and_validate_image_input(
350 + self, **kwargs: object
351 + ) -> Phi4SiglipImagePixelInputs | None:
352 + pixel_values = kwargs.pop("pixel_values", None)
353 + pixel_attention_mask = kwargs.pop("pixel_attention_mask", None)
354 + spatial_shapes = kwargs.pop("spatial_shapes", None)
355 + if pixel_values is None:
356 + return None
357 +
358 + return Phi4SiglipImagePixelInputs(
359 + type="pixel_values",
360 + pixel_values=pixel_values,
361 + pixel_attention_mask=pixel_attention_mask,
362 + spatial_shapes=spatial_shapes,
363 + )
364 +
365 + def _process_image_input(
366 + self, image_input: Phi4SiglipImagePixelInputs
367 + ) -> MultiModalEmbeddings:
368 + pixel_values = image_input["pixel_values"]
369 + pixel_attention_mask = image_input["pixel_attention_mask"]
370 + spatial_shapes = image_input["spatial_shapes"]
371 +
372 + (
373 + pixel_values_packed,
374 + spatial_shapes_packed,
375 + cu_seqlens,
376 + max_seqlen,
377 + ) = self._packed_from_padded(pixel_values, pixel_attention_mask, spatial_shapes)
378 +
379 + vision_features = self.vision_tower(
380 + pixel_values_packed=pixel_values_packed,
381 + spatial_shapes=spatial_shapes_packed,
382 + cu_seqlens=cu_seqlens,
383 + max_seqlen=max_seqlen,
384 + select_layers=[-2],
385 + )
386 +
387 + if vision_features.dim() == 3:
388 + vision_features = vision_features.squeeze(0)
389 +
390 + image_features = self.multi_modal_projector(vision_features)
391 +
392 + valid_counts = pixel_attention_mask.sum(dim=1).tolist()
393 + return torch.split(image_features, [int(c) for c in valid_counts])
394 +
395 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
396 + image_input = self._parse_and_validate_image_input(**kwargs)
397 + if image_input is None:
398 + return []
399 +
400 + return self._process_image_input(image_input)
401 +
402 + def forward(
403 + self,
404 + input_ids: torch.Tensor | None,
405 + positions: torch.Tensor,
406 + intermediate_tensors: IntermediateTensors | None = None,
407 + inputs_embeds: torch.Tensor | None = None,
408 + **kwargs: object,
409 + ) -> torch.Tensor | IntermediateTensors:
410 + if intermediate_tensors is not None:
411 + inputs_embeds = None
412 +
413 + hidden_states = self.language_model.model(
414 + input_ids,
415 + positions,
416 + intermediate_tensors,
417 + inputs_embeds=inputs_embeds,
418 + )
419 + return hidden_states
420 +
421 + def compute_logits(
422 + self,
423 + hidden_states: torch.Tensor,
424 + ) -> torch.Tensor | None:
425 + return self.language_model.compute_logits(hidden_states)
426 +
427 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
428 + loader = AutoWeightsLoader(self)
429 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/registry.py
+1
@@ -481,6 +481,7 @@ _MULTIMODAL_MODELS = {
481 "PaliGemmaForConditionalGeneration",
482 ),
483 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
484 + "Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
485 "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
486 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
487 "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),