@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d249a9e90e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 21:15 UTC

Add Granite 4.1 Vision as built-in multimodal model (#40282)

Signed-off-by: Artem Spector <artems@il.ibm.com> Signed-off-by: artemspector <artems@il.ibm.com> Co-authored-by: artemspector <artems@il.ibm.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

artem-spector committed Apr 21, 2026 at 15:43 UTC d249a9e90eda9c8588b4f093293be6c08f3bf9ec
11 files changed +1186
docs/models/supported_models.md
+1
@@ -560,6 +560,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
560 | `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ |
561 | `Glm4vMoeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | ✅︎ | ✅︎ |
562 | `GlmOcrForConditionalGeneration` | GLM-OCR | T + I<sup>E+</sup> | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ |
563 +| `Granite4VisionForConditionalGeneration` | Granite 4 Vision | T + I<sup>E+</sup> | `ibm-granite/granite-4.1-3b-vision`, etc. | ✅︎ | ✅︎ |
564 | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
565 | `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
566 | `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
examples/offline_inference/vision_language_multi_image.py
+33
@@ -310,6 +310,38 @@ def load_gemma3(question: str, image_urls: list[str]) -> ModelRequestData:
310 )
311
312
313 +def load_granite4_vision(question: str, image_urls: list[str]) -> ModelRequestData:
314 + model_name = "ibm-granite/granite-vision-4.1-4b"
315 + engine_args = EngineArgs(
316 + model=model_name,
317 + max_model_len=4096,
318 + max_num_seqs=16,
319 + limit_mm_per_prompt={"image": len(image_urls)},
320 + )
321 +
322 + placeholders = [{"type": "image", "image": url} for url in image_urls]
323 + messages = [
324 + {
325 + "role": "user",
326 + "content": [
327 + *placeholders,
328 + {"type": "text", "text": question},
329 + ],
330 + }
331 + ]
332 +
333 + processor = AutoProcessor.from_pretrained(model_name)
334 + prompt = processor.apply_chat_template(
335 + messages, tokenize=False, add_generation_prompt=True
336 + )
337 +
338 + return ModelRequestData(
339 + engine_args=engine_args,
340 + prompt=prompt,
341 + image_data=[fetch_image(url) for url in image_urls],
342 + )
343 +
344 +
345 def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:
346 model_name = "h2oai/h2ovl-mississippi-800m"
347
@@ -1487,6 +1519,7 @@ model_example_map = {
1519 "deepseek_ocr": load_deepseek_ocr,
1520 "exaone4_5": load_exaone4_5,
1521 "gemma3": load_gemma3,
1522 + "granite4_vision": load_granite4_vision,
1523 "h2ovl_chat": load_h2ovl,
1524 "hunyuan_vl": load_hunyuan_vl,
1525 "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
tests/models/multimodal/generation/test_common.py
+37
@@ -86,6 +86,29 @@ COMMON_BROADCAST_SETTINGS = {
86 # which cases would be selected and deselected by pytest. In general,
87 # this is a good idea for checking your command first, since tests are slow.
88
89 +
90 +def _granite4_vision_vllm_to_hf_output(vllm_output, model):
91 + """Post-processor for granite4_vision vLLM output.
92 +
93 + Self-contained to avoid calling AutoConfig/AutoTokenizer without
94 + trust_remote_code (needed while the model is not in upstream HF).
95 + """
96 + output_ids, output_str, out_logprobs = vllm_output
97 + mm_token_id = 100352
98 + hf_output_ids = [
99 + token_id
100 + for idx, token_id in enumerate(output_ids)
101 + if token_id != mm_token_id or idx == 0 or output_ids[idx - 1] != mm_token_id
102 + ]
103 + hf_output_str = (
104 + output_str[1:] if output_str and output_str[0] == " " else output_str
105 + )
106 + eos_token_id = 100257
107 + if hf_output_ids and hf_output_ids[-1] == eos_token_id:
108 + hf_output_str = hf_output_str + "<|end_of_text|>"
109 + return hf_output_ids, hf_output_str, out_logprobs
110 +
111 +
112 VLM_TEST_SETTINGS = {
113 #### Core tests to always run in the CI
114 "llava": VLMTestInfo(
@@ -492,6 +515,20 @@ VLM_TEST_SETTINGS = {
515 auto_cls=AutoModelForImageTextToText,
516 marks=[large_gpu_mark(min_gb=32)],
517 ),
518 + "granite4_vision": VLMTestInfo(
519 + models=["ibm-granite/granite-vision-4.1-4b"],
520 + test_type=(VLMTestType.IMAGE),
521 + prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}\n<|assistant|>\n",
522 + max_model_len=8192,
523 + auto_cls=AutoModelForImageTextToText,
524 + vllm_output_post_proc=_granite4_vision_vllm_to_hf_output,
525 + image_size_factors=[(1.0,)],
526 + vllm_runner_kwargs={
527 + "enable_lora": True,
528 + "max_lora_rank": 256,
529 + "default_mm_loras": {"image": "ibm-granite/granite-vision-4.1-4b"},
530 + },
531 + ),
532 "h2ovl": VLMTestInfo(
533 models=[
534 "h2oai/h2ovl-mississippi-800m",
tests/models/registry.py
+4
@@ -899,6 +899,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
899 "zai-org/GLM-ASR-Nano-2512",
900 min_transformers_version="5.0.0",
901 ),
902 + "Granite4VisionForConditionalGeneration": _HfExamplesInfo(
903 + "ibm-granite/granite-vision-4.1-4b",
904 + is_available_online=False,
905 + ),
906 "GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"),
907 "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
908 "ibm-granite/granite-speech-3.3-2b",
vllm/model_executor/models/granite4_vision.py new
+930
@@ -0,0 +1,930 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""vLLM implementation of Granite 4 Vision.
4 +
5 +Uses GraniteForCausalLM as the language backbone with SigLIP vision encoder
6 +and deepstack feature injection via WindowQFormer projectors.
7 +
8 +LoRA support: use --enable-lora --default-mm-loras for LM-only LoRA adapters.
9 +"""
10 +
11 +import math
12 +from collections.abc import Iterable, Mapping
13 +from fractions import Fraction
14 +from itertools import islice
15 +
16 +import torch
17 +import torch.nn as nn
18 +from transformers import BatchFeature
19 +from transformers.models.blip_2.configuration_blip_2 import Blip2QFormerConfig
20 +from transformers.models.llava_next.modeling_llava_next import (
21 + get_anyres_image_grid_shape,
22 + image_size_to_num_patches,
23 + unpad_image,
24 +)
25 +
26 +from vllm.compilation.decorators import support_torch_compile
27 +from vllm.config import CacheConfig, VllmConfig
28 +from vllm.distributed.parallel_state import get_pp_group
29 +from vllm.logger import init_logger
30 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
31 +from vllm.model_executor.layers.quantization import QuantizationConfig
32 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
33 +from vllm.model_executor.models.granite import GraniteForCausalLM, GraniteModel
34 +from vllm.model_executor.models.interfaces import (
35 + MultiModalEmbeddings,
36 + SupportsLoRA,
37 + SupportsMultiModal,
38 + SupportsPP,
39 +)
40 +from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
41 +from vllm.model_executor.models.llava_next import (
42 + BaseLlavaNextMultiModalProcessor,
43 + LlavaNextImageEmbeddingInputs,
44 + LlavaNextImageInputs,
45 + LlavaNextImagePixelInputs,
46 + LlavaNextProcessingInfo,
47 +)
48 +from vllm.model_executor.models.module_mapping import MultiModelKeys
49 +from vllm.model_executor.models.siglip import SiglipVisionModel
50 +from vllm.model_executor.models.utils import (
51 + AutoWeightsLoader,
52 + PPMissingLayer,
53 + WeightsMapper,
54 + maybe_prefix,
55 +)
56 +from vllm.multimodal import MULTIMODAL_REGISTRY
57 +from vllm.multimodal.inputs import MultiModalFieldConfig
58 +from vllm.sequence import IntermediateTensors
59 +
60 +from .blip2 import Blip2QFormerModel
61 +
62 +logger = init_logger(__name__)
63 +
64 +
65 +# ---------------------------------------------------------------------------
66 +# Downsampler modules (translated from HF downsampling.py)
67 +# ---------------------------------------------------------------------------
68 +
69 +
70 +class InterpolateDownsampler:
71 + """Spatial downsampling via area interpolation."""
72 +
73 + def __init__(self, config, mode="area"):
74 + self.orig_image_side = (
75 + config.vision_config.image_size // config.vision_config.patch_size
76 + )
77 + self.new_image_side = int(
78 + self.orig_image_side * Fraction(config.downsample_rate)
79 + )
80 + self.mode = mode
81 +
82 + def __call__(self, image_features: torch.Tensor) -> torch.Tensor:
83 + batch_size, _, dim = image_features.size()
84 + up_shape = [batch_size, self.orig_image_side, self.orig_image_side, dim]
85 + large = image_features.view(up_shape).permute(0, 3, 1, 2)
86 + small = torch.nn.functional.interpolate(
87 + large,
88 + size=(self.new_image_side, self.new_image_side),
89 + mode=self.mode,
90 + )
91 + return small.permute(0, 2, 3, 1).flatten(1, 2)
92 +
93 +
94 +class SpatialOffsetDownsampler:
95 + """Sample one position from each 2x2 block (offset 0-3 = TL/TR/BL/BR)."""
96 +
97 + def __init__(self, config, offset: int = 0):
98 + self.orig_image_side = (
99 + config.vision_config.image_size // config.vision_config.patch_size
100 + )
101 + self.new_image_side = self.orig_image_side // 2
102 + offsets = [(0, 0), (0, 1), (1, 0), (1, 1)]
103 + self.offset_h, self.offset_w = offsets[offset]
104 +
105 + def __call__(self, image_features: torch.Tensor) -> torch.Tensor:
106 + B, _, C = image_features.shape
107 + features_2d = image_features.reshape(
108 + B, self.orig_image_side, self.orig_image_side, C
109 + )
110 + n = self.new_image_side
111 + blocks = features_2d.reshape(B, n, 2, n, 2, C)
112 + sampled = blocks[:, :, self.offset_h, :, self.offset_w, :]
113 + return sampled.reshape(B, -1, C)
114 +
115 +
116 +class WindowQFormerDownsampler(nn.Module):
117 + """Window-based QFormer downsampler (matches HF downsampling.py exactly)."""
118 +
119 + def __init__(
120 + self,
121 + config,
122 + quant_config: QuantizationConfig | None = None,
123 + cache_config: CacheConfig | None = None,
124 + spatial_offset: int | None = None,
125 + prefix: str = "",
126 + ):
127 + super().__init__()
128 + llm_hidden_size = config.text_config.hidden_size
129 + vision_hidden_size = config.vision_config.hidden_size
130 +
131 + self.dropout = nn.Dropout(config.projector_dropout)
132 +
133 + if spatial_offset is not None:
134 + self.downsampler = SpatialOffsetDownsampler(config, offset=spatial_offset)
135 + else:
136 + self.downsampler = InterpolateDownsampler(config)
137 +
138 + qformer_config = Blip2QFormerConfig(
139 + hidden_size=vision_hidden_size,
140 + num_attention_heads=vision_hidden_size // 64,
141 + intermediate_size=3072,
142 + num_hidden_layers=1,
143 + encoder_hidden_size=vision_hidden_size,
144 + cross_attention_frequency=1,
145 + max_position_embeddings=2048,
146 + use_qformer_text_input=False,
147 + )
148 + self.qformer = Blip2QFormerModel(
149 + qformer_config,
150 + quant_config=quant_config,
151 + cache_config=cache_config,
152 + prefix=f"{prefix}.qformer",
153 + )
154 +
155 + self.image_side = (
156 + config.vision_config.image_size // config.vision_config.patch_size
157 + )
158 + q, w = config.downsample_rate.split("/")
159 + self.query_side, self.window_side = int(q), int(w)
160 + self.query_length = self.query_side**2
161 +
162 + embed_std = 1 / math.sqrt(vision_hidden_size)
163 + self.norm = nn.LayerNorm(vision_hidden_size, eps=1e-6)
164 + self.query = nn.Parameter(
165 + torch.randn(1, self.query_length, vision_hidden_size) * embed_std
166 + )
167 + self.image_positions = nn.Parameter(
168 + torch.randn(1, self.window_side**2, vision_hidden_size) * embed_std
169 + )
170 + self.out_linear = nn.Linear(vision_hidden_size, llm_hidden_size, bias=True)
171 +
172 + def _win(self, x: torch.Tensor, side: int, win: int) -> torch.Tensor:
173 + """(B, side*side, C) → (B*n*n, win*win, C) where n=side//win."""
174 + B, _, C = x.shape
175 + n = side // win
176 + return (
177 + x.view(B, side, side, C)
178 + .view(B, n, win, n, win, C)
179 + .transpose(2, 3)
180 + .flatten(0, 2)
181 + .flatten(1, 2)
182 + )
183 +
184 + def _unwin(self, xw: torch.Tensor, n: int, win: int) -> torch.Tensor:
185 + """(B*n*n, win*win, C) → (B, (n*win)^2, C)."""
186 + Bnn, _, C = xw.shape
187 + B = Bnn // (n * n)
188 + side = n * win
189 + return (
190 + xw.view(B, n, n, win, win, C)
191 + .transpose(2, 3)
192 + .contiguous()
193 + .view(B, side, side, C)
194 + .flatten(1, 2)
195 + )
196 +
197 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
198 + B, HW, C = image_features.shape
199 + assert self.image_side * self.image_side == HW
200 + n = self.image_side // self.window_side
201 +
202 + image_features = self.norm(image_features)
203 + enc = self._win(image_features, self.image_side, self.window_side)
204 +
205 + downsampled = self.downsampler(image_features)
206 + new_side = n * self.query_side
207 + downsampled_w = self._win(downsampled, new_side, self.query_side)
208 +
209 + query_embeds = self.query + downsampled_w
210 + encoder_embeds = self.dropout(enc + self.image_positions)
211 + out_w = self.qformer(
212 + query_embeds=query_embeds,
213 + encoder_hidden_states=encoder_embeds,
214 + )
215 +
216 + out = self._unwin(out_w, n=n, win=self.query_side)
217 + out = self.dropout(out)
218 + return self.out_linear(out)
219 +
220 +
221 +# ---------------------------------------------------------------------------
222 +# LLM subclasses with deepstack injection in the layer loop
223 +# ---------------------------------------------------------------------------
224 +
225 +
226 +@support_torch_compile(
227 + dynamic_arg_dims={
228 + "input_ids": 0,
229 + "positions": 0,
230 + "intermediate_tensors": 0,
231 + "inputs_embeds": 0,
232 + "deepstack_input_embeds": 0,
233 + }
234 +)
235 +class Granite4VisionLLMModel(GraniteModel):
236 + """GraniteModel with deepstack feature injection in the layer loop."""
237 +
238 + def forward(
239 + self,
240 + input_ids: torch.Tensor | None,
241 + positions: torch.Tensor,
242 + intermediate_tensors: IntermediateTensors | None = None,
243 + inputs_embeds: torch.Tensor | None = None,
244 + deepstack_input_embeds: IntermediateTensors | None = None,
245 + ) -> torch.Tensor | IntermediateTensors:
246 + if get_pp_group().is_first_rank:
247 + if inputs_embeds is not None:
248 + hidden_states = inputs_embeds
249 + else:
250 + hidden_states = self.embed_input_ids(input_ids)
251 + hidden_states = hidden_states * self.config.embedding_multiplier
252 + else:
253 + assert intermediate_tensors is not None
254 + hidden_states = intermediate_tensors["hidden_states"]
255 + # Recover deepstack features forwarded from the previous PP rank.
256 + if deepstack_input_embeds is None:
257 + ds_keys = [
258 + k for k in intermediate_tensors.tensors if k.startswith("ds_")
259 + ]
260 + if ds_keys:
261 + deepstack_input_embeds = IntermediateTensors(
262 + {k: intermediate_tensors[k] for k in ds_keys}
263 + )
264 +
265 + for layer_idx, layer in islice(
266 + enumerate(self.layers), self.start_layer, self.end_layer
267 + ):
268 + if deepstack_input_embeds is not None:
269 + key = f"ds_{layer_idx}"
270 + if key in deepstack_input_embeds.tensors:
271 + feat = deepstack_input_embeds[key]
272 + # Resize to match hidden_states in case of CUDA graph padding
273 + num_tokens = hidden_states.size(0)
274 + buf_len = feat.shape[0]
275 + if buf_len != num_tokens:
276 + feat = torch.nn.functional.pad(
277 + feat[:num_tokens],
278 + (0, 0, 0, max(0, num_tokens - buf_len)),
279 + )
280 + hidden_states = hidden_states + feat
281 + hidden_states = layer(positions, hidden_states)
282 +
283 + if not get_pp_group().is_last_rank:
284 + # Forward hidden_states and any deepstack features for later ranks.
285 + it = {"hidden_states": hidden_states}
286 + if deepstack_input_embeds is not None:
287 + remaining = {
288 + k: v
289 + for k, v in deepstack_input_embeds.tensors.items()
290 + if int(k.split("_")[1]) >= self.end_layer
291 + }
292 + it.update(remaining)
293 + return IntermediateTensors(it)
294 +
295 + hidden_states = self.norm(hidden_states)
296 + return hidden_states
297 +
298 +
299 +class Granite4VisionLLMForCausalLM(GraniteForCausalLM):
300 + """GraniteForCausalLM backed by Granite4VisionLLMModel."""
301 +
302 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
303 + nn.Module.__init__(self)
304 + config = vllm_config.model_config.hf_config
305 + quant_config = vllm_config.quant_config
306 + self.config = config
307 + self.quant_config = quant_config
308 + self.model = Granite4VisionLLMModel(
309 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
310 + )
311 + if get_pp_group().is_last_rank:
312 + self.lm_head = ParallelLMHead(
313 + config.vocab_size,
314 + config.hidden_size,
315 + quant_config=quant_config,
316 + prefix=maybe_prefix(prefix, "lm_head"),
317 + )
318 + if config.tie_word_embeddings:
319 + self.lm_head.weight = self.model.embed_tokens.weight
320 + logit_scale = getattr(config, "logit_scale", 1.0)
321 + if hasattr(config, "logits_scaling"):
322 + logit_scale /= config.logits_scaling
323 + self.logits_processor = LogitsProcessor(
324 + config.vocab_size, scale=logit_scale
325 + )
326 + else:
327 + self.lm_head = PPMissingLayer()
328 +
329 + def make_empty_intermediate_tensors(
330 + self, batch_size: int, dtype: torch.dtype, device: torch.device
331 + ) -> IntermediateTensors:
332 + tensors = super().make_empty_intermediate_tensors(batch_size, dtype, device)
333 + # Include deepstack buffers so non-first PP ranks receive them.
334 + # _ds_layer_indices is set directly on this instance by the outer model.
335 + for llm_layer in getattr(self, "_ds_layer_indices", []):
336 + tensors.tensors[f"ds_{llm_layer}"] = torch.zeros(
337 + (batch_size, self.config.hidden_size), dtype=dtype, device=device
338 + )
339 + return tensors
340 +
341 +
342 +# ---------------------------------------------------------------------------
343 +# Processing info / processor (reuses LlavaNext patterns)
344 +# ---------------------------------------------------------------------------
345 +
346 +
347 +class Granite4VisionProcessingInfo(LlavaNextProcessingInfo):
348 + def get_hf_config(self):
349 + return self.ctx.get_hf_config()
350 +
351 + def get_hf_processor(self, **kwargs):
352 + return self.ctx.get_hf_processor(**kwargs)
353 +
354 + def get_num_image_tokens(
355 + self,
356 + *,
357 + image_width: int,
358 + image_height: int,
359 + ) -> int:
360 + hf_config = self.get_hf_config()
361 + vision_encoder_info = self.get_vision_encoder_info()
362 +
363 + # After QFormer downsampling, patch grid is scaled by downsample_rate
364 + ds_rate = Fraction(hf_config.downsample_rate)
365 + patch_grid = vision_encoder_info.get_patch_grid_length() # 24 for 384/16
366 + downsampled_grid = int(patch_grid * ds_rate) # 12 for rate 4/8
367 +
368 + # Base feature: downsampled_grid^2
369 + base_feature_size = downsampled_grid * downsampled_grid
370 +
371 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
372 + image_size=(image_height, image_width),
373 + grid_pinpoints=hf_config.image_grid_pinpoints,
374 + patch_size=vision_encoder_info.get_image_size(),
375 + )
376 +
377 + (
378 + unpadded_feature_size,
379 + newline_feature_size,
380 + ) = self._get_num_unpadded_features(
381 + original_height=image_height,
382 + original_width=image_width,
383 + npatches=downsampled_grid,
384 + num_patch_height=num_patch_height,
385 + num_patch_width=num_patch_width,
386 + )
387 +
388 + return unpadded_feature_size + newline_feature_size + base_feature_size
389 +
390 +
391 +class Granite4VisionMultiModalProcessor(
392 + BaseLlavaNextMultiModalProcessor[Granite4VisionProcessingInfo]
393 +):
394 + def _get_mm_fields_config(
395 + self,
396 + hf_inputs: BatchFeature,
397 + hf_processor_mm_kwargs: Mapping[str, object],
398 + ) -> Mapping[str, MultiModalFieldConfig]:
399 + return dict(
400 + pixel_values=MultiModalFieldConfig.batched("image"),
401 + image_sizes=MultiModalFieldConfig.batched("image"),
402 + )
403 +
404 +
405 +# ---------------------------------------------------------------------------
406 +# Top-level model
407 +# ---------------------------------------------------------------------------
408 +
409 +
410 +@MULTIMODAL_REGISTRY.register_processor(
411 + Granite4VisionMultiModalProcessor,
412 + info=Granite4VisionProcessingInfo,
413 + dummy_inputs=LlavaDummyInputsBuilder,
414 +)
415 +class Granite4VisionForConditionalGeneration(
416 + nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
417 +):
418 + """vLLM implementation of Granite 4 Vision.
419 +
420 + Architecture:
421 + - SigLIP vision tower -> WindowQFormerDownsampler projectors
422 + - Deepstack: 4 vision layers projected and injected at 4 LLM layers
423 + - Spatial: 4 offset groups from last vision layer injected at 4 more LLM layers
424 + - Granite language backbone with embedding_multiplier
425 + - logits_scaling via LogitsProcessor
426 +
427 + The outer model runs the LLM layer loop directly (like HF does) to inject
428 + deepstack features. This avoids wrapping the inner model and keeps weight
429 + loading simple.
430 +
431 + LoRA support:
432 + - Full merge: --hf-overrides '{"adapter_path": "path/to/lora"}' merges
433 + LM-only LoRA deltas at load time (W += scaling * B @ A).
434 + - Native LoRA: --enable-lora --default-mm-loras '{"image": "path/to/lora"}'
435 + lets vLLM runtime serve LM LoRA per-request.
436 + Both modes expect a LM-only adapter (no modules_to_save).
437 + """
438 +
439 + # LoRA class attributes (matches GraniteForCausalLM)
440 + packed_modules_mapping = {
441 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
442 + "gate_up_proj": ["gate_proj", "up_proj"],
443 + }
444 + embedding_modules = {}
445 +
446 + # Weight mapping: HF checkpoint -> vLLM parameter names
447 + # HF: model.language_model.layers.0...
448 + # vLLM: language_model.model.layers.0...
449 + # (because GraniteForCausalLM.model = GraniteModel)
450 + hf_to_vllm_mapper = WeightsMapper(
451 + orig_to_new_prefix={
452 + "model.language_model.": "language_model.model.",
453 + "model.layerwise_projectors.": "layerwise_projectors.",
454 + "model.spatial_projectors.": "spatial_projectors.",
455 + "model.image_newline": "image_newline",
456 + "model.vision_tower.": "vision_tower.",
457 + "lm_head.": "language_model.lm_head.",
458 + }
459 + )
460 +
461 + @classmethod
462 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
463 + if modality.startswith("image"):
464 + return "<image>"
465 + raise ValueError(f"Only image modality is supported, got {modality}")
466 +
467 + def get_mm_mapping(self) -> MultiModelKeys:
468 + return MultiModelKeys.from_string_field(
469 + language_model="language_model",
470 + connector=["layerwise_projectors", "spatial_projectors"],
471 + tower_model="vision_tower",
472 + )
473 +
474 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
475 + super().__init__()
476 +
477 + config = vllm_config.model_config.hf_config
478 + quant_config = vllm_config.quant_config
479 + self.config = config
480 + self.vllm_config = vllm_config
481 +
482 + # ----- Vision tower + projectors (marked as tower) -----
483 + with self._mark_tower_model(vllm_config, "image"):
484 + # Do NOT use init_vision_tower_for_llava here — it truncates the
485 + # encoder to vision_feature_layer depth. Deepstack needs ALL hidden
486 + # states (deepstack_layer_map uses negative indices into the full
487 + # encoder output list).
488 + self.vision_tower = SiglipVisionModel(
489 + config.vision_config,
490 + quant_config=quant_config,
491 + require_post_norm=False,
492 + prefix=maybe_prefix(prefix, "vision_tower"),
493 + )
494 +
495 + # image_newline parameter
496 + if config.use_image_newline_parameter:
497 + self.image_newline = nn.Parameter(
498 + torch.empty(config.text_config.hidden_size)
499 + )
500 + else:
501 + self.image_newline = None
502 +
503 + cache_config = vllm_config.cache_config
504 +
505 + # Deepstack projectors: one per (vision_layer, llm_layer) pair
506 + self.layerwise_projectors = nn.ModuleList(
507 + [
508 + WindowQFormerDownsampler(
509 + config,
510 + quant_config=quant_config,
511 + cache_config=cache_config,
512 + prefix=maybe_prefix(prefix, f"layerwise_projectors.{i}"),
513 + )
514 + for i in range(len(config.deepstack_layer_map))
515 + ]
516 + )
517 +
518 + # Spatial projectors: 4 offset groups
519 + self.spatial_projectors = None
520 + if config.use_spatial_sampling:
521 + self.spatial_projectors = nn.ModuleList(
522 + [
523 + WindowQFormerDownsampler(
524 + config,
525 + quant_config=quant_config,
526 + cache_config=cache_config,
527 + spatial_offset=i,
528 + prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
529 + )
530 + for i in range(4)
531 + ]
532 + )
533 +
534 + # ----- Language model (marked as LM) -----
535 + with self._mark_language_model(vllm_config):
536 + self.language_model = Granite4VisionLLMForCausalLM(
537 + vllm_config=vllm_config.with_hf_config(config.text_config),
538 + prefix=maybe_prefix(prefix, "language_model"),
539 + )
540 +
541 + self.make_empty_intermediate_tensors = (
542 + self.language_model.make_empty_intermediate_tensors
543 + )
544 +
545 + # Store config values we need
546 + self._deepstack_layer_map = config.deepstack_layer_map # [[-19, 9], ...]
547 + self._use_spatial_sampling = getattr(config, "use_spatial_sampling", False)
548 + self._spatial_vision_layer = getattr(config, "spatial_vision_layer", -1)
549 + self._spatial_target_layers = getattr(config, "spatial_target_layers", [])
550 + self._vision_feature_select_strategy = getattr(
551 + config, "vision_feature_select_strategy", "full"
552 + )
553 + self._downsample_rate = Fraction(config.downsample_rate)
554 +
555 + # Ordered list of LLM layer indices for each deepstack level.
556 + # Pre-populated from config so it's available during CUDA graph capture
557 + # (before any embed_multimodal call).
558 + self._ds_layer_indices: list[int] = [
559 + llm_layer for _, llm_layer in config.deepstack_layer_map
560 + ] + list(getattr(config, "spatial_target_layers", []))
561 +
562 + # Share ds_layer_indices with the LLM causal model so
563 + # make_empty_intermediate_tensors includes the correct keys
564 + # (its self.config is text_config, no deepstack_layer_map).
565 + self.language_model._ds_layer_indices = self._ds_layer_indices
566 +
567 + # Pre-allocated persistent GPU buffers for deepstack features.
568 + # Written via .copy_() in embed_input_ids(), read by forward() via a
569 + # slice. Because the buffer address is fixed, CUDA graph replay sees
570 + # the updated values written just before each prefill.
571 + # Shape: (max_num_batched_tokens, lm_hidden_size) per level.
572 + n_layerwise = len(config.deepstack_layer_map)
573 + n_spatial = len(getattr(config, "spatial_target_layers", []))
574 + num_ds_levels = n_layerwise + n_spatial
575 + lm_hidden = config.text_config.hidden_size
576 + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
577 + # Allocated on CPU first; moved to GPU in embed_input_ids on first use.
578 + self._ds_buffers: list[torch.Tensor] = [
579 + torch.zeros(max_tokens, lm_hidden) for _ in range(num_ds_levels)
580 + ]
581 + self._ds_num_tokens: int = 0 # tokens written in last embed_input_ids call
582 +
583 + # ----- Vision feature extraction -----
584 +
585 + def _get_vision_hidden_states(
586 + self, pixel_values: torch.Tensor
587 + ) -> list[torch.Tensor]:
588 + """Run vision tower and return all hidden states (including input embeddings).
589 +
590 + Uses SiglipEncoder's built-in return_all_hidden_states support.
591 + Returns list[Tensor] where index 0 = embeddings, index i = after layer i-1.
592 + """
593 + vt = self.vision_tower
594 + vm = vt.vision_model if hasattr(vt, "vision_model") else vt
595 +
596 + hidden_states = vm.embeddings(pixel_values)
597 + all_hidden_states = vm.encoder(
598 + inputs_embeds=hidden_states,
599 + return_all_hidden_states=True,
600 + )
601 + return all_hidden_states
602 +
603 + def _pack_and_unpad_image_features(
604 + self,
605 + image_features: list[torch.Tensor] | tuple[torch.Tensor, ...],
606 + image_sizes: torch.Tensor,
607 + ) -> list[torch.Tensor]:
608 + """Reshape, unpad, and pack image features.
609 +
610 + Matches HF Granite4VisionModel.pack_and_unpad_image_features exactly.
611 + """
612 + config = self.config
613 + ds_rate = self._downsample_rate
614 + new_image_features = []
615 +
616 + for image_idx, image_feature in enumerate(image_features):
617 + if image_feature.shape[0] > 1:
618 + # Multi-patch: first is base, rest are high-res
619 + base_image_feature = image_feature[0]
620 + image_feature = image_feature[1:]
621 +
622 + height = width = (
623 + config.vision_config.image_size // config.vision_config.patch_size
624 + )
625 + # After QFormer downsampling
626 + height = int(height * ds_rate)
627 + width = int(width * ds_rate)
628 +
629 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
630 + image_sizes[image_idx],
631 + config.image_grid_pinpoints,
632 + config.vision_config.image_size,
633 + )
634 +
635 + image_feature = image_feature.view(
636 + num_patch_height, num_patch_width, height, width, -1
637 + )
638 + image_feature = (
639 + image_feature.permute(4, 0, 2, 1, 3)
640 + .contiguous()
641 + .flatten(1, 2)
642 + .flatten(2, 3)
643 + )
644 + image_feature = unpad_image(image_feature, image_sizes[image_idx])
645 +
646 + if self.image_newline is not None:
647 + image_feature = torch.cat(
648 + (
649 + image_feature,
650 + self.image_newline[:, None, None]
651 + .expand(*image_feature.shape[:-1], 1)
652 + .to(image_feature.device, image_feature.dtype),
653 + ),
654 + dim=-1,
655 + )
656 +
657 + image_feature = image_feature.flatten(1, 2).transpose(0, 1)
658 + image_feature = torch.cat((base_image_feature, image_feature), dim=0)
659 + else:
660 + image_feature = image_feature[0]
661 + if self.image_newline is not None:
662 + image_feature = torch.cat(
663 + (image_feature, self.image_newline[None].to(image_feature)),
664 + dim=0,
665 + )
666 +
667 + new_image_features.append(image_feature)
668 +
669 + return new_image_features
670 +
671 + def _get_all_layer_features(
672 + self,
673 + pixel_values: torch.Tensor,
674 + image_sizes: torch.Tensor,
675 + ) -> tuple[list[int], list[torch.Tensor]]:
676 + """Extract deepstack + spatial features for all levels.
677 +
678 + Returns:
679 + llm_layer_indices: ordered list of target LLM layer indices
680 + per_image_packed: one tensor per image, shape
681 + (num_tokens_i, lm_hidden_size * num_levels),
682 + all levels packed on dim=-1.
683 +
684 + Packing on dim=-1 means the framework's token-level slicing for
685 + chunked prefill preserves all levels intact.
686 + """
687 + select_strategy = self._vision_feature_select_strategy
688 +
689 + image_num_patches = [
690 + image_size_to_num_patches(
691 + image_size=imsize,
692 + grid_pinpoints=self.config.image_grid_pinpoints,
693 + patch_size=self.config.vision_config.image_size,
694 + )
695 + for imsize in image_sizes
696 + ]
697 +
698 + if pixel_values.dim() == 5:
699 + pixel_values = torch.cat(
700 + [pv[:np_] for pv, np_ in zip(pixel_values, image_num_patches)],
701 + dim=0,
702 + )
703 +
704 + all_hidden_states = self._get_vision_hidden_states(pixel_values)
705 +
706 + # Collect per-level: (llm_layer, [per_image_tensor, ...])
707 + levels: list[tuple[int, list[torch.Tensor]]] = []
708 +
709 + for proj_idx, (vision_layer, llm_layer) in enumerate(self._deepstack_layer_map):
710 + selected = all_hidden_states[vision_layer]
711 + if select_strategy == "default":
712 + selected = selected[:, 1:]
713 + projected = self.layerwise_projectors[proj_idx](selected)
714 + per_image = self._pack_and_unpad_image_features(
715 + torch.split(projected, image_num_patches, dim=0), image_sizes
716 + )
717 + levels.append((llm_layer, per_image))
718 +
719 + if self._use_spatial_sampling and self.spatial_projectors is not None:
720 + spatial_hidden = all_hidden_states[self._spatial_vision_layer]
721 + if select_strategy == "default":
722 + spatial_hidden = spatial_hidden[:, 1:]
723 + for group_idx, llm_layer in enumerate(self._spatial_target_layers):
724 + projected = self.spatial_projectors[group_idx](spatial_hidden)
725 + per_image = self._pack_and_unpad_image_features(
726 + torch.split(projected, image_num_patches, dim=0), image_sizes
727 + )
728 + levels.append((llm_layer, per_image))
729 +
730 + llm_layer_indices = [llm_layer for llm_layer, _ in levels]
731 + num_images = len(image_sizes)
732 + per_image_packed = [
733 + torch.cat([levels[lvl][1][img] for lvl in range(len(levels))], dim=-1)
734 + for img in range(num_images)
735 + ]
736 +
737 + return llm_layer_indices, per_image_packed
738 +
739 + # ----- Multimodal interface -----
740 +
741 + def _parse_and_validate_image_input(
742 + self, **kwargs: object
743 + ) -> LlavaNextImageInputs | None:
744 + pixel_values = kwargs.pop("pixel_values", None)
745 + image_sizes = kwargs.pop("image_sizes", None)
746 + image_embeds = kwargs.pop("image_embeds", None)
747 +
748 + if pixel_values is None and image_embeds is None:
749 + return None
750 +
751 + if pixel_values is not None:
752 + expected_h = expected_w = self.config.vision_config.image_size
753 + return LlavaNextImagePixelInputs(
754 + type="pixel_values",
755 + pixel_values=pixel_values,
756 + image_sizes=image_sizes,
757 + resolve_bindings={"h": expected_h, "w": expected_w},
758 + )
759 +
760 + if image_embeds is not None:
761 + return LlavaNextImageEmbeddingInputs(
762 + type="image_embeds",
763 + data=image_embeds,
764 + )
765 +
766 + raise AssertionError("Unreachable")
767 +
768 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
769 + """Run vision tower and return per-image packed feature tensors.
770 +
771 + Each returned tensor has shape (num_tokens_i, lm_hidden_size * num_levels)
772 + with all deepstack levels packed on dim=-1. The framework caches these
773 + tensors and slices along dim=0 for chunked prefill — all levels survive
774 + intact because slicing is token-wise, not feature-wise.
775 +
776 + embed_input_ids() splits the packed tensor back into per-level buffers.
777 + """
778 + image_input = self._parse_and_validate_image_input(**kwargs)
779 + if image_input is None:
780 + return []
781 +
782 + if image_input["type"] == "image_embeds":
783 + return [image_input["data"]]
784 +
785 + pixel_values = image_input["pixel_values"]
786 + image_sizes = image_input.get("image_sizes")
787 +
788 + if isinstance(pixel_values, list):
789 + pixel_values = torch.cat(pixel_values, dim=0)
790 +
791 + llm_layer_indices, per_image_packed = self._get_all_layer_features(
792 + pixel_values, image_sizes
793 + )
794 + self._ds_layer_indices = llm_layer_indices
795 + return per_image_packed
796 +
797 + def embed_input_ids(
798 + self,
799 + input_ids: torch.Tensor,
800 + multimodal_embeddings: MultiModalEmbeddings | None = None,
801 + *,
802 + is_multimodal: torch.Tensor | None = None,
803 + handle_oov_mm_token: bool = True,
804 + ) -> torch.Tensor:
805 + """Merge text and vision embeddings, apply embedding_multiplier.
806 +
807 + HF flow:
808 + 1. inputs_embeds = embed_tokens(input_ids)
809 + 2. inputs_embeds.masked_fill(vision_mask, 0.0)
810 + 3. hidden_states = inputs_embeds * embedding_multiplier
811 + 4. layer loop injects deepstack features at target layers
812 +
813 + multimodal_embeddings contains packed tensors from embed_multimodal():
814 + shape (num_tokens_i, lm_hidden_size * num_levels). We split on dim=-1
815 + to get per-level features, build batch-sized buffers (zero at text
816 + positions), and store in self._ds_features for forward().
817 + """
818 + lm_inner = self.language_model.model
819 +
820 + has_vision = (
821 + multimodal_embeddings is not None
822 + and is_multimodal is not None
823 + and len(multimodal_embeddings) > 0
824 + and is_multimodal.any()
825 + )
826 +
827 + if not has_vision:
828 + self._ds_num_tokens = 0
829 + embeds = lm_inner.embed_input_ids(input_ids)
830 + return embeds * lm_inner.config.embedding_multiplier
831 +
832 + # 1. Text embeddings
833 + text_embeds = lm_inner.embed_input_ids(input_ids)
834 +
835 + # 2. Zero image positions (matches HF masked_fill(vision_mask, 0.0))
836 + text_embeds[is_multimodal] = 0.0
837 +
838 + # 3. Apply embedding_multiplier
839 + inputs_embeds = text_embeds * lm_inner.config.embedding_multiplier
840 +
841 + # 4. Split packed tensors into per-level features and build buffers.
842 + # multimodal_embeddings is a list of per-image packed tensors
843 + # (possibly a chunk slice from the framework's encoder cache).
844 + # Concatenate along token dim → (total_mm_tokens, lm_h * num_levels).
845 + N, lm_h = inputs_embeds.shape
846 + all_packed = torch.cat(
847 + [t.to(dtype=inputs_embeds.dtype) for t in multimodal_embeddings],
848 + dim=0,
849 + )
850 + level_features = all_packed.split(lm_h, dim=-1) # num_levels tensors
851 +
852 + # Ensure persistent buffers are on the right device/dtype (first call).
853 + buf0 = self._ds_buffers[0]
854 + if buf0.device != inputs_embeds.device or buf0.dtype != inputs_embeds.dtype:
855 + self._ds_buffers = [
856 + b.to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
857 + for b in self._ds_buffers
858 + ]
859 +
860 + for level_idx in range(len(self._ds_layer_indices)):
861 + target = self._ds_buffers[level_idx][:N]
862 + target.zero_()
863 + target[is_multimodal] = level_features[level_idx]
864 +
865 + self._ds_num_tokens = N
866 + return inputs_embeds
867 +
868 + # ----- Forward -----
869 +
870 + def forward(
871 + self,
872 + input_ids: torch.Tensor,
873 + positions: torch.Tensor,
874 + intermediate_tensors: IntermediateTensors | None = None,
875 + inputs_embeds: torch.Tensor | None = None,
876 + **kwargs: object,
877 + ) -> torch.Tensor | IntermediateTensors:
878 + if intermediate_tensors is not None:
879 + inputs_embeds = None
880 +
881 + # Build IntermediateTensors from pre-allocated persistent buffers.
882 + # Always pass deepstack when inputs_embeds is non-None (prefill path),
883 + # including during CUDA graph capture (buffers are zero → no-op injection).
884 + # This ensures the graph captures the injection code path.
885 + if (
886 + inputs_embeds is not None
887 + and get_pp_group().is_first_rank
888 + and self._ds_layer_indices
889 + ):
890 + ds: IntermediateTensors | None = IntermediateTensors(
891 + {
892 + f"ds_{llm_layer}": self._ds_buffers[lvl]
893 + for lvl, llm_layer in enumerate(self._ds_layer_indices)
894 + }
895 + )
896 + else:
897 + ds = None
898 +
899 + hidden_states = self.language_model.model(
900 + input_ids=input_ids,
901 + positions=positions,
902 + intermediate_tensors=intermediate_tensors,
903 + inputs_embeds=inputs_embeds,
904 + deepstack_input_embeds=ds,
905 + )
906 +
907 + # Clear buffers after use so stale features don't leak into the next request.
908 + if (
909 + inputs_embeds is not None
910 + and get_pp_group().is_first_rank
911 + and self._ds_num_tokens > 0
912 + ):
913 + n = self._ds_num_tokens
914 + for buf in self._ds_buffers:
915 + buf[:n].zero_()
916 + self._ds_num_tokens = 0
917 +
918 + return hidden_states
919 +
920 + def compute_logits(
921 + self,
922 + hidden_states: torch.Tensor,
923 + ) -> torch.Tensor | None:
924 + # GraniteForCausalLM.compute_logits uses
925 + # LogitsProcessor(scale=1/logits_scaling)
926 + return self.language_model.compute_logits(hidden_states)
927 +
928 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
929 + loader = AutoWeightsLoader(self)
930 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/registry.py
+4
@@ -407,6 +407,10 @@ _MULTIMODAL_MODELS = {
407 "granite_speech",
408 "GraniteSpeechForConditionalGeneration",
409 ),
410 + "Granite4VisionForConditionalGeneration": (
411 + "granite4_vision",
412 + "Granite4VisionForConditionalGeneration",
413 + ),
414 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
415 "HunYuanVLForConditionalGeneration": (
416 "hunyuan_vision",
vllm/transformers_utils/config.py
+1
@@ -92,6 +92,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
92 flex_olmo="FlexOlmoConfig",
93 fireredlid="FireRedLIDConfig",
94 funaudiochat="FunAudioChatConfig",
95 + granite4_vision="Granite4VisionConfig",
96 hunyuan_vl="HunYuanVLConfig",
97 isaac="IsaacConfig",
98 kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
vllm/transformers_utils/configs/__init__.py
+2
@@ -32,6 +32,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
32 "FlexOlmoConfig": "vllm.transformers_utils.configs.flex_olmo",
33 "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
34 "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
35 + "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
36 "HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
37 "HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
38 "HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
@@ -92,6 +93,7 @@ __all__ = [
93 "FireRedLIDConfig",
94 "FunAudioChatConfig",
95 "FunAudioChatAudioEncoderConfig",
96 + "Granite4VisionConfig",
97 "HunYuanVLConfig",
98 "HunYuanVLTextConfig",
99 "HunYuanVLVisionConfig",
vllm/transformers_utils/configs/granite4_vision.py new
+94
@@ -0,0 +1,94 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from typing import Any
5 +
6 +import transformers
7 +
8 +
9 +class Granite4VisionConfig(transformers.PretrainedConfig):
10 + """Configuration for Granite 4 Vision model.
11 +
12 + This config is needed because the granite4_vision model type is not yet
13 + in the transformers version pinned by vLLM. Once transformers adds native
14 + support, this file can be removed and the _CONFIG_REGISTRY entry dropped.
15 + """
16 +
17 + model_type = "granite4_vision"
18 + is_composition = False
19 +
20 + def __init__(
21 + self,
22 + vision_config: dict[str, Any] | None = None,
23 + text_config: dict[str, Any] | None = None,
24 + image_token_index: int = 100352,
25 + image_seq_length: int = 576,
26 + image_grid_pinpoints: list[list[int]] | None = None,
27 + vision_feature_select_strategy: str = "full",
28 + vision_feature_layer: int | list[int] = -2,
29 + projector_hidden_act: str = "gelu",
30 + projector_dropout: float = 0.1,
31 + downsample_rate: str | None = None,
32 + use_image_newline_parameter: bool = True,
33 + deepstack_layer_map: list[list[int]] | None = None,
34 + use_spatial_sampling: bool = False,
35 + spatial_stride: int = 2,
36 + spatial_vision_layer: int = -1,
37 + spatial_target_layers: list[int] | None = None,
38 + # Hub aliases — base model config uses different field names
39 + vision_layer_to_llm_layer: list[list[int]] | None = None,
40 + use_checkerboard_sampling: bool | None = None,
41 + checkerboard_stride: int | None = None,
42 + checkerboard_vision_layer: int | None = None,
43 + checkerboard_llm_layers: list[int] | None = None,
44 + **kwargs: Any,
45 + ):
46 + self.image_token_index = image_token_index
47 + self.image_seq_length = image_seq_length
48 + self.image_grid_pinpoints = image_grid_pinpoints or []
49 + self.vision_feature_select_strategy = vision_feature_select_strategy
50 + self.vision_feature_layer = vision_feature_layer
51 + self.projector_hidden_act = projector_hidden_act
52 + self.projector_dropout = projector_dropout
53 + self.downsample_rate = downsample_rate
54 + self.use_image_newline_parameter = use_image_newline_parameter
55 + self.deepstack_layer_map = deepstack_layer_map or vision_layer_to_llm_layer
56 + self.use_spatial_sampling = (
57 + use_spatial_sampling
58 + if use_checkerboard_sampling is None
59 + else use_checkerboard_sampling
60 + )
61 + self.spatial_stride = (
62 + spatial_stride if checkerboard_stride is None else checkerboard_stride
63 + )
64 + self.spatial_vision_layer = (
65 + spatial_vision_layer
66 + if checkerboard_vision_layer is None
67 + else checkerboard_vision_layer
68 + )
69 + self.spatial_target_layers = (
70 + spatial_target_layers or checkerboard_llm_layers or [0, 10, 20, 30]
71 + )
72 +
73 + if vision_config is None:
74 + vision_config = {}
75 + if text_config is None:
76 + text_config = {}
77 +
78 + vision_model_type = vision_config.get("model_type", "siglip_vision_model")
79 + if vision_model_type in transformers.CONFIG_MAPPING:
80 + self.vision_config = transformers.CONFIG_MAPPING[vision_model_type](
81 + **vision_config
82 + )
83 + else:
84 + self.vision_config = transformers.PretrainedConfig(**vision_config)
85 +
86 + text_model_type = text_config.get("model_type", "granite")
87 + if text_model_type in transformers.CONFIG_MAPPING:
88 + self.text_config = transformers.CONFIG_MAPPING[text_model_type](
89 + **text_config
90 + )
91 + else:
92 + self.text_config = transformers.PretrainedConfig(**text_config)
93 +
94 + super().__init__(**kwargs)
vllm/transformers_utils/processors/__init__.py
+2
@@ -19,6 +19,7 @@ __all__ = [
19 "FireRedLIDProcessor",
20 "FunASRProcessor",
21 "GLM4VProcessor",
22 + "Granite4VisionProcessor",
23 "H2OVLProcessor",
24 "HunYuanVLProcessor",
25 "HunYuanVLImageProcessor",
@@ -48,6 +49,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
49 "FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
50 "FunASRProcessor": "vllm.transformers_utils.processors.funasr",
51 "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
52 + "Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
53 "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
54 "HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
55 "HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
vllm/transformers_utils/processors/granite4_vision.py new
+78
@@ -0,0 +1,78 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from fractions import Fraction
5 +
6 +from transformers import LlavaNextProcessor
7 +from transformers.image_processing_utils import select_best_resolution
8 +
9 +
10 +class Granite4VisionProcessor(LlavaNextProcessor):
11 + """Processor for Granite 4 Vision.
12 +
13 + Extends LlavaNextProcessor to account for the Window Q-Former
14 + downsampling when computing the number of image features.
15 +
16 + This processor is needed because the granite4_vision processor type
17 + is not yet in the transformers version pinned by vLLM.
18 + """
19 +
20 + model_type = "granite4_vision"
21 +
22 + def __init__(
23 + self,
24 + image_processor=None,
25 + tokenizer=None,
26 + patch_size=None,
27 + vision_feature_select_strategy=None,
28 + chat_template=None,
29 + image_token="<image>",
30 + num_additional_image_tokens=0,
31 + downsample_rate=None,
32 + **kwargs,
33 + ):
34 + super().__init__(
35 + image_processor=image_processor,
36 + tokenizer=tokenizer,
37 + patch_size=patch_size,
38 + vision_feature_select_strategy=vision_feature_select_strategy,
39 + chat_template=chat_template,
40 + image_token=image_token,
41 + num_additional_image_tokens=num_additional_image_tokens,
42 + )
43 + self.downsample_rate = downsample_rate
44 +
45 + def _get_number_of_features(
46 + self,
47 + orig_height: int,
48 + orig_width: int,
49 + height: int,
50 + width: int,
51 + ) -> int:
52 + image_grid_pinpoints = self.image_processor.image_grid_pinpoints
53 +
54 + height_best_resolution, width_best_resolution = select_best_resolution(
55 + [orig_height, orig_width], image_grid_pinpoints
56 + )
57 + scale_height = height_best_resolution // height
58 + scale_width = width_best_resolution // width
59 +
60 + patches_height = height // self.patch_size
61 + patches_width = width // self.patch_size
62 + if self.downsample_rate is not None:
63 + ds_rate = Fraction(self.downsample_rate)
64 + patches_height = int(patches_height * ds_rate)
65 + patches_width = int(patches_width * ds_rate)
66 +
67 + unpadded_features, newline_features = self._get_unpadded_features(
68 + orig_height,
69 + orig_width,
70 + patches_height,
71 + patches_width,
72 + scale_height,
73 + scale_width,
74 + )
75 + base_features = (
76 + patches_height * patches_width + self.num_additional_image_tokens
77 + )
78 + return unpadded_features + newline_features + base_features