@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 38327cf454
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 16:00 UTC

[Model] Aya Vision (#15441)

Signed-off-by: Jennifer Zhao <ai.jenniferzhao@gmail.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Jennifer Zhao committed Apr 1, 2025 at 09:30 UTC 38327cf454ab44bf55790abd6fe25e176bea5d82
10 files changed +617 -2
docs/source/models/supported_models.md
+7
@@ -753,6 +753,13 @@ See [this page](#generative-models) for more information on how to use generativ
753 *
754 * ✅︎
755 * ✅︎
756 +- * `AyaVisionForConditionalGeneration`
757 + * Aya Vision
758 + * T + I<sup>+</sup>
759 + * `CohereForAI/aya-vision-8b`, `CohereForAI/aya-vision-32b`, etc.
760 + *
761 + * ✅︎
762 + * ✅︎
763 - * `Blip2ForConditionalGeneration`
764 * BLIP-2
765 * T + I<sup>E</sup>
examples/offline_inference/vision_language.py
+23
@@ -60,6 +60,28 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:
60 )
61
62
63 +# Aya Vision
64 +def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
65 + assert modality == "image"
66 + model_name = "CohereForAI/aya-vision-8b"
67 +
68 + engine_args = EngineArgs(
69 + model=model_name,
70 + max_model_len=2048,
71 + max_num_seqs=2,
72 + mm_processor_kwargs={"crop_to_patches": True},
73 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
74 + )
75 + prompts = [
76 + f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
77 + for question in questions
78 + ]
79 + return ModelRequestData(
80 + engine_args=engine_args,
81 + prompts=prompts,
82 + )
83 +
84 +
85 # BLIP-2
86 def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
87 assert modality == "image"
@@ -865,6 +887,7 @@ def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
887
888 model_example_map = {
889 "aria": run_aria,
890 + "aya_vision": run_aya_vision,
891 "blip-2": run_blip2,
892 "chameleon": run_chameleon,
893 "deepseek_vl_v2": run_deepseek_vl2,
examples/offline_inference/vision_language_multi_image.py
+36
@@ -61,6 +61,41 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
61 )
62
63
64 +def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:
65 + model_name = "CohereForAI/aya-vision-8b"
66 +
67 + engine_args = EngineArgs(
68 + model=model_name,
69 + max_num_seqs=2,
70 + limit_mm_per_prompt={"image": len(image_urls)},
71 + )
72 +
73 + placeholders = [{"type": "image", "image": url} for url in image_urls]
74 + messages = [{
75 + "role":
76 + "user",
77 + "content": [
78 + *placeholders,
79 + {
80 + "type": "text",
81 + "text": question
82 + },
83 + ],
84 + }]
85 +
86 + processor = AutoProcessor.from_pretrained(model_name)
87 +
88 + prompt = processor.apply_chat_template(messages,
89 + tokenize=False,
90 + add_generation_prompt=True)
91 +
92 + return ModelRequestData(
93 + engine_args=engine_args,
94 + prompt=prompt,
95 + image_data=[fetch_image(url) for url in image_urls],
96 + )
97 +
98 +
99 def load_deepseek_vl2(question: str,
100 image_urls: list[str]) -> ModelRequestData:
101 model_name = "deepseek-ai/deepseek-vl2-tiny"
@@ -526,6 +561,7 @@ def load_qwen2_5_vl(question: str, image_urls: list[str]) -> ModelRequestData:
561
562 model_example_map = {
563 "aria": load_aria,
564 + "aya_vision": load_aya_vision,
565 "deepseek_vl_v2": load_deepseek_vl2,
566 "gemma3": load_gemma3,
567 "h2ovl_chat": load_h2ovl,
tests/models/decoder_only/vision_language/test_models.py
+14
@@ -158,6 +158,20 @@ VLM_TEST_SETTINGS = {
158 max_tokens=64,
159 marks=[large_gpu_mark(min_gb=64)],
160 ),
161 + "aya_vision": VLMTestInfo(
162 + models=["CohereForAI/aya-vision-8b"],
163 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
164 + prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
165 + single_image_prompts=IMAGE_ASSETS.prompts({
166 + "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
167 + "cherry_blossom": "<image>What is the season?", # noqa: E501
168 + }),
169 + multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
170 + max_model_len=8192,
171 + max_num_seqs=2,
172 + auto_cls=AutoModelForImageTextToText,
173 + vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}}
174 + ),
175 "blip2": VLMTestInfo(
176 # TODO: Change back to 2.7b once head_dim = 80 is supported
177 models=["Salesforce/blip2-opt-6.7b"],
tests/models/multimodal/processing/test_common.py
+1
@@ -246,6 +246,7 @@ def _test_processing_correctness_mistral(
246 # yapf: disable
247 @pytest.mark.parametrize("model_id", [
248 "rhymes-ai/Aria",
249 + "CohereForAI/aya-vision-8b",
250 "Salesforce/blip2-opt-2.7b",
251 "facebook/chameleon-7b",
252 "deepseek-ai/deepseek-vl2-tiny",
tests/models/registry.py
+1
@@ -259,6 +259,7 @@ _CROSS_ENCODER_EXAMPLE_MODELS = {
259 _MULTIMODAL_EXAMPLE_MODELS = {
260 # [Decoder-only]
261 "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
262 + "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereForAI/aya-vision-8b"), # noqa: E501
263 "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b", # noqa: E501
264 extras={"6b": "Salesforce/blip2-opt-6.7b"}), # noqa: E501
265 "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
vllm/config.py
+4
@@ -2716,6 +2716,10 @@ def _get_and_verify_max_len(
2716 max_len_key = key if max_len < derived_max_model_len \
2717 else max_len_key
2718 derived_max_model_len = min(derived_max_model_len, max_len)
2719 + # For Command-R / Cohere, Cohere2 / Aya Vision models
2720 + if tmp_max_len := getattr(hf_config, "model_max_length", None):
2721 + max_len_key = "model_max_length"
2722 + derived_max_model_len = tmp_max_len
2723
2724 # If sliding window is manually disabled, max_length should be less
2725 # than the sliding window length in the model config.
vllm/entrypoints/chat_utils.py
+3 -2
@@ -496,8 +496,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
496 if model_type.startswith("llava"):
497 return self._cached_token_str(self._tokenizer,
498 hf_config.image_token_index)
499 - if model_type in ("chameleon", "deepseek_vl_v2", "internvl_chat",
500 - "skywork_chat", "NVLM_D", "h2ovl_chat"):
499 + if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",
500 + "internvl_chat", "skywork_chat", "NVLM_D",
501 + "h2ovl_chat"):
502 return "<image>"
503 if model_type == "mllama":
504 return "<|image|>"
vllm/model_executor/models/aya_vision.py new
+527
@@ -0,0 +1,527 @@
1 +# SPDX-License-Identifier: Apache-2.0 Adapted from
2 +# https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision
3 +from functools import cached_property
4 +from typing import (Iterable, Literal, Mapping, Optional, Sequence, Set, Tuple,
5 + TypedDict, Union, cast)
6 +
7 +import torch
8 +from torch import nn
9 +from transformers import BatchFeature, GotOcr2ImageProcessor
10 +from transformers.activations import ACT2FN
11 +from transformers.image_processing_utils import get_size_dict
12 +from transformers.models.aya_vision import AyaVisionConfig
13 +from transformers.models.aya_vision.processing_aya_vision import (
14 + AyaVisionProcessor)
15 +from transformers.models.got_ocr2.image_processing_got_ocr2 import (
16 + get_optimal_tiled_canvas)
17 +
18 +from vllm.config import VllmConfig
19 +from vllm.jsontree import json_map_leaves
20 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21 +from vllm.model_executor.sampling_metadata import SamplingMetadata
22 +from vllm.multimodal import MULTIMODAL_REGISTRY
23 +from vllm.multimodal.inputs import MultiModalKwargs
24 +from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
25 + MultiModalDataItems)
26 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 + BaseProcessingInfo,
28 + MultiModalFieldConfig,
29 + PromptReplacement, PromptUpdate,
30 + encode_tokens)
31 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32 +from vllm.sequence import IntermediateTensors
33 +
34 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
35 +from .siglip import SiglipVisionModel
36 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
37 + maybe_prefix, merge_multimodal_embeddings)
38 +from .vision import scatter_patch_features, select_patch_features
39 +
40 +
41 +class AyaVisionImagePixelInputs(TypedDict):
42 + type: Literal["pixel_values"]
43 + pixel_values: torch.Tensor
44 + """
45 + Shape: `(num_patches_total, num_channels, height, width)`
46 +
47 + `num_patches_total` is the total number of patches over each image over each
48 + prompt in the batch.
49 + """
50 +
51 + num_patches: torch.Tensor
52 + """Shape: `(batch_size * num_images)`"""
53 +
54 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
55 + """
56 + A boolean mask indicating which image embeddings correspond to patch tokens.
57 +
58 + Shape: `(batch_size * num_images, num_embeds)`
59 + """
60 +
61 +
62 +class AyaVisionMultiModalProjector(nn.Module):
63 +
64 + def __init__(self, config: AyaVisionConfig):
65 + super().__init__()
66 + self.config = config
67 + self.downsample_factor = config.downsample_factor
68 + self.alignment_intermediate_size = getattr(
69 + config, "alignment_intermediate_size",
70 + config.text_config.hidden_size)
71 + self.layernorm = nn.LayerNorm(config.vision_config.hidden_size *
72 + (config.downsample_factor**2),
73 + eps=config.adapter_layer_norm_eps)
74 +
75 + self.linear_1 = nn.Linear(
76 + config.vision_config.hidden_size * (config.downsample_factor**2),
77 + self.alignment_intermediate_size,
78 + bias=True,
79 + )
80 +
81 + self.act = ACT2FN["silu"] # SwiGLU uses SiLU activation
82 + # For SwiGLU, project down to half size since we split intermediate dim
83 + self.linear_2 = nn.Linear(self.alignment_intermediate_size // 2,
84 + config.text_config.hidden_size,
85 + bias=True)
86 +
87 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
88 + image_features = self.pixel_shuffle(image_features)
89 + image_features = self.layernorm(image_features)
90 + hidden_states = self.linear_1(image_features)
91 +
92 + # Split along last dimension and apply SwiGLU
93 + x, gate = hidden_states.chunk(2, dim=-1)
94 + hidden_states = self.act(gate) * x
95 +
96 + hidden_states = self.linear_2(hidden_states)
97 + return hidden_states
98 +
99 + def pixel_shuffle(self,
100 + image_features: torch.Tensor) -> torch.Tensor: # B, S, D
101 + batch_size, seq_length, _ = image_features.shape
102 + height = width = int(seq_length**0.5)
103 + image_features = image_features.reshape(image_features.shape[0], width,
104 + height, -1)
105 + channels = image_features.shape[-1]
106 + image_features = image_features.reshape(
107 + batch_size, width, int(height / self.downsample_factor),
108 + int(channels * self.downsample_factor))
109 + image_features = image_features.permute(0, 2, 1, 3)
110 + image_features = image_features.reshape(
111 + batch_size, int(height / self.downsample_factor),
112 + int(width / self.downsample_factor), -1)
113 + image_features = image_features.permute(0, 2, 1, 3)
114 + return image_features
115 +
116 +
117 +class AyaVisionProcessingInfo(BaseProcessingInfo):
118 +
119 + def get_hf_config(self) -> AyaVisionConfig:
120 + return self.ctx.get_hf_config(AyaVisionConfig)
121 +
122 + def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:
123 + return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
124 +
125 + def get_image_processor(self) -> GotOcr2ImageProcessor:
126 + return self.get_hf_processor().image_processor
127 +
128 + def get_mm_max_tokens_per_item(
129 + self,
130 + seq_len: int,
131 + mm_counts: Mapping[str, int],
132 + ) -> Mapping[str, int]:
133 + return {"image": self.get_max_image_tokens()}
134 +
135 + def get_max_image_tokens(self) -> int:
136 + hf_processor = self.get_hf_processor()
137 + image_processor = hf_processor.image_processor
138 + image_size = self.get_image_size_with_most_features()
139 + tokenizer = hf_processor.tokenizer
140 + num_patches = self.get_num_patches(
141 + image_width=image_size.width,
142 + image_height=image_size.height,
143 + size=image_processor.size,
144 + min_patches=image_processor.min_patches,
145 + max_patches=image_processor.max_patches)
146 + image_string = hf_processor._prompt_split_image(num_patches)
147 + x = encode_tokens(
148 + tokenizer,
149 + image_string,
150 + add_special_tokens=False,
151 + )
152 + return len(x)
153 +
154 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
155 + return {"image": None}
156 +
157 + def get_image_size_with_most_features(self) -> ImageSize:
158 + image_processor = self.get_image_processor()
159 + height = image_processor.size['height']
160 + width = image_processor.size['width']
161 + max_patches = image_processor.max_patches
162 + return ImageSize(height=height * max_patches,
163 + width=width * max_patches)
164 +
165 + def get_num_patches(self, *, image_width: int, image_height: int,
166 + size: dict, min_patches: int, max_patches: int) -> int:
167 + """
168 + Calculate the number of patches needed for a given image based on size
169 + constraints. This method replicates and adjusts the logic from:
170 + transformers/models/got_ocr2/image_processing_got_ocr2
171 + """
172 + size = get_size_dict(size, default_to_square=False)
173 + num_columns, num_rows = get_optimal_tiled_canvas(
174 + (image_height, image_width), (size["height"], size["width"]),
175 + min_patches, max_patches)
176 + num_blocks = num_columns * num_rows
177 + return num_blocks if num_blocks == 1 else num_blocks + 1
178 +
179 +
180 +class AyaVisionDummyInputsBuilder(
181 + BaseDummyInputsBuilder[AyaVisionProcessingInfo]):
182 +
183 + def get_dummy_processor_inputs(
184 + self,
185 + seq_len: int,
186 + mm_counts: Mapping[str, int],
187 + ) -> ProcessorInputs:
188 + processor = self.info.get_hf_processor()
189 + image_token = processor.image_token
190 +
191 + num_images = mm_counts.get("image", 0)
192 + image_size = \
193 + self.info.get_image_size_with_most_features()
194 +
195 + mm_data = {
196 + "image":
197 + self._get_dummy_images(width=image_size.width,
198 + height=image_size.height,
199 + num_images=num_images)
200 + }
201 + return ProcessorInputs(
202 + prompt_text=image_token * num_images,
203 + mm_data=mm_data,
204 + )
205 +
206 +
207 +class AyaVisionMultiModalProcessor(
208 + BaseMultiModalProcessor[AyaVisionProcessingInfo]):
209 +
210 + def _call_hf_processor(
211 + self,
212 + prompt: str,
213 + mm_data: Mapping[str, object],
214 + mm_kwargs: Mapping[str, object],
215 + ) -> BatchFeature:
216 + processed_outputs = super()._call_hf_processor(
217 + prompt,
218 + mm_data,
219 + mm_kwargs,
220 + )
221 + hf_processor = self.info.get_hf_processor(**mm_kwargs)
222 + image_processor = hf_processor.image_processor
223 +
224 + hf_config = self.info.get_hf_config()
225 + # HF processor pops the `num_patches` kwarg, which is needed by vLLM
226 + if (images :=
227 + mm_data.get("images")) is not None and '<image>' in prompt:
228 + assert isinstance(images, list)
229 + parsed_images = (self._get_data_parser().parse_mm_data({
230 + "image":
231 + images
232 + }).get_items("image", ImageProcessorItems))
233 + image_sizes = [
234 + parsed_images.get_image_size(i)
235 + for i in range(len(parsed_images))
236 + ]
237 + num_patches = [
238 + self.info.get_num_patches(
239 + image_width=image_size.width,
240 + image_height=image_size.height,
241 + size=image_processor.size,
242 + min_patches=image_processor.min_patches,
243 + max_patches=image_processor.max_patches)
244 + for image_size in image_sizes
245 + ]
246 + image_tokens_list = [
247 + hf_processor._prompt_split_image(num_patch)
248 + for num_patch in num_patches
249 + ]
250 + tokenizer = self.info.get_tokenizer()
251 + image_token_ids = [
252 + tokenizer.encode(image_tokens, add_special_tokens=False)
253 + for image_tokens in image_tokens_list
254 + ]
255 + embed_is_patch = [
256 + torch.tensor(image_repl_tokens) == hf_config.image_token_index
257 + for image_repl_tokens in image_token_ids
258 + ]
259 + processed_outputs["embed_is_patch"] = embed_is_patch
260 + processed_outputs["num_patches"] = torch.tensor(num_patches)
261 +
262 + return processed_outputs
263 +
264 + def _get_mm_fields_config(
265 + self,
266 + hf_inputs: BatchFeature,
267 + hf_processor_mm_kwargs: Mapping[str, object],
268 + ) -> Mapping[str, MultiModalFieldConfig]:
269 + num_patches = hf_inputs.get("num_patches", torch.empty(0))
270 + return dict(
271 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
272 + "image", num_patches),
273 + num_patches=MultiModalFieldConfig.batched("image"),
274 + embed_is_patch=MultiModalFieldConfig.batched("image"),
275 + image_embeds=MultiModalFieldConfig.batched("image"),
276 + )
277 +
278 + def _get_prompt_updates(
279 + self,
280 + mm_items: MultiModalDataItems,
281 + hf_processor_mm_kwargs: Mapping[str, object],
282 + out_mm_kwargs: MultiModalKwargs,
283 + ) -> Sequence[PromptUpdate]:
284 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
285 + image_token = hf_processor.image_token
286 + image_processor = hf_processor.image_processor
287 +
288 + def get_replacement(item_idx: int):
289 + images: ImageProcessorItems = mm_items.get("image",
290 + ImageProcessorItems)
291 + image_size: ImageSize = images.get_image_size(item_idx)
292 + num_patches = self.info.get_num_patches(
293 + image_width=image_size.width,
294 + image_height=image_size.height,
295 + size=image_processor.size,
296 + min_patches=image_processor.min_patches,
297 + max_patches=image_processor.max_patches)
298 + return hf_processor._prompt_split_image(num_patches=num_patches)
299 +
300 + return [
301 + PromptReplacement(
302 + modality="image",
303 + target=image_token,
304 + replacement=get_replacement,
305 + )
306 + ]
307 +
308 +
309 +def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:
310 + feature_layers = hf_config.vision_feature_layer
311 + num_hidden_layers = hf_config.vision_config.num_hidden_layers
312 + # If we have one feature layer, initialize up to that layer
313 + if isinstance(feature_layers, int):
314 + return _get_layer_index(feature_layers, num_hidden_layers)
315 + # If we have multiple feature layers, initialize up to the deepest m
316 + elif isinstance(feature_layers, (list, tuple)):
317 + return max(
318 + _get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
319 + raise TypeError(f"vision_layer_feature type: {type(feature_layers)}"
320 + " is not supported")
321 +
322 +
323 +def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
324 + if feature_layer_index < 0:
325 + return num_hidden_layers + feature_layer_index + 1
326 + return feature_layer_index
327 +
328 +
329 +@MULTIMODAL_REGISTRY.register_processor(
330 + AyaVisionMultiModalProcessor,
331 + info=AyaVisionProcessingInfo,
332 + dummy_inputs=AyaVisionDummyInputsBuilder)
333 +class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
334 + SupportsPP):
335 +
336 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
337 + super().__init__()
338 + config: AyaVisionConfig = vllm_config.model_config.hf_config
339 + quant_config = vllm_config.quant_config
340 + multimodal_config = vllm_config.model_config.multimodal_config
341 + num_hidden_layers = _get_num_hidden_layers(config)
342 + self.config = config
343 + self.quant_config = quant_config
344 + self.multimodal_config = multimodal_config
345 +
346 + self.vision_tower = SiglipVisionModel(
347 + config.vision_config,
348 + quant_config,
349 + num_hidden_layers_override=num_hidden_layers,
350 + prefix=maybe_prefix(prefix, "vision_model"))
351 + self.vocab_size = config.text_config.vocab_size
352 + self.multi_modal_projector = AyaVisionMultiModalProjector(config)
353 + self.language_model = init_vllm_registered_model(
354 + vllm_config=vllm_config,
355 + hf_config=config.text_config,
356 + prefix=maybe_prefix(prefix, "model"),
357 + # Cohere2ForCausalLM and CohereForCausalLM are the same on vllm
358 + architectures=["Cohere2ForCausalLM"])
359 +
360 + @property
361 + def dtype(self):
362 + return next(self.parameters()).dtype
363 +
364 + def load_weights(self, weights: Iterable[Tuple[str,
365 + torch.Tensor]]) -> Set[str]:
366 + loader = AutoWeightsLoader(self)
367 + return loader.load_weights(weights)
368 +
369 + def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
370 + pixel_values: torch.Tensor,
371 + **kwargs) -> torch.Tensor:
372 + target_dtype = vision_tower.get_input_embeddings().weight.dtype
373 + image_features = vision_tower(pixel_values.to(dtype=target_dtype),
374 + **kwargs)
375 +
376 + def select_features(leaf: torch.Tensor):
377 + return self._select_image_features(
378 + leaf,
379 + strategy=self.config.vision_feature_select_strategy,
380 + )
381 +
382 + return cast(
383 + Union[torch.Tensor, tuple[torch.Tensor, ...]],
384 + json_map_leaves(select_features, image_features),
385 + )
386 +
387 + def _select_image_features(self, image_features: torch.Tensor, *,
388 + strategy: str) -> torch.Tensor:
389 + if strategy == "default":
390 + return image_features[:, 1:]
391 + elif strategy == "full":
392 + return image_features
393 +
394 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
395 +
396 + def _process_image_input(self, image_input: AyaVisionImagePixelInputs,
397 + **kwargs) -> list[torch.Tensor]:
398 + assert self.vision_tower is not None
399 + pixel_values = image_input["pixel_values"]
400 + num_patches = image_input["num_patches"]
401 + image_features = self._image_pixels_to_features(
402 + self.vision_tower, pixel_values=pixel_values)
403 + image_embeds = self.multi_modal_projector(image_features)
404 + return [
405 + e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())
406 + ]
407 +
408 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
409 + h = w = self.config.vision_config.image_size
410 + expected_dims = (3, h, w)
411 +
412 + def _validate_shape(d: torch.Tensor):
413 + if d.shape != expected_dims:
414 + raise ValueError(
415 + "The expected shape of pixel values per image per batch "
416 + f"is {expected_dims}. You supplied {tuple(d.shape)}.")
417 +
418 + for d in data:
419 + _validate_shape(d)
420 +
421 + return data
422 +
423 + def _parse_and_validate_image_input(
424 + self, **kwargs: object) -> Optional[AyaVisionImagePixelInputs]:
425 + pixel_values = kwargs.pop("pixel_values", None)
426 + num_patches = kwargs.pop("num_patches", None)
427 + embed_is_patch = kwargs.pop("embed_is_patch", None)
428 + image_embeds = kwargs.pop("image_embeds", None)
429 + assert image_embeds is None, "Aya Vision does not support image_embeds."
430 +
431 + if not isinstance(pixel_values, (torch.Tensor, list)):
432 + raise ValueError("Incorrect type of pixel values. "
433 + f"Got type: {type(pixel_values)}")
434 + if num_patches is not None and not isinstance(num_patches,
435 + (torch.Tensor, list)):
436 + raise ValueError("Incorrect type of num_patches. "
437 + f"Got type: {type(num_patches)}")
438 +
439 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
440 + raise ValueError("Incorrect type of embed_is_patch. "
441 + f"Got type: {type(embed_is_patch)}")
442 +
443 + pixel_values = flatten_bn(pixel_values, concat=True)
444 + num_patches = flatten_bn(num_patches, concat=True)
445 + embed_is_patch = flatten_bn(embed_is_patch)
446 + return AyaVisionImagePixelInputs(
447 + type="pixel_values",
448 + pixel_values=self._validate_pixel_values(pixel_values),
449 + num_patches=num_patches,
450 + embed_is_patch=embed_is_patch,
451 + )
452 +
453 + def get_multimodal_embeddings(
454 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
455 + image_input = self._parse_and_validate_image_input(**kwargs)
456 + if image_input is None:
457 + return None
458 + image_features = self._process_image_input(image_input, **kwargs)
459 + return scatter_patch_features(
460 + image_features,
461 + image_input["embed_is_patch"],
462 + )
463 +
464 + def get_input_embeddings(
465 + self,
466 + input_ids: torch.Tensor,
467 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
468 + ) -> torch.Tensor:
469 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
470 + if multimodal_embeddings is not None:
471 + inputs_embeds = merge_multimodal_embeddings(
472 + input_ids=input_ids,
473 + inputs_embeds=inputs_embeds,
474 + multimodal_embeddings=select_patch_features(
475 + multimodal_embeddings),
476 + placeholder_token_id=self.config.image_token_index)
477 +
478 + return inputs_embeds
479 +
480 + def forward(
481 + self,
482 + input_ids: torch.Tensor,
483 + positions: torch.Tensor,
484 + intermediate_tensors: Optional[IntermediateTensors] = None,
485 + inputs_embeds: Optional[torch.Tensor] = None,
486 + **kwargs: object,
487 + ) -> Union[torch.Tensor, IntermediateTensors]:
488 + if intermediate_tensors is not None:
489 + inputs_embeds = None
490 +
491 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
492 + # condition is for v0 compatibility.
493 + elif inputs_embeds is None:
494 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
495 + inputs_embeds = self.get_input_embeddings(input_ids,
496 + vision_embeddings)
497 + input_ids = None
498 +
499 + hidden_states = self.language_model.model(
500 + input_ids=input_ids,
501 + positions=positions,
502 + intermediate_tensors=intermediate_tensors,
503 + inputs_embeds=inputs_embeds,
504 + )
505 + return hidden_states
506 +
507 + def compute_logits(
508 + self,
509 + hidden_states: torch.Tensor,
510 + sampling_metadata: SamplingMetadata,
511 + ) -> Optional[torch.Tensor]:
512 + return self.language_model.compute_logits(hidden_states,
513 + sampling_metadata)
514 +
515 + @cached_property
516 + def sampler(self):
517 + if hasattr(self.language_model, "sampler"):
518 + return self.language_model.sampler
519 +
520 + return get_sampler()
521 +
522 + def sample(
523 + self,
524 + logits: torch.Tensor,
525 + sampling_metadata: SamplingMetadata,
526 + ) -> Optional[SamplerOutput]:
527 + return self.language_model.sample(logits, sampling_metadata)
vllm/model_executor/models/registry.py
+1
@@ -161,6 +161,7 @@ _CROSS_ENCODER_MODELS = {
161 _MULTIMODAL_MODELS = {
162 # [Decoder-only]
163 "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
164 + "AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"), # noqa: E501
165 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
166 "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
167 "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),