A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[New Model] Support Command-A-Vision (#22660)
Signed-off-by: donglu <donglu@cohere.com>
dongluw committed
Aug 12, 2025 at 04:39 UTC
9f909b89963aa71b06b490a78ac9905d11879454
6 files changed
+510
-1
docs/models/supported_models.md
+2
-1
@@ -331,7 +331,7 @@ th {
331
| `BloomForCausalLM` | BLOOM, BLOOMZ, BLOOMChat | `bigscience/bloom`, `bigscience/bloomz`, etc. | | ✅︎ | |
332
| `BartForConditionalGeneration` | BART | `facebook/bart-base`, `facebook/bart-large-cnn`, etc. | | | |
333
| `ChatGLMModel`, `ChatGLMForConditionalGeneration` | ChatGLM | `zai-org/chatglm2-6b`, `zai-org/chatglm3-6b`, `ShieldLM-6B-chatglm3`, etc. | ✅︎ | ✅︎ | ✅︎ |
334
-| `CohereForCausalLM`, `Cohere2ForCausalLM` | Command-R | `CohereForAI/c4ai-command-r-v01`, `CohereForAI/c4ai-command-r7b-12-2024`, etc. | ✅︎ | ✅︎ | ✅︎ |
334
+| `CohereForCausalLM`, `Cohere2ForCausalLM` | Command-R | `CohereLabs/c4ai-command-r-v01`, `CohereLabs/c4ai-command-r7b-12-2024`, etc. | ✅︎ | ✅︎ | ✅︎ |
335
| `DbrxForCausalLM` | DBRX | `databricks/dbrx-base`, `databricks/dbrx-instruct`, etc. | | ✅︎ | ✅︎ |
336
| `DeciLMForCausalLM` | DeciLM | `nvidia/Llama-3_3-Nemotron-Super-49B-v1`, etc. | ✅︎ | ✅︎ | ✅︎ |
337
| `DeepseekForCausalLM` | DeepSeek | `deepseek-ai/deepseek-llm-67b-base`, `deepseek-ai/deepseek-llm-7b-chat`, etc. | | ✅︎ | ✅︎ |
@@ -601,6 +601,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
601
| `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereForAI/aya-vision-8b`, `CohereForAI/aya-vision-32b`, etc. | | ✅︎ | ✅︎ |
602
| `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | | ✅︎ | ✅︎ |
603
| `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ | ✅︎ |
604
+| `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ | ✅︎ |
605
| `DeepseekVLV2ForCausalLM`<sup>^</sup> | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ | ✅︎ |
606
| `Florence2ForConditionalGeneration` | Florence-2 | T + I | `microsoft/Florence-2-base`, `microsoft/Florence-2-large`, etc. | | | |
607
| `FuyuForCausalLM` | Fuyu | T + I | `adept/fuyu-8b`, etc. | | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+24
@@ -126,6 +126,29 @@ def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
126
)
127
128
129
+def run_command_a_vision(questions: list[str], modality: str) -> ModelRequestData:
130
+ assert modality == "image"
131
+
132
+ model_name = "CohereLabs/command-a-vision-07-2025"
133
+
134
+ engine_args = EngineArgs(
135
+ model=model_name,
136
+ max_model_len=32768,
137
+ tensor_parallel_size=4,
138
+ limit_mm_per_prompt={modality: 1},
139
+ )
140
+
141
+ prompts = [
142
+ f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><|IMG_PATCH|>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
143
+ for question in questions
144
+ ]
145
+
146
+ return ModelRequestData(
147
+ engine_args=engine_args,
148
+ prompts=prompts,
149
+ )
150
+
151
+
152
# Deepseek-VL2
153
def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
154
assert modality == "image"
@@ -1417,6 +1440,7 @@ model_example_map = {
1440
"aya_vision": run_aya_vision,
1441
"blip-2": run_blip2,
1442
"chameleon": run_chameleon,
1443
+ "command_a_vision": run_command_a_vision,
1444
"deepseek_vl_v2": run_deepseek_vl2,
1445
"florence2": run_florence2,
1446
"fuyu": run_fuyu,
examples/offline_inference/vision_language_multi_image.py
+37
@@ -107,6 +107,42 @@ def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:
107
)
108
109
110
+def load_command_a_vision(question: str, image_urls: list[str]) -> ModelRequestData:
111
+ model_name = "CohereLabs/command-a-vision-07-2025"
112
+
113
+ # NOTE: This model is 122B parameters and requires tensor parallelism
114
+ # Recommended to use tp=4 on H100 GPUs
115
+ engine_args = EngineArgs(
116
+ model=model_name,
117
+ max_model_len=32768,
118
+ tensor_parallel_size=4,
119
+ limit_mm_per_prompt={"image": len(image_urls)},
120
+ )
121
+
122
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
123
+ messages = [
124
+ {
125
+ "role": "user",
126
+ "content": [
127
+ *placeholders,
128
+ {"type": "text", "text": question},
129
+ ],
130
+ }
131
+ ]
132
+
133
+ processor = AutoProcessor.from_pretrained(model_name)
134
+
135
+ prompt = processor.apply_chat_template(
136
+ messages, tokenize=False, add_generation_prompt=True
137
+ )
138
+
139
+ return ModelRequestData(
140
+ engine_args=engine_args,
141
+ prompt=prompt,
142
+ image_data=[fetch_image(url) for url in image_urls],
143
+ )
144
+
145
+
146
def load_deepseek_vl2(question: str, image_urls: list[str]) -> ModelRequestData:
147
model_name = "deepseek-ai/deepseek-vl2-tiny"
148
@@ -1031,6 +1067,7 @@ def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
1067
model_example_map = {
1068
"aria": load_aria,
1069
"aya_vision": load_aya_vision,
1070
+ "command_a_vision": load_command_a_vision,
1071
"deepseek_vl_v2": load_deepseek_vl2,
1072
"gemma3": load_gemma3,
1073
"h2ovl_chat": load_h2ovl,
tests/models/registry.py
+1
@@ -383,6 +383,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
383
"Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b", # noqa: E501
384
extras={"6b": "Salesforce/blip2-opt-6.7b"}), # noqa: E501
385
"ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
386
+ "Cohere2VisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/command-a-vision-07-2025"), # noqa: E501
387
"DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501
388
extras={"fork": "Isotr0py/deepseek-vl2-tiny"}, # noqa: E501
389
max_transformers_version="4.48", # noqa: E501
vllm/model_executor/models/cohere2_vision.py
new
+445
@@ -0,0 +1,445 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# Adapted from vllm/model_executor/models/aya_vision.py
4
+"""Command-A-Vision (Cohere2Vision) multimodal model implementation for vLLM."""
5
+
6
+from collections.abc import Iterable, Mapping, Sequence
7
+from typing import Annotated, Literal, Optional, Union
8
+
9
+import torch
10
+from torch import nn
11
+from transformers import BatchFeature, PretrainedConfig
12
+from transformers.models.cohere2_vision import Cohere2VisionConfig
13
+from transformers.models.cohere2_vision.processing_cohere2_vision import (
14
+ Cohere2VisionProcessor)
15
+
16
+from vllm.config import VllmConfig
17
+from vllm.model_executor.layers.activation import MulAndSilu
18
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
19
+ RowParallelLinear)
20
+from vllm.model_executor.layers.quantization import QuantizationConfig
21
+from vllm.model_executor.layers.quantization.awq import AWQConfig
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.multimodal import MULTIMODAL_REGISTRY
24
+from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs
25
+from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
26
+ MultiModalDataItems)
27
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
+ BaseProcessingInfo,
29
+ MultiModalFieldConfig,
30
+ PromptReplacement, PromptUpdate,
31
+ PromptUpdateDetails)
32
+from vllm.multimodal.profiling import BaseDummyInputsBuilder
33
+from vllm.sequence import IntermediateTensors
34
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
35
+
36
+from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
37
+from .siglip import SiglipVisionModel
38
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39
+ init_vllm_registered_model, maybe_prefix,
40
+ merge_multimodal_embeddings)
41
+
42
+
43
+class Cohere2VisionImagePixelInputs(TensorSchema):
44
+ """
45
+ Dimensions:
46
+ - np: The total number of patches over each image over each prompt in
47
+ the batch
48
+ - c: Number of channels
49
+ - h: Height of each image patch
50
+ - w: Width of each image patch
51
+ - bn: Batch size * number of images
52
+ """
53
+
54
+ type: Literal["pixel_values"]
55
+
56
+ pixel_values: Annotated[
57
+ torch.Tensor,
58
+ TensorShape("np", 3, "h", "w"),
59
+ ]
60
+
61
+ num_patches: Annotated[
62
+ torch.Tensor,
63
+ TensorShape("bn"),
64
+ ]
65
+
66
+
67
+class Cohere2VisionMultiModalProjector(nn.Module):
68
+ """Multimodal projector that maps vision features to text embedding space.
69
+
70
+ Uses pixel shuffle downsampling followed by SwiGLU activation.
71
+ """
72
+
73
+ def __init__(self, config: Cohere2VisionConfig, prefix: str = ""):
74
+ super().__init__()
75
+ self.downsample_factor = config.downsample_factor
76
+
77
+ # Input dimension after pixel shuffle downsampling
78
+ input_dim = config.vision_config.hidden_size * (
79
+ config.downsample_factor**2)
80
+ # MergedColumnParallelLinear expects the intermediate size to be a list
81
+ # of sizes, so that it will load the weights as two separate linear
82
+ # layers before applying any parallelism.
83
+ # We need to divide the alignment intermediate size by 2 because
84
+ # the weights are merged weights of two linear layers for SwiGLU.
85
+ self.intermediate_size = config.alignment_intermediate_size // 2
86
+
87
+ self.linear_1 = MergedColumnParallelLinear(
88
+ input_dim,
89
+ [self.intermediate_size] * 2,
90
+ bias=True,
91
+ return_bias=False,
92
+ prefix=f"{prefix}.linear_1",
93
+ )
94
+ self.act = MulAndSilu()
95
+ self.linear_2 = RowParallelLinear(
96
+ self.intermediate_size,
97
+ config.text_config.hidden_size,
98
+ bias=True,
99
+ return_bias=False,
100
+ prefix=f"{prefix}.linear_2",
101
+ )
102
+
103
+ def forward(self, image_features):
104
+ image_features = self.pixel_shuffle(image_features)
105
+ hidden_states = self.linear_1(image_features)
106
+ hidden_states = self.act(hidden_states)
107
+ hidden_states = self.linear_2(hidden_states)
108
+ return hidden_states
109
+
110
+ def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor:
111
+ """Apply pixel shuffle downsampling to reduce spatial dimensions.
112
+
113
+ Args:
114
+ image_features: Input tensor of shape [B, S, D] where S = H*W
115
+
116
+ Returns:
117
+ Downsampled tensor with increased channel dimension
118
+ """
119
+ height = width = int(image_features.shape[1]**0.5)
120
+ x = image_features.reshape(image_features.shape[0], width, height, -1)
121
+ n, h, w, c = x.size()
122
+ scale_factor = 1. / self.downsample_factor
123
+ nh = int(h * scale_factor)
124
+ nw = int(w * scale_factor)
125
+ x = x.reshape(n, nh, self.downsample_factor, nw,
126
+ self.downsample_factor, c)
127
+ x = x.permute(0, 1, 3, 2, 4, 5).contiguous()
128
+ x = x.reshape(n, nh, nw, -1)
129
+ return x
130
+
131
+
132
+class Cohere2VisionProcessingInfo(BaseProcessingInfo):
133
+
134
+ def get_hf_config(self) -> Cohere2VisionConfig:
135
+ return self.ctx.get_hf_config(Cohere2VisionConfig)
136
+
137
+ def get_hf_processor(self, **kwargs: object) -> Cohere2VisionProcessor:
138
+ return self.ctx.get_hf_processor(Cohere2VisionProcessor, **kwargs)
139
+
140
+ def get_image_processor(self, **kwargs: object):
141
+ return self.get_hf_processor(**kwargs).image_processor
142
+
143
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
144
+ return {"image": None}
145
+
146
+ def get_image_size_with_most_features(self) -> ImageSize:
147
+ image_processor = self.get_image_processor()
148
+ height = image_processor.size['height']
149
+ width = image_processor.size['width']
150
+ max_patches = image_processor.max_patches
151
+ return ImageSize(height=height * max_patches, width=width)
152
+
153
+ def get_num_patches(self, image_width: int, image_height: int) -> int:
154
+ """
155
+ Calculate the number of image patches for a given image.
156
+ Uses the HF processor to determine the actual number of patches.
157
+ """
158
+ return self.get_hf_processor(
159
+ ).image_processor.get_number_of_image_patches(image_height,
160
+ image_width, {})
161
+
162
+
163
+class Cohere2VisionDummyInputsBuilder(
164
+ BaseDummyInputsBuilder[Cohere2VisionProcessingInfo]):
165
+
166
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
167
+ num_images = mm_counts.get("image", 0)
168
+
169
+ processor = self.info.get_hf_processor()
170
+ image_token = processor.image_token
171
+
172
+ return image_token * num_images
173
+
174
+ def get_dummy_mm_data(
175
+ self,
176
+ seq_len: int,
177
+ mm_counts: Mapping[str, int],
178
+ ) -> MultiModalDataDict:
179
+ num_images = mm_counts.get("image", 0)
180
+ image_size = \
181
+ self.info.get_image_size_with_most_features()
182
+
183
+ return {
184
+ "image":
185
+ self._get_dummy_images(width=image_size.width,
186
+ height=image_size.height,
187
+ num_images=num_images)
188
+ }
189
+
190
+
191
+class Cohere2VisionMultiModalProcessor(
192
+ BaseMultiModalProcessor[Cohere2VisionProcessingInfo]):
193
+
194
+ def _call_hf_processor(
195
+ self,
196
+ prompt: str,
197
+ mm_data: Mapping[str, object],
198
+ mm_kwargs: Mapping[str, object],
199
+ tok_kwargs: Mapping[str, object],
200
+ ) -> BatchFeature:
201
+ processed_outputs = super()._call_hf_processor(
202
+ prompt,
203
+ mm_data,
204
+ mm_kwargs,
205
+ tok_kwargs,
206
+ )
207
+
208
+ # Ensure num_patches is available for proper tensor splitting
209
+ if "num_patches" not in processed_outputs and (
210
+ images := mm_data.get("images")) is not None:
211
+ # Fallback calculation if HF processor didn't provide num_patches
212
+ parsed_images = self._get_data_parser().parse_mm_data({
213
+ "image":
214
+ images
215
+ }).get_items("image", ImageProcessorItems)
216
+
217
+ num_patches = [
218
+ self.info.get_num_patches(
219
+ image_width=parsed_images.get_image_size(i).width,
220
+ image_height=parsed_images.get_image_size(i).height)
221
+ for i in range(len(parsed_images))
222
+ ]
223
+ processed_outputs["num_patches"] = torch.tensor(num_patches)
224
+
225
+ return processed_outputs
226
+
227
+ def _get_mm_fields_config(
228
+ self,
229
+ hf_inputs: BatchFeature,
230
+ hf_processor_mm_kwargs: Mapping[str, object],
231
+ ) -> Mapping[str, MultiModalFieldConfig]:
232
+ num_patches = hf_inputs.get("num_patches", torch.empty(0))
233
+ return dict(
234
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
235
+ "image", num_patches),
236
+ num_patches=MultiModalFieldConfig.batched("image"),
237
+ image_embeds=MultiModalFieldConfig.batched("image"),
238
+ )
239
+
240
+ def _get_prompt_updates(
241
+ self,
242
+ mm_items: MultiModalDataItems,
243
+ hf_processor_mm_kwargs: Mapping[str, object],
244
+ out_mm_kwargs: MultiModalKwargs,
245
+ ) -> Sequence[PromptUpdate]:
246
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
247
+ image_token = hf_processor.image_token
248
+ img_line_break_token = hf_processor.img_line_break_token
249
+ boi_token = hf_processor.boi_token
250
+ eoi_token = hf_processor.eoi_token
251
+
252
+ def get_replacement(item_idx: int):
253
+ images: ImageProcessorItems = mm_items.get("image",
254
+ ImageProcessorItems)
255
+ image_size: ImageSize = images.get_image_size(item_idx)
256
+
257
+ num_patches = self.info.get_num_patches(image_size.height,
258
+ image_size.width)
259
+ img_tokens_per_tile = int(hf_processor.patch_size**2)
260
+ single_tile_tokens = image_token * img_tokens_per_tile + \
261
+ img_line_break_token
262
+ img_string = f"{boi_token}\
263
+ {single_tile_tokens * num_patches}\
264
+ {eoi_token}"
265
+
266
+ return PromptUpdateDetails.select_text(img_string, image_token)
267
+
268
+ return [
269
+ PromptReplacement(
270
+ modality="image",
271
+ target=image_token,
272
+ replacement=get_replacement,
273
+ )
274
+ ]
275
+
276
+
277
+@MULTIMODAL_REGISTRY.register_processor(
278
+ Cohere2VisionMultiModalProcessor,
279
+ info=Cohere2VisionProcessingInfo,
280
+ dummy_inputs=Cohere2VisionDummyInputsBuilder)
281
+class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal,
282
+ SupportsPP):
283
+
284
+ hf_to_vllm_mapper = WeightsMapper(
285
+ orig_to_new_prefix={
286
+ "model.vision_tower.": "vision_tower.",
287
+ "model.multi_modal_projector.": "multi_modal_projector.",
288
+ "model.language_model.": "language_model.model.",
289
+ "lm_head.": "language_model.lm_head.",
290
+ })
291
+
292
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
293
+ super().__init__()
294
+ config: Cohere2VisionConfig = vllm_config.model_config.hf_config
295
+ quant_config = vllm_config.quant_config
296
+ multimodal_config = vllm_config.model_config.multimodal_config
297
+ self.config = config
298
+ self.quant_config = quant_config
299
+ self.multimodal_config = multimodal_config
300
+ self._patch_quant_config(config, quant_config)
301
+
302
+ self.vision_tower = SiglipVisionModel(config.vision_config,
303
+ quant_config,
304
+ prefix=maybe_prefix(
305
+ prefix, "vision_tower"))
306
+ self.vocab_size = config.text_config.vocab_size
307
+ self.multi_modal_projector = \
308
+ Cohere2VisionMultiModalProjector(
309
+ config, prefix=maybe_prefix(prefix, "multi_modal_projector"))
310
+ self.language_model = init_vllm_registered_model(
311
+ vllm_config=vllm_config,
312
+ hf_config=config.text_config,
313
+ prefix=maybe_prefix(prefix, "language_model"),
314
+ architectures=["Cohere2ForCausalLM"])
315
+
316
+ @property
317
+ def dtype(self):
318
+ return next(self.parameters()).dtype
319
+
320
+ def load_weights(self, weights: Iterable[tuple[str,
321
+ torch.Tensor]]) -> set[str]:
322
+ loader = AutoWeightsLoader(self)
323
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
324
+
325
+ def _process_image_input(self, image_input: Cohere2VisionImagePixelInputs,
326
+ **kwargs) -> list[torch.Tensor]:
327
+ """Process image pixels through vision tower and projector.
328
+
329
+ Args:
330
+ image_input: Validated image input containing pixel values and
331
+ patch counts
332
+
333
+ Returns:
334
+ List of flattened image embeddings, one per image
335
+ """
336
+ assert self.vision_tower is not None, "Vision tower is required"
337
+
338
+ pixel_values = image_input["pixel_values"]
339
+ num_patches = image_input["num_patches"]
340
+
341
+ # Extract visual features
342
+ image_features = self.vision_tower(pixel_values)
343
+
344
+ # Project to text embedding space
345
+ image_embeds = self.multi_modal_projector(image_features)
346
+
347
+ # Split and flatten embeddings per image
348
+ return [
349
+ e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())
350
+ ]
351
+
352
+ def _parse_and_validate_image_input(
353
+ self, **kwargs: object) -> Optional[Cohere2VisionImagePixelInputs]:
354
+ pixel_values = kwargs.pop("pixel_values", None)
355
+ num_patches = kwargs.pop("num_patches", None)
356
+ image_embeds = kwargs.pop("image_embeds", None)
357
+ assert image_embeds is None, \
358
+ "Cohere2Vision does not support image_embeds."
359
+
360
+ if pixel_values is None:
361
+ return None
362
+
363
+ return Cohere2VisionImagePixelInputs(
364
+ type="pixel_values",
365
+ pixel_values=flatten_bn(pixel_values, concat=True),
366
+ num_patches=flatten_bn(num_patches, concat=True),
367
+ resolve_bindings={
368
+ "h": self.config.vision_config.image_size,
369
+ "w": self.config.vision_config.image_size,
370
+ })
371
+
372
+ def _patch_quant_config(self, config: PretrainedConfig,
373
+ quant_config: QuantizationConfig):
374
+ # the awq models from OpenGVLab missing `modules_to_not_convert`
375
+ # patch the quant_config to add `modules_to_not_convert` back
376
+ if isinstance(quant_config, AWQConfig):
377
+ text_config = config.text_config
378
+ llm_quant_config = getattr(text_config, "quantization_config",
379
+ None)
380
+ if (not quant_config.modules_to_not_convert) and (llm_quant_config
381
+ is not None):
382
+ quant_config.modules_to_not_convert.append("vision_tower")
383
+
384
+ def get_language_model(self) -> torch.nn.Module:
385
+ return self.language_model
386
+
387
+ def get_multimodal_embeddings(self,
388
+ **kwargs: object) -> MultiModalEmbeddings:
389
+ image_input = self._parse_and_validate_image_input(**kwargs)
390
+ if image_input is None:
391
+ return []
392
+
393
+ return self._process_image_input(image_input, **kwargs)
394
+
395
+ def get_input_embeddings(
396
+ self,
397
+ input_ids: torch.Tensor,
398
+ multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
399
+ ) -> torch.Tensor:
400
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
401
+ if multimodal_embeddings is not None \
402
+ and len(multimodal_embeddings) != 0:
403
+ inputs_embeds = merge_multimodal_embeddings(
404
+ input_ids=input_ids,
405
+ inputs_embeds=inputs_embeds,
406
+ multimodal_embeddings=multimodal_embeddings,
407
+ placeholder_token_id=self.config.image_token_id,
408
+ )
409
+
410
+ return inputs_embeds
411
+
412
+ def forward(
413
+ self,
414
+ input_ids: torch.Tensor,
415
+ positions: torch.Tensor,
416
+ intermediate_tensors: Optional[IntermediateTensors] = None,
417
+ inputs_embeds: Optional[torch.Tensor] = None,
418
+ **kwargs: object,
419
+ ) -> Union[torch.Tensor, IntermediateTensors]:
420
+ if intermediate_tensors is not None:
421
+ inputs_embeds = None
422
+
423
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
424
+ # condition is for v0 compatibility.
425
+ elif inputs_embeds is None:
426
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
427
+ inputs_embeds = self.get_input_embeddings(input_ids,
428
+ vision_embeddings)
429
+ input_ids = None
430
+
431
+ hidden_states = self.language_model.model(
432
+ input_ids=input_ids,
433
+ positions=positions,
434
+ intermediate_tensors=intermediate_tensors,
435
+ inputs_embeds=inputs_embeds,
436
+ )
437
+ return hidden_states
438
+
439
+ def compute_logits(
440
+ self,
441
+ hidden_states: torch.Tensor,
442
+ sampling_metadata: SamplingMetadata,
443
+ ) -> Optional[torch.Tensor]:
444
+ return self.language_model.compute_logits(hidden_states,
445
+ sampling_metadata)
vllm/model_executor/models/registry.py
+1
@@ -201,6 +201,7 @@ _MULTIMODAL_MODELS = {
201
"AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"), # noqa: E501
202
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
203
"ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
204
+ "Cohere2VisionForConditionalGeneration": ("cohere2_vision", "Cohere2VisionForConditionalGeneration"), # noqa: E501
205
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
206
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
207
"Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"), # noqa: E501