A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 08:45 UTC
[New Model] support new model ovis2.6 (#34426)
Signed-off-by: myselvess <23743269+myselvess@users.noreply.github.com>
myselvess committed
Feb 13, 2026 at 16:12 UTC
bcf0731aa07c11d92b6261c58f42d9ad07b949c6
6 files changed
+52
-43
docs/models/supported_models.md
+2
@@ -728,6 +728,8 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
728
| `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + I<sup>E+</sup> + V<sup>E+</sup> |`FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
729
| `Ovis` | Ovis2, Ovis1.6 | T + I<sup>+</sup> | `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis1.6-Llama3.2-3B`, etc. | | ✅︎ |
730
| `Ovis2_5` | Ovis2.5 | T + I<sup>+</sup> + V | `AIDC-AI/Ovis2.5-9B`, etc. | | |
731
+| `Ovis2_6ForCausalLM` | Ovis2.6 | T + I<sup>+</sup> + V | `AIDC-AI/Ovis2.6-2B`, etc. | | |
732
+| `Ovis2_6_MoeForCausalLM` | Ovis2.6 | T + I<sup>+</sup> + V | `AIDC-AI/Ovis2.6-30B-A3B`, etc. | | |
733
| `PaddleOCRVLForConditionalGeneration` | Paddle-OCR | T + I<sup>+</sup> | `PaddlePaddle/PaddleOCR-VL`, etc. | | |
734
| `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | ✅︎ | ✅︎ |
735
| `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
tests/models/registry.py
+6
@@ -915,6 +915,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
915
},
916
),
917
"Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
918
+ "Ovis2_6ForCausalLM": _HfExamplesInfo(
919
+ "AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
920
+ ),
921
+ "Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
922
+ "AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
923
+ ),
924
"PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
925
"PaddlePaddle/PaddleOCR-VL",
926
trust_remote_code=True,
vllm/model_executor/models/ovis2_5.py
+17
-25
@@ -42,21 +42,12 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
42
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
43
44
IMAGE_TOKEN = "<image>"
45
+IMAGE_PLACEHOLDER_ID = 151669
46
VIDEO_TOKEN = "<video>"
46
-INDICATOR_IDS = [-301, -302, -303, -304]
47
-
48
-IMAGE_PAD_TOKEN_MAP = {
49
- "gemma2": "<unused0>",
50
- "llama": "<|reserved_special_token_0|>",
51
- "qwen2": "<|image_pad|>",
52
- "qwen3": "<|image_pad|>",
53
-}
54
-IMAGE_PAD_TOKEN_ID_MAP = {
55
- "gemma2": 7,
56
- "llama": 128002,
57
- "qwen2": 151655,
58
- "qwen3": 151655,
59
-}
47
+VIDEO_PLACEHOLDER_ID = 151670
48
+INDICATOR_IDS = [151672, 151673, 151674, 151675]
49
+IMAGE_PAD_TOKEN_ID = 151655
50
+THINK_END_TOKEN_ID = 151668
51
52
53
class Ovis2_5ImagePatchInputs(TensorSchema):
@@ -187,17 +178,11 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
178
vit_config = self.get_hf_config().vit_config
179
return self.ctx.get_hf_processor(
180
Ovis2_5Processor,
190
- image_pad_token=self.get_image_pad_token(),
181
patch_size=vit_config.patch_size,
182
hidden_stride=vit_config.hidden_stride,
183
temporal_patch_size=vit_config.temporal_patch_size,
184
)
185
196
- def get_image_pad_token(self) -> str:
197
- hf_text_config = self.get_hf_config().get_text_config()
198
- text_model_type = hf_text_config.model_type
199
- return IMAGE_PAD_TOKEN_MAP.get(text_model_type)
200
-
186
def get_image_processor(self) -> BaseImageProcessor:
187
return self.get_hf_processor().image_processor # type: ignore
188
@@ -342,9 +327,9 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
327
hf_config = self.info.get_hf_config()
328
vte_vocab_size = hf_config.visual_vocab_size
329
return [
345
- vte_vocab_size - len(INDICATOR_IDS) + abs(x + 300) - 1
330
+ vte_vocab_size - len(INDICATOR_IDS) + (x - INDICATOR_IDS[0])
331
for x in visual_indicators
347
- if x < -300
332
+ if x >= INDICATOR_IDS[0]
333
]
334
335
def _call_hf_processor(
@@ -417,6 +402,14 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
402
hf_processor_mm_kwargs: Mapping[str, object],
403
out_mm_kwargs: MultiModalKwargsItems,
404
) -> list[PromptReplacement]:
405
+ tokenizer = self.info.get_tokenizer()
406
+ vocab = tokenizer.get_vocab()
407
+
408
+ placeholder = {
409
+ "image": vocab[IMAGE_TOKEN],
410
+ "video": vocab[VIDEO_TOKEN],
411
+ }
412
+
413
def get_replacement_ovis(item_idx, modality: str):
414
if modality == "image":
415
out_item = out_mm_kwargs["image"][item_idx]
@@ -432,7 +425,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
425
return [
426
PromptReplacement(
427
modality=modality,
435
- target=IMAGE_TOKEN if modality == "image" else VIDEO_TOKEN,
428
+ target=[placeholder[modality]],
429
replacement=partial(get_replacement_ovis, modality=modality),
430
)
431
for modality in ("image", "video")
@@ -476,8 +469,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
469
)
470
self.vte = VisualEmbedding(config.visual_vocab_size, config.hidden_size)
471
479
- text_model_type = self.config.get_text_config().model_type
480
- self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
472
+ self.image_pad_token_id: int = IMAGE_PAD_TOKEN_ID
473
474
self.make_empty_intermediate_tensors = (
475
self.get_language_model().make_empty_intermediate_tensors
vllm/model_executor/models/registry.py
+2
@@ -425,6 +425,8 @@ _MULTIMODAL_MODELS = {
425
),
426
"Ovis": ("ovis", "Ovis"),
427
"Ovis2_5": ("ovis2_5", "Ovis2_5"),
428
+ "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
429
+ "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
430
"PaddleOCRVLForConditionalGeneration": (
431
"paddleocr_vl",
432
"PaddleOCRVLForConditionalGeneration",
vllm/model_executor/models/siglip2navit.py
-1
@@ -582,7 +582,6 @@ class Siglip2VisionTransformer(nn.Module):
582
hidden_states = self.embeddings(pixel_values, grid_thws)
583
584
last_hidden_state = self.encoder(hidden_states, grid_thws)
585
- last_hidden_state = self.post_layernorm(last_hidden_state)
585
586
return last_hidden_state
587
vllm/transformers_utils/processors/ovis2_5.py
+25
-17
@@ -78,17 +78,32 @@ class Ovis2_5Processor(ProcessorMixin):
78
79
@cached_property
80
def extra_special_tokens(self):
81
- image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]
82
- extra_special_tokens = {
83
- "image_token": -200,
84
- "video_token": -201,
85
- "visual_atom": -300,
86
- "image_start": -301,
87
- "image_end": -302,
88
- "video_start": -303,
89
- "video_end": -304,
90
- "image_pad": image_pad_token_id,
81
+ vocab = self.tokenizer.get_vocab()
82
+ required_tokens = {
83
+ "image_token": "<image>",
84
+ "video_token": "<video>",
85
+ "visual_atom": "<ovis_visual_atom>",
86
+ "image_start": "<ovis_image_start>",
87
+ "image_end": "<ovis_image_end>",
88
+ "video_start": "<ovis_video_start>",
89
+ "video_end": "<ovis_video_end>",
90
+ "image_pad": "<|image_pad|>",
91
}
92
+
93
+ extra_special_tokens = {}
94
+ suggestion = (
95
+ "please add '<image>', '<video>', '<ovis_visual_atom>', "
96
+ "'<ovis_image_start>', '<ovis_image_end>', '<ovis_video_start>', "
97
+ "'<ovis_video_end>' in 'additional_special_tokens' of "
98
+ "tokenizer_config.json, You can refer to "
99
+ "https://huggingface.co/AIDC-AI/Ovis2.6-30B-A3B/blob/main/tokenizer_config.json"
100
+ )
101
+
102
+ for key, token_name in required_tokens.items():
103
+ if token_name not in vocab:
104
+ raise ValueError(f"Can not find {token_name}, {suggestion}")
105
+ extra_special_tokens[key] = vocab[token_name]
106
+
107
return extra_special_tokens
108
109
def __call__(
@@ -156,9 +171,6 @@ class Ovis2_5Processor(ProcessorMixin):
171
- **second_per_grid_ts** -- list of video seconds per time grid.
172
Returned when `videos` is not `None`.
173
"""
159
- min_pixels = kwargs.pop("min_pixels", MIN_PIXELS)
160
- max_pixels = kwargs.pop("max_pixels", MAX_PIXELS)
161
-
174
output_kwargs = self._merge_kwargs(
175
Ovis2_5ProcessorKwargs,
176
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
@@ -175,8 +187,6 @@ class Ovis2_5Processor(ProcessorMixin):
187
for image in images if isinstance(images, list) else [images]:
188
pixel_values, image_placeholders, grid = self.preprocess_multidata(
189
images=image,
178
- min_pixels=min_pixels,
179
- max_pixels=max_pixels,
190
**output_kwargs["images_kwargs"],
191
)
192
processed_images.append(pixel_values)
@@ -197,8 +207,6 @@ class Ovis2_5Processor(ProcessorMixin):
207
for video in videos if isinstance(videos, list) else [videos]:
208
pixel_values, video_placeholders, grid = self.preprocess_multidata(
209
video=video,
200
- min_pixels=min_pixels,
201
- max_pixels=max_pixels,
210
**output_kwargs["videos_kwargs"],
211
)
212
processed_videos.append(pixel_values)