@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 021c16c7ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 11:00 UTC

[Model] Broadcast Ovis2 implementation to fit Ovis1.6 (#17861)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed May 12, 2025 at 08:56 UTC 021c16c7caaa6886248f1d048edbcdb678415964
16 files changed +330 -212
docs/source/models/supported_models.md
+3 -3
@@ -1045,10 +1045,10 @@ Specified using `--task generate`.
1045 *
1046 * ✅︎
1047 * ✅︎
1048 -- * `Ovis2ForConditionalGeneration`<sup>^</sup>
1049 - * Ovis2
1048 +- * `Ovis`
1049 + * Ovis2, Ovis1.6
1050 * T + I<sup>+</sup>
1051 - * `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis2-2B`, etc.
1051 + * `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis1.6-Llama3.2-3B`, etc.
1052 *
1053 *
1054 * ✅︎
examples/offline_inference/vision_language.py
+12 -9
@@ -725,8 +725,8 @@ def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
725 )
726
727
728 -# Ovis2
729 -def run_ovis2(questions: list[str], modality: str) -> ModelRequestData:
728 +# Ovis
729 +def run_ovis(questions: list[str], modality: str) -> ModelRequestData:
730 assert modality == "image"
731
732 model_name = "AIDC-AI/Ovis2-1B"
@@ -737,15 +737,18 @@ def run_ovis2(questions: list[str], modality: str) -> ModelRequestData:
737 max_num_seqs=2,
738 trust_remote_code=True,
739 dtype="half",
740 - hf_overrides={"architectures": ["Ovis2ForConditionalGeneration"]},
740 limit_mm_per_prompt={modality: 1},
741 )
742
744 - placeholder = "<image>\n"
745 - prompts = [("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
746 - f"<|im_start|>user\n{placeholder}"
747 - f"{question}<|im_end|>\n"
748 - "<|im_start|>assistant\n") for question in questions]
743 + tokenizer = AutoTokenizer.from_pretrained(model_name,
744 + trust_remote_code=True)
745 + messages = [[{
746 + 'role': 'user',
747 + 'content': f"<image>\n{question}"
748 + }] for question in questions]
749 + prompts = tokenizer.apply_chat_template(messages,
750 + tokenize=False,
751 + add_generation_prompt=True)
752
753 return ModelRequestData(
754 engine_args=engine_args,
@@ -1069,7 +1072,7 @@ model_example_map = {
1072 "llama4": run_llama4,
1073 "molmo": run_molmo,
1074 "NVLM_D": run_nvlm_d,
1072 - "ovis2": run_ovis2,
1075 + "ovis": run_ovis,
1076 "paligemma": run_paligemma,
1077 "paligemma2": run_paligemma2,
1078 "phi3_v": run_phi3v,
examples/offline_inference/vision_language_multi_image.py
+12 -10
@@ -436,8 +436,8 @@ def load_nvlm_d(question: str, image_urls: list[str]) -> ModelRequestData:
436 )
437
438
439 -# Ovis2
440 -def load_ovis2(question: str, image_urls: list[str]) -> ModelRequestData:
439 +# Ovis
440 +def load_ovis(question: str, image_urls: list[str]) -> ModelRequestData:
441 model_name = "AIDC-AI/Ovis2-1B"
442
443 engine_args = EngineArgs(
@@ -447,15 +447,17 @@ def load_ovis2(question: str, image_urls: list[str]) -> ModelRequestData:
447 trust_remote_code=True,
448 dtype="half",
449 limit_mm_per_prompt={"image": len(image_urls)},
450 - hf_overrides={"architectures": ["Ovis2ForConditionalGeneration"]},
450 )
451
453 - placeholder = '\n'.join(
454 - [f'Image {i+1}: <image>' for i in range(len(image_urls))]) + '\n'
455 - prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
456 - f"<|im_start|>user\n{placeholder}"
457 - f"{question}<|im_end|>\n"
458 - "<|im_start|>assistant\n")
452 + placeholders = "\n".join(f"Image-{i}: <image>\n"
453 + for i, _ in enumerate(image_urls, start=1))
454 + messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
455 +
456 + tokenizer = AutoTokenizer.from_pretrained(model_name,
457 + trust_remote_code=True)
458 + prompt = tokenizer.apply_chat_template(messages,
459 + tokenize=False,
460 + add_generation_prompt=True)
461
462 return ModelRequestData(
463 engine_args=engine_args,
@@ -713,7 +715,7 @@ model_example_map = {
715 "mistral3": load_mistral3,
716 "mllama": load_mllama,
717 "NVLM_D": load_nvlm_d,
716 - "ovis2": load_ovis2,
718 + "ovis": load_ovis,
719 "phi3_v": load_phi3v,
720 "phi4_mm": load_phi4mm,
721 "pixtral_hf": load_pixtral_hf,
tests/conftest.py
+7 -1
@@ -355,10 +355,16 @@ class HfRunner:
355 **model_kwargs,
356 )
357
358 + # in case some unquantized custom models are not in same dtype
359 + if (getattr(model, "quantization_method", None) is None
360 + and any(p.dtype != self.dtype
361 + for p in model.parameters())):
362 + model = model.to(dtype=self.dtype)
363 +
364 if (getattr(model, "quantization_method", None) != "bitsandbytes"
365 and len({p.device
366 for p in model.parameters()}) < 2):
361 - model = model.to(self.device)
367 + model = model.to(device=self.device)
368
369 self.model = model
370
tests/models/multimodal/generation/test_common.py
+26 -1
@@ -476,6 +476,31 @@ VLM_TEST_SETTINGS = {
476 max_num_seqs=2,
477 patch_hf_runner=model_utils.molmo_patch_hf_runner,
478 ),
479 + "ovis1_6-gemma2": VLMTestInfo(
480 + models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
481 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
482 + prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
483 + img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
484 + max_model_len=4096,
485 + max_num_seqs=2,
486 + dtype="half",
487 + # use sdpa mode for hf runner since ovis2 didn't work with flash_attn
488 + hf_model_kwargs={"llm_attn_implementation": "sdpa"},
489 + patch_hf_runner=model_utils.ovis_patch_hf_runner,
490 + marks=[large_gpu_mark(min_gb=32)],
491 + ),
492 + "ovis1_6": VLMTestInfo(
493 + models=["AIDC-AI/Ovis1.6-Llama3.2-3B"],
494 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
495 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful and honest multimodal assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
496 + img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
497 + max_model_len=4096,
498 + max_num_seqs=2,
499 + dtype="half",
500 + # use sdpa mode for hf runner since ovis2 didn't work with flash_attn
501 + hf_model_kwargs={"llm_attn_implementation": "sdpa"},
502 + patch_hf_runner=model_utils.ovis_patch_hf_runner,
503 + ),
504 "ovis2": VLMTestInfo(
505 models=["AIDC-AI/Ovis2-1B"],
506 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
@@ -486,7 +511,7 @@ VLM_TEST_SETTINGS = {
511 dtype="half",
512 # use sdpa mode for hf runner since ovis2 didn't work with flash_attn
513 hf_model_kwargs={"llm_attn_implementation": "sdpa"},
489 - patch_hf_runner=model_utils.ovis2_patch_hf_runner,
514 + patch_hf_runner=model_utils.ovis_patch_hf_runner,
515 ),
516 "phi3v": VLMTestInfo(
517 models=["microsoft/Phi-3.5-vision-instruct"],
tests/models/multimodal/generation/vlm_utils/model_utils.py
+11 -6
@@ -678,12 +678,8 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
678 return hf_model
679
680
681 -def ovis2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
681 +def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
682 """Patches and returns an instance of the HfRunner to use for Ovis2."""
683 - hf_model.model.visual_tokenizer.to(hf_model.dtype)
684 - hf_model.model.vte.to(hf_model.dtype)
685 - hf_model.model.llm.to(hf_model.dtype)
686 -
683 hf_model.model.get_output_embeddings = lambda: \
684 hf_model.model.llm.get_output_embeddings()
685
@@ -691,7 +687,16 @@ def ovis2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
687 text_tokenizer = hf_model.model.get_text_tokenizer()
688 images = [images] if isinstance(images, Image) else images
689
694 - text = text.split("<|im_start|>user\n")[1].split("<|im_end|>\n")[0]
690 + prompt_start_and_end = {
691 + "qwen2": ("<|im_start|>user\n", "<|im_end|>\n"),
692 + "llama":
693 + ("<|start_header_id|>user<|end_header_id|>\n\n", "<|eot_id|>"),
694 + "gemma2": ("<start_of_turn>user\n", "<end_of_turn>\n"),
695 + }
696 + for start, end in prompt_start_and_end.values():
697 + if start in text and end in text:
698 + text = text.split(start)[1].split(end)[0]
699 + break
700
701 prompt, input_ids, pixel_values = hf_model.model.preprocess_inputs(
702 text_or_conversations=text, images=images)
tests/models/multimodal/processing/test_common.py
+4 -1
@@ -146,7 +146,8 @@ def _test_processing_correctness_hf(
146 batch_idx: int,
147 ignore_mm_keys: Optional[set[str]] = None,
148 ):
149 - if model_config.hf_config.model_type in ("mllama", "whisper", "ultravox"):
149 + if model_config.hf_config.model_type in ("mllama", "ovis", "ultravox",
150 + "whisper"):
151 # For some multimodal models, tokenizer will always add bos_token
152 # at the beginning of prompt by default, causing hf_processor outputs
153 # incorrect token ids. So we need use `add_special_tokens=False` here
@@ -274,6 +275,8 @@ def _test_processing_correctness_mistral(
275 "allenai/Molmo-7B-D-0924",
276 "allenai/Molmo-7B-O-0924",
277 "nvidia/NVLM-D-72B",
278 + "AIDC-AI/Ovis1.6-Gemma2-9B",
279 + "AIDC-AI/Ovis1.6-Llama3.2-3B",
280 "AIDC-AI/Ovis2-1B",
281 "google/paligemma-3b-mix-224",
282 "google/paligemma2-3b-ft-docci-448",
tests/models/registry.py
+3 -3
@@ -355,9 +355,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
355 max_transformers_version="4.48",
356 transformers_version_reason="Use of deprecated imports which have been removed.", # noqa: E501
357 extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"}), # noqa: E501
358 - "Ovis2ForConditionalGeneration": _HfExamplesInfo("AIDC-AI/Ovis2-1B",
359 - trust_remote_code=True,
360 - hf_overrides={"architectures": ["Ovis2ForConditionalGeneration"]}), # noqa: E501
358 + "Ovis": _HfExamplesInfo("AIDC-AI/Ovis2-1B", trust_remote_code=True,
359 + extras={"1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
360 + "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B"}), # noqa: E501
361 "Phi4MMForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct",
362 trust_remote_code=True),
363 "PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
vllm/entrypoints/chat_utils.py
+1 -1
@@ -512,7 +512,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
512 hf_config.image_token_index)
513
514 if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",
515 - "internvl_chat", "ovis2", "skywork_chat",
515 + "internvl_chat", "ovis", "skywork_chat",
516 "NVLM_D", "h2ovl_chat", "idefics3", "smolvlm"):
517 return "<image>"
518 if model_type in ("mllama", "llama4"):
vllm/model_executor/models/aimv2.py
+2 -125
@@ -5,129 +5,14 @@
5 from typing import Optional
6
7 import torch
8 -from torch import nn, softmax
8 +import torch.nn as nn
9 from torch.nn import functional as F
10 -from torch.nn.functional import gumbel_softmax, pad
10
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.linear import ReplicatedLinear
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig)
16 -from vllm.transformers_utils.configs.ovis2 import (AIMv2Config,
17 - Aimv2VisualTokenizerConfig)
18 -
19 -IMAGE_INDICATOR_IDS = [-301, -302, -303, -304,
20 - -305] # kept for vocab prefixed tokens
21 -
22 -
23 -def st_argmax(y_soft: torch.Tensor, dim: int): # straight-through softmax
24 - index = y_soft.max(dim, keepdim=True)[1]
25 - y_hard = torch.zeros_like(
26 - y_soft, memory_format=torch.legacy_contiguous_format).scatter_(
27 - dim, index, 1.0)
28 - ret = y_hard - y_soft.detach() + y_soft
29 - return ret
30 -
31 -
32 -class Aimv2VisualTokenizer(torch.nn.Module):
33 -
34 - def __init__(self,
35 - config: Aimv2VisualTokenizerConfig,
36 - quant_config: Optional[QuantizationConfig] = None,
37 - prefix: str = "",
38 - **kwargs):
39 - super().__init__()
40 - self.config = config
41 - self.backbone = AIMv2Model(
42 - config=config.backbone_config, # noqa
43 - quant_config=quant_config,
44 - prefix=f"{prefix}.visual_tokenizer")
45 - # reserved tokens for IMAGE_INDICATORS
46 - head_dim = config.vocab_size - len(IMAGE_INDICATOR_IDS)
47 - self.head = torch.nn.Sequential(
48 - ReplicatedLinear(
49 - config.backbone_config.hidden_size * config.hidden_stride *
50 - config.hidden_stride,
51 - head_dim,
52 - bias=False,
53 - ), torch.nn.LayerNorm(head_dim))
54 -
55 - @property
56 - def dtype(self):
57 - return self.backbone.dtype
58 -
59 - @property
60 - def device(self):
61 - return self.backbone.device
62 -
63 - def tokenize(self, logits):
64 - if self.config.tokenize_function == 'softmax':
65 - tokens = softmax(logits, dim=-1)
66 - elif self.config.tokenize_function == 'gumbel_argmax':
67 - tokens = gumbel_softmax(logits, tau=self.config.tau, hard=True)
68 - elif self.config.tokenize_function == 'st_argmax':
69 - tokens = st_argmax(logits, dim=-1)
70 - else:
71 - raise ValueError(
72 - 'Invalid `max_type`, expected softmax or gumbel_argmax '
73 - f'or st_argmax, but got {self.config.tokenize_function}')
74 - return tokens
75 -
76 - def encode(self, pixel_values):
77 - features = self.backbone(pixel_values)
78 - if self.config.drop_cls_token:
79 - features = features[:, 1:, :]
80 -
81 - # merge number of `hidden_stride * hidden_stride` hidden states together
82 - # to reduce token sequence length
83 - # e.g., for hidden_stride=2, this leads to a token length reduction:
84 - # 1024 -> 256 for aimv2
85 - if self.config.hidden_stride > 1:
86 - # this `d` maybe different from the above `d``
87 - n, L, d = features.shape
88 - sqrt_l = int(L**0.5)
89 - assert sqrt_l**2 == L, (
90 - "The token sequence length should be a perfect square.")
91 - features = features.reshape(n, sqrt_l, sqrt_l, d)
92 - pl = (self.config.hidden_stride -
93 - (sqrt_l %
94 - self.config.hidden_stride)) % self.config.hidden_stride
95 - features = pad(features, (0, 0, 0, pl, 0, pl), "constant", 0)
96 - sqrt_l += pl
97 - features = features.reshape(n, sqrt_l // self.config.hidden_stride,
98 - self.config.hidden_stride,
99 - sqrt_l // self.config.hidden_stride,
100 - self.config.hidden_stride, d)
101 - # [n, sqrt_l/hs, sqrt_l/hs, hs, hs, d]
102 - features = features.permute(0, 1, 3, 2, 4, 5)
103 - # [n, sqrt_l/hs, sqrt_l/hs, hs*hs*d]
104 - features = features.flatten(3)
105 - # [n, sqrt_l/hs*sqrt_l/hs, hs*hs*d]
106 - features = features.reshape(
107 - n, -1,
108 - self.config.hidden_stride * self.config.hidden_stride * d)
109 -
110 - return features
111 -
112 - def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
113 - """[BatchSize, ImageShape] -> [BatchSize, Token, VocabSize]"""
114 - features = self.encode(pixel_values)
115 - logits, _ = self.head[0](
116 - features) # we spllit the sequncial here for not throwing an error
117 - logits = self.head[1](logits)
118 - tokens = self.tokenize(logits)
119 - # tokens' shape is [BatchSize, #Token, VocabSize-5], so padding with
120 - # [BatchSize, #Token, 5], after which, tokens' shape should become
121 - # [BatchSize, #Token, VocabSize]
122 - batch_size, token_len, _ = tokens.shape
123 - padding_tensor = torch.zeros(size=(batch_size, token_len,
124 - len(IMAGE_INDICATOR_IDS)),
125 - dtype=tokens.dtype,
126 - device=tokens.device,
127 - layout=tokens.layout,
128 - requires_grad=False)
129 - tokens = torch.cat((tokens, padding_tensor), dim=2)
130 - return tokens
15 +from vllm.transformers_utils.configs.ovis import AIMv2Config
16
17
18 class AIMv2SwiGLUFFN(nn.Module):
@@ -302,14 +187,6 @@ class AIMv2Model(torch.nn.Module):
187 quant_config=quant_config,
188 prefix=f"{prefix}.trunk")
189
305 - @property
306 - def dtype(self):
307 - return self.trunk.blocks[0].attn.qkv.weight.dtype
308 -
309 - @property
310 - def device(self):
311 - return self.trunk.blocks[0].attn.qkv.device
312 -
190 def forward(
191 self,
192 pixel_values: torch.Tensor,
vllm/model_executor/models/ovis.py renamed
+203 -37
@@ -15,17 +15,23 @@
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 -""" PyTorch Ovis2 model."""
18 +""" PyTorch Ovis model."""
19 +import math
20 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
21 TypedDict, Union)
22
23 import torch
24 import torch.nn as nn
25 from torch import Tensor
25 -from transformers import BatchFeature
26 +from torch.nn.functional import gumbel_softmax, pad, softmax
27 +from transformers import BaseImageProcessor, BatchFeature
28
29 from vllm.config import VllmConfig
28 -from vllm.model_executor.models.aimv2 import Aimv2VisualTokenizer
30 +from vllm.model_executor.layers.linear import ReplicatedLinear
31 +from vllm.model_executor.layers.quantization.base_config import (
32 + QuantizationConfig)
33 +from vllm.model_executor.models.aimv2 import AIMv2Model
34 +from vllm.model_executor.models.siglip import SiglipVisionModel
35 from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,
36 init_vllm_registered_model,
37 maybe_prefix)
@@ -38,19 +44,160 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
44 BaseProcessingInfo, PromptReplacement)
45 from vllm.multimodal.profiling import BaseDummyInputsBuilder
46 from vllm.sequence import IntermediateTensors
41 -from vllm.transformers_utils.configs.ovis2 import OvisConfig
42 -from vllm.transformers_utils.processors.ovis2 import OvisProcessor
47 +from vllm.transformers_utils.configs.ovis import (BaseVisualTokenizerConfig,
48 + OvisConfig)
49 +from vllm.transformers_utils.processors.ovis import OvisProcessor
50
51 from .interfaces import MultiModalEmbeddings, SupportsMultiModal
52 from .utils import merge_multimodal_embeddings
53
54 # Cannot find the following number from hf config.
55 IMAGE_TOKEN = "<image>"
49 -IMAGE_PAD_TOKEN_ID = 151655
50 -NUMBER_OF_TOKEN_TO_RESERVE_FOR_SEGMENT = 256
56 +IMAGE_INDICATOR_IDS = [-301, -302, -303, -304, -305]
57
58 +IMAGE_PAD_TOKEN_MAP = {
59 + "gemma2": "<unused0>",
60 + "llama": "<|reserved_special_token_0|>",
61 + "qwen2": "<|image_pad|>",
62 +}
63 +IMAGE_PAD_TOKEN_ID_MAP = {
64 + "gemma2": 7,
65 + "llama": 128002,
66 + "qwen2": 151655,
67 +}
68
53 -class Ovis2ImagePatchInputs(TypedDict):
69 +
70 +def st_argmax(y_soft: torch.Tensor, dim: int): # straight-through softmax
71 + index = y_soft.argmax(dim, keepdim=True)
72 + return torch.zeros_like(
73 + y_soft,
74 + memory_format=torch.legacy_contiguous_format,
75 + ).scatter_(dim, index, 1.0)
76 +
77 +
78 +class VisualTokenizer(torch.nn.Module):
79 +
80 + def __init__(
81 + self,
82 + config: BaseVisualTokenizerConfig,
83 + quant_config: Optional[QuantizationConfig] = None,
84 + prefix: str = "",
85 + ):
86 + super().__init__()
87 + self.config = config
88 + self.backbone = self._init_backbone(
89 + config=config,
90 + quant_config=quant_config,
91 + prefix=f"{prefix}.backbone",
92 + )
93 + # reserved tokens for IMAGE_INDICATORS
94 + head_dim = config.vocab_size - len(IMAGE_INDICATOR_IDS)
95 + self.head = torch.nn.Sequential(
96 + ReplicatedLinear(
97 + config.backbone_config.hidden_size * config.hidden_stride *
98 + config.hidden_stride,
99 + head_dim,
100 + bias=False,
101 + return_bias=False,
102 + ), torch.nn.LayerNorm(head_dim))
103 +
104 + def _init_backbone(
105 + self,
106 + config: BaseVisualTokenizerConfig,
107 + quant_config: Optional[QuantizationConfig] = None,
108 + prefix: str = "",
109 + ):
110 + model_type = config.backbone_config.model_type
111 + if model_type == "aimv2":
112 + return AIMv2Model(
113 + config=config.backbone_config,
114 + quant_config=quant_config,
115 + prefix=prefix,
116 + )
117 + elif model_type == "siglip_vision_model":
118 + return SiglipVisionModel(
119 + config=config.backbone_config,
120 + quant_config=quant_config,
121 + prefix=prefix,
122 + )
123 + raise ValueError(
124 + f"Unsupported visual tokenizer model_type: {model_type}")
125 +
126 + @property
127 + def dtype(self):
128 + return next(self.head.parameters()).dtype
129 +
130 + @property
131 + def device(self):
132 + return next(self.head.parameters()).device
133 +
134 + def tokenize(self, logits):
135 + if self.config.tokenize_function == 'softmax':
136 + tokens = softmax(logits, dim=-1)
137 + elif self.config.tokenize_function == 'gumbel_argmax':
138 + tokens = gumbel_softmax(logits, tau=self.config.tau, hard=True)
139 + elif self.config.tokenize_function == 'st_argmax':
140 + tokens = st_argmax(logits, dim=-1)
141 + else:
142 + raise ValueError(
143 + 'Invalid `max_type`, expected softmax or gumbel_argmax '
144 + f'or st_argmax, but got {self.config.tokenize_function}')
145 + return tokens
146 +
147 + def encode(self, pixel_values):
148 + features = self.backbone(pixel_values)
149 + if self.config.drop_cls_token:
150 + features = features[:, 1:, :]
151 +
152 + # merge number of `hidden_stride * hidden_stride` hidden states together
153 + # to reduce token sequence length
154 + # e.g., for hidden_stride=2, this leads to a token length reduction:
155 + # 1024 -> 256 for aimv2
156 + if self.config.hidden_stride > 1:
157 + # this `d` maybe different from the above `d``
158 + n, L, d = features.shape
159 + sqrt_l = int(L**0.5)
160 + assert sqrt_l**2 == L, (
161 + "The token sequence length should be a perfect square.")
162 + features = features.reshape(n, sqrt_l, sqrt_l, d)
163 + pl = (self.config.hidden_stride -
164 + (sqrt_l %
165 + self.config.hidden_stride)) % self.config.hidden_stride
166 + features = pad(features, (0, 0, 0, pl, 0, pl), "constant", 0)
167 + sqrt_l += pl
168 + features = features.reshape(n, sqrt_l // self.config.hidden_stride,
169 + self.config.hidden_stride,
170 + sqrt_l // self.config.hidden_stride,
171 + self.config.hidden_stride, d)
172 + # [n, sqrt_l/hs, sqrt_l/hs, hs, hs, d]
173 + features = features.permute(0, 1, 3, 2, 4, 5)
174 + # [n, sqrt_l/hs, sqrt_l/hs, hs*hs*d]
175 + features = features.flatten(3)
176 + # [n, sqrt_l/hs*sqrt_l/hs, hs*hs*d]
177 + features = features.reshape(
178 + n, -1,
179 + self.config.hidden_stride * self.config.hidden_stride * d)
180 +
181 + return features
182 +
183 + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
184 + """[BatchSize, ImageShape] -> [BatchSize, Token, VocabSize]"""
185 + features = self.encode(pixel_values)
186 + logits = self.head(features)
187 + tokens = self.tokenize(logits)
188 + # tokens' shape is [BatchSize, #Token, VocabSize-5], so padding with
189 + # [BatchSize, #Token, 5], after which, tokens' shape should become
190 + # [BatchSize, #Token, VocabSize]
191 + tokens = torch.nn.functional.pad(
192 + tokens,
193 + (0, len(IMAGE_INDICATOR_IDS)),
194 + mode="constant",
195 + value=0,
196 + )
197 + return tokens
198 +
199 +
200 +class OvisImagePatchInputs(TypedDict):
201 type: Literal["image_patches"]
202 flat_data: torch.Tensor
203 """
@@ -92,31 +239,50 @@ class VisualEmbedding(torch.nn.Embedding):
239 return self.weight.dtype
240
241
95 -class Ovis2ProcessingInfo(BaseProcessingInfo):
242 +class OvisProcessingInfo(BaseProcessingInfo):
243
244 def get_hf_config(self):
245 return self.ctx.get_hf_config(OvisConfig)
246
247 def get_hf_processor(self, **kwargs):
101 - return self.ctx.get_hf_processor(OvisProcessor)
248 + return self.ctx.get_hf_processor(
249 + OvisProcessor,
250 + image_pad_token=self.get_image_pad_token(),
251 + image_segment_len=self.get_image_segment_len(),
252 + )
253
103 - def get_image_processor(self) -> OvisProcessor:
254 + def get_image_segment_len(self) -> int:
255 + visual_tokenizer_config = self.get_hf_config().visual_tokenizer_config
256 + image_size = visual_tokenizer_config.backbone_config.image_size
257 + patch_size = visual_tokenizer_config.backbone_config.patch_size
258 + hidden_stride = visual_tokenizer_config.hidden_stride
259 + patch_grid_length = math.ceil(image_size / patch_size)
260 + assert patch_grid_length % hidden_stride == 0, (
261 + f"patch_grid_length {patch_grid_length} is not divisible by "
262 + f"hidden_stride {hidden_stride}")
263 + # minus 1 for presented image token
264 + return (patch_grid_length // hidden_stride)**2 - 1
265 +
266 + def get_image_pad_token(self) -> str:
267 + hf_text_config = self.get_hf_config().get_text_config()
268 + text_model_type = hf_text_config.model_type
269 + return IMAGE_PAD_TOKEN_MAP.get(text_model_type)
270 +
271 + def get_image_processor(self) -> BaseImageProcessor:
272 return self.get_hf_processor().image_processor # type: ignore
273
274 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
107 - return { # 32k is model token limit at the moment
108 - "image":
109 - self.get_hf_config().multimodal_max_length //
110 - ((9 + 1) * NUMBER_OF_TOKEN_TO_RESERVE_FOR_SEGMENT)
111 - }
275 + return {"image": None}
276
277 def get_image_size_with_most_features(self) -> ImageSize:
114 - image_processor = self.get_image_processor()
115 - return ImageSize(width=image_processor.size['shortest_edge'] * 9 * 2,
116 - height=image_processor.size['shortest_edge'] * 9 * 2)
278 + height, width = self.get_hf_processor().get_image_size()
279 + hs = self.get_hf_config().visual_tokenizer_config.hidden_stride
280 + # NOTE(Isotr0py): 9 is `max_partion` hardcoded in original code
281 + # https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/modeling_ovis.py#L96
282 + return ImageSize(width=width * hs * 9, height=height * hs * 9)
283
284
119 -class Ovis2DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2ProcessingInfo]):
285 +class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
286
287 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
288 num_images = mm_counts.get("image", 0)
@@ -141,7 +307,7 @@ class Ovis2DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2ProcessingInfo]):
307 return mm_data
308
309
144 -class Ovis2MultiModalProcessor(BaseMultiModalProcessor[Ovis2ProcessingInfo]):
310 +class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
311
312 def image_indicators_to_visual_tokens(
313 self,
@@ -165,9 +331,9 @@ class Ovis2MultiModalProcessor(BaseMultiModalProcessor[Ovis2ProcessingInfo]):
331 mm_kwargs: Mapping[str, object],
332 ) -> BatchFeature:
333 if not mm_data:
168 - # # Avoid warning from HF logger for text-only input
169 - prompt_ids = self.info.get_tokenizer().encode(prompt)
170 - # prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) nope
334 + # Avoid warning from HF logger for text-only input
335 + tokenizer = self.info.get_tokenizer()
336 + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
337 return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
338
339 processed_outputs = super()._call_hf_processor(
@@ -226,10 +392,10 @@ class Ovis2MultiModalProcessor(BaseMultiModalProcessor[Ovis2ProcessingInfo]):
392 ]
393
394
229 -@MULTIMODAL_REGISTRY.register_processor(Ovis2MultiModalProcessor,
230 - info=Ovis2ProcessingInfo,
231 - dummy_inputs=Ovis2DummyInputsBuilder)
232 -class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):
395 +@MULTIMODAL_REGISTRY.register_processor(OvisMultiModalProcessor,
396 + info=OvisProcessingInfo,
397 + dummy_inputs=OvisDummyInputsBuilder)
398 +class Ovis(nn.Module, SupportsMultiModal):
399
400 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
401 super().__init__()
@@ -242,24 +408,25 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):
408 prefix=maybe_prefix(prefix, "llm"),
409 )
410
245 - self.visual_tokenizer = Aimv2VisualTokenizer(
411 + self.visual_tokenizer = VisualTokenizer(
412 config=config.visual_tokenizer_config,
413 quant_config=quant_config,
414 prefix=f"{prefix}.visual_tokenizer",
249 - image_processor_name_or_path=config.visual_tokenizer_config.
250 - backbone_config.name_or_path,
415 )
416
417 self.vte = VisualEmbedding(
418 self.config.visual_tokenizer_config.vocab_size,
419 self.config.hidden_size)
420
421 + text_model_type = self.config.get_text_config().model_type
422 + self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
423 +
424 # TODO(Isotr0py): PP support
425 # self.make_empty_intermediate_tensors = (
426 # self.language_model.make_empty_intermediate_tensors)
427
428 def _parse_and_validate_image_input(
262 - self, **kwargs: object) -> Optional[Ovis2ImagePatchInputs]:
429 + self, **kwargs: object) -> Optional[OvisImagePatchInputs]:
430 pixel_values = kwargs.pop("pixel_values", None)
431 indicator_tokens = kwargs.pop("indicator_tokens", None)
432
@@ -275,7 +442,7 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):
442 raise ValueError("Incorrect type of indicator_tokens. "
443 f"Got type: {type(pixel_values)}")
444
278 - return Ovis2ImagePatchInputs(
445 + return OvisImagePatchInputs(
446 type="image_patches",
447 flat_data=flatten_bn(flatten_bn(pixel_values), concat=True),
448 patches_per_image=[
@@ -288,7 +455,7 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):
455 raise AssertionError("This line should be unreachable.")
456
457 def _process_image_input(
291 - self, image_input: Ovis2ImagePatchInputs) -> MultiModalEmbeddings:
458 + self, image_input: OvisImagePatchInputs) -> MultiModalEmbeddings:
459 image_patches_flat = image_input["flat_data"]
460 patches_per_image = image_input["patches_per_image"]
461 indicator_tokens = image_input["indicator_tokens"]
@@ -338,7 +505,7 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):
505 if multimodal_embeddings is not None:
506 inputs_embeds = merge_multimodal_embeddings(
507 input_ids, inputs_embeds, multimodal_embeddings,
341 - [IMAGE_PAD_TOKEN_ID])
508 + self.image_pad_token_id)
509 return inputs_embeds
510
511 def forward(
@@ -375,8 +542,7 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):
542 hidden_states: torch.Tensor,
543 sampling_metadata: SamplingMetadata,
544 ) -> Optional[torch.Tensor]:
378 - logits = self.llm.logits_processor(self.llm.lm_head, hidden_states,
379 - sampling_metadata)
545 + logits = self.llm.compute_logits(hidden_states, sampling_metadata)
546 return logits
547
548 def load_weights(self, weights: Iterable[Tuple[str,
vllm/model_executor/models/registry.py
+1 -1
@@ -195,7 +195,7 @@ _MULTIMODAL_MODELS = {
195 "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"), # noqa: E501
196 "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
197 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
198 - "Ovis2ForConditionalGeneration": ("ovis2", "Ovis2ForConditionalGeneration"),
198 + "Ovis": ("ovis", "Ovis"),
199 "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
200 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
201 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
vllm/transformers_utils/configs/__init__.py
+1 -1
@@ -23,7 +23,7 @@ from vllm.transformers_utils.configs.moonvit import MoonViTConfig
23 from vllm.transformers_utils.configs.mpt import MPTConfig
24 from vllm.transformers_utils.configs.nemotron import NemotronConfig
25 from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
26 -from vllm.transformers_utils.configs.ovis2 import OvisConfig
26 +from vllm.transformers_utils.configs.ovis import OvisConfig
27 from vllm.transformers_utils.configs.skyworkr1v import SkyworkR1VChatConfig
28 from vllm.transformers_utils.configs.solar import SolarConfig
29 from vllm.transformers_utils.configs.telechat2 import Telechat2Config
vllm/transformers_utils/configs/ovis.py renamed
+13
@@ -123,6 +123,19 @@ class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):
123 self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
124
125
126 +class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):
127 + model_type = "siglip_visual_tokenizer"
128 +
129 + def __init__(self, **kwargs):
130 + super().__init__(**kwargs)
131 + if self.drop_cls_token:
132 + self.drop_cls_token = False
133 + if self.depths:
134 + assert len(self.depths) == 1
135 + self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
136 +
137 +
138 +AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig)
139 AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)
140
141
vllm/transformers_utils/processors/__init__.py
+1 -1
@@ -2,6 +2,6 @@
2
3 from vllm.transformers_utils.processors.deepseek_vl2 import (
4 DeepseekVLV2Processor)
5 -from vllm.transformers_utils.processors.ovis2 import OvisProcessor
5 +from vllm.transformers_utils.processors.ovis import OvisProcessor
6
7 __all__ = ["DeepseekVLV2Processor", "OvisProcessor"]
vllm/transformers_utils/processors/ovis.py renamed
+30 -12
@@ -22,6 +22,7 @@
22 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23 # See the License for the specific language governing permissions and
24 # limitations under the License.
25 +from functools import cached_property
26 from typing import List, Union
27
28 import PIL
@@ -32,7 +33,7 @@ from transformers.processing_utils import (ProcessingKwargs, ProcessorMixin,
33 Unpack)
34 from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
35
35 -__all__ = [ 'OvisProcessor']
36 +__all__ = ['OvisProcessor']
37 IGNORE_ID = -100
38
39 class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
@@ -64,18 +65,29 @@ class OvisProcessor(ProcessorMixin):
65 """
66
67 attributes = ["image_processor", "tokenizer"]
67 - valid_kwargs = ["chat_template"]
68 + valid_kwargs = ["chat_template", "image_pad_token", "image_segement_len"]
69
70 image_processor_class = "AutoImageProcessor"
70 - tokenizer_class = "Qwen2Tokenizer"
71 + tokenizer_class = "AutoTokenizer"
72
72 - def __init__(self, image_processor=None, tokenizer=None, chat_template=None, image_pad_token=None, **kwargs):
73 + def __init__(
74 + self,
75 + image_processor=None,
76 + tokenizer=None,
77 + chat_template=None,
78 + image_pad_token=None,
79 + image_segment_len=255,
80 + **kwargs,
81 + ):
82 self.image_token = "<image>"
74 - self.image_pad_token = "<|image_pad|>" if image_pad_token is None else image_pad_token
83 + self.image_pad_token = image_pad_token
84 + self.image_segment_len = image_segment_len
85 super().__init__(image_processor, tokenizer, chat_template=chat_template)
86
77 - self.image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]
78 - self.extra_special_tokens = {
87 + @cached_property
88 + def extra_special_tokens(self):
89 + image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]
90 + extra_special_tokens = {
91 "image_token": -200,
92 "image_atom": -300,
93 "image_start": -301,
@@ -83,8 +95,9 @@ class OvisProcessor(ProcessorMixin):
95 "image_col_sep": -303,
96 "image_row_sep": -304,
97 "image_end": -305,
86 - 'image_pad': self.image_pad_token_id,
98 + 'image_pad': image_pad_token_id,
99 }
100 + return extra_special_tokens
101
102 def __call__(
103 self,
@@ -224,8 +237,14 @@ class OvisProcessor(ProcessorMixin):
237 return torch.tensor(batch_token_ids, dtype=torch.long)
238
239 def get_image_size(self):
227 - height = self.image_processor.crop_size["height"]
228 - width = self.image_processor.crop_size["width"]
240 + size = self.image_processor.size
241 + if 'shortest_edge' in size:
242 + width = height = size['shortest_edge']
243 + elif "height" in size and "width" in size:
244 + width = size['width']
245 + height = size['height']
246 + else:
247 + raise ValueError( "Can't parse image size from image_processor config.")
248 return height, width
249
250 def get_token_value(self, tok):
@@ -259,8 +278,7 @@ class OvisProcessor(ProcessorMixin):
278 for token in image_placeholders:
279 padded_placeholder_tokens.append(image_padding_token_id)
280 if token == image_atom_token_id:
262 - # Add 255 padding tokens after each image atom token
263 - padded_placeholder_tokens.extend([image_padding_token_id] * 255)
281 + padded_placeholder_tokens.extend([image_padding_token_id] * self.image_segment_len)
282 return padded_placeholder_tokens
283
284 def preprocess_image(self, image: PIL.Image.Image, max_partition, covering_threshold, convert_to_rgb, return_tensors):