@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b87cb97a53

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Model] support new model ovis2.5 (#23084)

Signed-off-by: myselvess <244285088@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

myselvess committed Aug 19, 2025 at 21:12 UTC b87cb97a53bcff92a90308528b3f313e43aff102

12 files changed +1787 -1

docs/models/supported_models.md

@@ -641,6 +641,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

641 | `MolmoForCausalLM` | Molmo | T + I+ | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ | ✅︎ |

642 | `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ | ✅︎ |

643 | `Ovis` | Ovis2, Ovis1.6 | T + I+ | `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis1.6-Llama3.2-3B`, etc. | | ✅︎ | ✅︎ |

644 +| `Ovis2_5` | Ovis2.5 | T + I+ + V | `AIDC-AI/Ovis2.5-9B`, etc. | | | ✅︎ |

645 | `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + IE | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | | ✅︎ | ⚠️ |

646 | `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + IE+ | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ | ✅︎ |

647 | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |

examples/offline_inference/vision_language.py

+33

@@ -1105,6 +1105,38 @@ def run_ovis(questions: list[str], modality: str) -> ModelRequestData:

1105 )

1106

1107

1108 +# Ovis2_5

1109 +def run_ovis2_5(questions: list[str], modality: str) -> ModelRequestData:

1110 + model_name = "AIDC-AI/Ovis2.5-2B"

1111 +

1112 + engine_args = EngineArgs(

1113 + model=model_name,

1114 + max_model_len=4096,

1115 + max_num_seqs=2,

1116 + trust_remote_code=True,

1117 + dtype="half",

1118 + limit_mm_per_prompt={modality: 1},

1119 + )

1120 + if modality == "image":

1121 + placeholder = "<image>"

1122 + elif modality == "video":

1123 + placeholder = "<video>"

1124 +

1125 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

1126 + messages = [

1127 + [{"role": "user", "content": f"{placeholder}\n{question}"}]

1128 + for question in questions

1129 + ]

1130 + prompts = tokenizer.apply_chat_template(

1131 + messages, tokenize=False, add_generation_prompt=True

1132 + )

1133 +

1134 + return ModelRequestData(

1135 + engine_args=engine_args,

1136 + prompts=prompts,

1137 + )

1138 +

1139 +

1140 # PaliGemma

1141 def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:

1142 assert modality == "image"

@@ -1579,6 +1611,7 @@ model_example_map = {

1611 "nemotron_vl": run_nemotron_vl,

1612 "NVLM_D": run_nvlm_d,

1613 "ovis": run_ovis,

1614 + "ovis2_5": run_ovis2_5,

1615 "paligemma": run_paligemma,

1616 "paligemma2": run_paligemma2,

1617 "phi3_v": run_phi3v,

examples/offline_inference/vision_language_multi_image.py

+31

@@ -680,6 +680,36 @@ def load_ovis(question: str, image_urls: list[str]) -> ModelRequestData:

680 )

681

682

683 +# ovis2_5

684 +def load_ovis2_5(question: str, image_urls: list[str]) -> ModelRequestData:

685 + model_name = "AIDC-AI/Ovis2.5-2B"

686 +

687 + engine_args = EngineArgs(

688 + model=model_name,

689 + max_model_len=8192,

690 + max_num_seqs=2,

691 + trust_remote_code=True,

692 + dtype="half",

693 + limit_mm_per_prompt={"image": len(image_urls)},

694 + )

695 +

696 + placeholders = "\n".join(

697 + f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)

698 + )

699 + messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]

700 +

701 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

702 + prompt = tokenizer.apply_chat_template(

703 + messages, tokenize=False, add_generation_prompt=True

704 + )

705 +

706 + return ModelRequestData(

707 + engine_args=engine_args,

708 + prompt=prompt,

709 + image_data=[fetch_image(url) for url in image_urls],

710 + )

711 +

712 +

713 def load_pixtral_hf(question: str, image_urls: list[str]) -> ModelRequestData:

714 model_name = "mistral-community/pixtral-12b"

715

@@ -1155,6 +1185,7 @@ model_example_map = {

1185 "mllama": load_mllama,

1186 "NVLM_D": load_nvlm_d,

1187 "ovis": load_ovis,

1188 + "ovis2_5": load_ovis2_5,

1189 "phi3_v": load_phi3v,

1190 "phi4_mm": load_phi4mm,

1191 "phi4_multimodal": load_phi4_multimodal,

tests/models/multimodal/generation/test_common.py

+21

@@ -11,6 +11,7 @@ from pathlib import PosixPath

11 import pytest

12 from transformers import (AutoModel, AutoModelForImageTextToText,

13 AutoModelForTextToWaveform, AutoModelForVision2Seq)

14 +from transformers.utils import is_flash_attn_2_available

16 from vllm.platforms import current_platform

17 from vllm.utils import identity

@@ -621,6 +622,26 @@ VLM_TEST_SETTINGS = {

622 hf_model_kwargs={"llm_attn_implementation": "sdpa"},

623 patch_hf_runner=model_utils.ovis_patch_hf_runner,

624 ),

625 + "ovis2_5": VLMTestInfo(

626 + models=["AIDC-AI/Ovis2.5-2B"],

627 + test_type=(

628 + VLMTestType.IMAGE,

629 + VLMTestType.MULTI_IMAGE,

630 + VLMTestType.VIDEO

631 + ),

632 + prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501

633 + img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501

634 + video_idx_to_prompt=lambda idx: "<video>\n",

635 + max_model_len=4096,

636 + max_num_seqs=2,

637 + dtype="half",

638 + num_logprobs=10,

639 + patch_hf_runner=model_utils.ovis2_5_patch_hf_runner,

640 + marks=[pytest.mark.skipif(

641 + not is_flash_attn_2_available(),

642 + reason="HF model needs `flash_attn` installed"

643 + )],

644 + ),

645 "phi3v": VLMTestInfo(

646 models=["microsoft/Phi-3.5-vision-instruct"],

647 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

tests/models/multimodal/generation/vlm_utils/model_utils.py

+58

@@ -10,6 +10,7 @@ from typing import Optional, Union

11 import numpy as np

12 import numpy.typing as npt

13 +import PIL.Image

14 import pytest

15 import regex as re

16 import torch

@@ -810,6 +811,63 @@ def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

811 return hf_model

812

813

814 +def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

815 + """Patches and returns an instance of the HfRunner to use for Ovis2."""

816 + hf_model.model.get_output_embeddings = lambda: \

817 + hf_model.model.llm.get_output_embeddings()

818 +

819 + def processor(*args, text="", images=None, videos=None, **kwargs):

820 + if images is None:

821 + images = []

822 + else:

823 + images = [images] if isinstance(images, Image) else images

824 + if videos is None:

825 + videos = []

826 + else:

827 + videos = [videos] if isinstance(videos, np.ndarray) else videos

828 + videos = [[PIL.Image.fromarray(frame) for frame in vid]

829 + for vid in videos]

830 +

831 + prompt_start_and_end = {

832 + "qwen2": ("<|im_start|>user\n", "<|im_end|>\n"),

833 + "llama":

834 + ("<|start_header_id|>user<|end_header_id|>\n\n", "<|eot_id|>"),

835 + "gemma2": ("<start_of_turn>user\n", "<end_of_turn>\n"),

836 + }

837 + for start, end in prompt_start_and_end.values():

838 + if start in text and end in text:

839 + text = text.split(start)[1].split(end)[0]

840 + break

841 +

842 + images_message = [{"type": "image", "image": img} for img in images]

843 + videos_message = [{"type": "video", "video": vid} for vid in videos]

844 +

845 + messages = [{

846 + "role":

847 + "user",

848 + "content": [

849 + *images_message,

850 + *videos_message,

851 + {

852 + "type": "text",

853 + "text": text

854 + },

855 + ],

856 + }]

857 +

858 + input_ids, pixel_values, grid_thws = hf_model.model.preprocess_inputs(

859 + messages=messages, enable_thinking=True)

860 + inputs = {

861 + "inputs": input_ids,

862 + "pixel_values": pixel_values,

863 + "grid_thws": grid_thws,

864 + }

865 + return BatchFeature(data=inputs, tensor_type="pt")

866 +

867 + hf_model.processor = processor

868 + return hf_model

869 +

870 +

871 def qwen2_5_omni_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

872 """Patches and returns an instance of the HfRunner for Qwen2.5-Omni."""

873 thinker = hf_model.model.thinker

tests/models/multimodal/processing/test_common.py

@@ -162,6 +162,7 @@ def _test_processing_correctness(

162 _ADD_SPECIAL_TOKENS_OVERRIDES = {

163 "mllama": False,

164 "ovis": False,

165 + "ovis2_5": False,

166 "paligemma": False,

167 "ultravox": False,

168 "whisper": False,

@@ -301,6 +302,7 @@ def _test_processing_correctness_one(

302 "AIDC-AI/Ovis1.6-Gemma2-9B",

303 "AIDC-AI/Ovis1.6-Llama3.2-3B",

304 "AIDC-AI/Ovis2-1B",

305 + "AIDC-AI/Ovis2.5-2B",

306 "google/paligemma-3b-mix-224",

307 "google/paligemma2-3b-ft-docci-448",

308 "microsoft/Phi-3.5-vision-instruct",

tests/models/registry.py

@@ -464,6 +464,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {

464 transformers_version_reason="HF model is not compatible", # noqa: E501

465 extras={"1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",

466 "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B"}), # noqa: E501

467 + "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True,

468 + max_transformers_version="4.53",

469 + transformers_version_reason="HF model is not compatible"), # noqa: E501

470 "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-mix-224", # noqa: E501

471 extras={"v2": "google/paligemma2-3b-ft-docci-448"}), # noqa: E501

472 "Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",

vllm/model_executor/models/ovis2_5.py new

+570

@@ -0,0 +1,570 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +""" PyTorch Ovis model."""

4 +from collections.abc import Iterable, Mapping

5 +from functools import partial

6 +from typing import Optional, Union

7 +

8 +import torch

9 +import torch.nn as nn

10 +from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

11 +

12 +from vllm.config import VllmConfig

13 +from vllm.model_executor.layers.linear import ReplicatedLinear

14 +from vllm.model_executor.layers.quantization.base_config import (

15 + QuantizationConfig)

16 +from vllm.model_executor.models.ovis import (OvisImagePatchInputs,

17 + VisualEmbedding)

18 +from vllm.model_executor.models.siglip2navit import Siglip2NavitModel

19 +from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,

20 + init_vllm_registered_model,

21 + maybe_prefix)

22 +from vllm.model_executor.sampling_metadata import SamplingMetadata

23 +from vllm.multimodal import MULTIMODAL_REGISTRY

24 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

25 + MultiModalKwargsItems)

26 +from vllm.multimodal.parse import ImageSize, MultiModalDataItems

27 +from vllm.multimodal.processing import (BaseMultiModalProcessor,

28 + BaseProcessingInfo, PromptReplacement)

29 +from vllm.multimodal.profiling import BaseDummyInputsBuilder

30 +from vllm.sequence import IntermediateTensors

31 +from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor

32 +

33 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal

34 +

35 +IMAGE_TOKEN = "<image>"

36 +VIDEO_TOKEN = "<video>"

37 +INDICATOR_IDS = [-301, -302, -303, -304]

38 +

39 +IMAGE_PAD_TOKEN_MAP = {

40 + "gemma2": "<unused0>",

41 + "llama": "<|reserved_special_token_0|>",

42 + "qwen2": "<|image_pad|>",

43 + "qwen3": "<|image_pad|>",

44 +}

45 +IMAGE_PAD_TOKEN_ID_MAP = {

46 + "gemma2": 7,

47 + "llama": 128002,

48 + "qwen2": 151655,

49 + "qwen3": 151655,

50 +}

51 +

52 +

53 +def _ovis2_5_field_config():

54 + return dict(pixel_values=MultiModalFieldConfig.batched("image"),

55 + grids=MultiModalFieldConfig.batched("image"),

56 + indicator_tokens=MultiModalFieldConfig.batched("image"),

57 + video_pixel_values=MultiModalFieldConfig.batched("video"),

58 + video_indicator_tokens=MultiModalFieldConfig.batched("video"),

59 + video_grids=MultiModalFieldConfig.batched("video"))

60 +

61 +

62 +class VisualTokenizer(torch.nn.Module):

63 + """

64 + VIT

65 + """

66 +

67 + def __init__(

68 + self,

69 + config: PretrainedConfig,

70 + visual_vocab_size: int,

71 + quant_config: Optional[QuantizationConfig] = None,

72 + prefix: str = "",

73 + ):

74 + super().__init__()

75 + self.config = config

76 + self.vit = self._init_backbone(

77 + config=config,

78 + quant_config=quant_config,

79 + prefix=f"{prefix}.vit",

80 + )

81 + # reserved tokens for INDICATOR_IDS

82 + head_dim = visual_vocab_size - len(INDICATOR_IDS)

83 + self.head = torch.nn.Sequential(

84 + ReplicatedLinear(

85 + self.config.hidden_size * self.config.hidden_stride**2,

86 + head_dim,

87 + bias=False,

88 + return_bias=False,

89 + ), torch.nn.LayerNorm(head_dim))

90 +

91 + def _init_backbone(

92 + self,

93 + config: PretrainedConfig,

94 + quant_config: Optional[QuantizationConfig] = None,

95 + prefix: str = "",

96 + ):

97 + model_type = config.model_type

98 + if model_type == "siglip2_navit":

99 + return Siglip2NavitModel(config=config, )

100 + raise ValueError(

101 + f"Unsupported visual tokenizer model_type: {model_type}")

102 +

103 + @property

104 + def dtype(self):

105 + return next(self.head.parameters()).dtype

106 +

107 + @property

108 + def device(self):

109 + return next(self.head.parameters()).device

110 +

111 + def tokenize(self, logits):

112 + tokens = torch.softmax(logits, dim=-1,

113 + dtype=torch.float32).to(logits.dtype)

114 + return tokens

115 +

116 + def encode(self, pixel_values, grid_thws):

117 + features = self.vit(pixel_values,

118 + grid_thws,

119 + output_hidden_states=True,

120 + return_dict=True)

121 + # refer to qwen2.5-vl patchmerger

122 + seq_len, _ = features.shape

123 + features = features.reshape(seq_len // (self.config.hidden_stride**2),

124 + -1)

125 +

126 + return features

127 +

128 + def forward(self, pixel_values, grid_thws) -> torch.Tensor:

129 + features = self.encode(pixel_values, grid_thws)

130 + logits = self.head(features)

131 + tokens = self.tokenize(logits)

132 + # tokens' shape is [#Token, VocabSize-4],

133 + # so padding with [#Token, 4], after which,

134 + # tokens' shape should become [#Token, VocabSize];

135 + tokens = torch.nn.functional.pad(

136 + tokens,

137 + (0, len(INDICATOR_IDS)),

138 + mode="constant",

139 + value=0,

140 + )

141 + return tokens

142 +

143 +

144 +class Ovis2_5ProcessingInfo(BaseProcessingInfo):

145 +

146 + def get_hf_config(self):

147 + return self.ctx.get_hf_config()

148 +

149 + def get_hf_processor(self, **kwargs):

150 + vit_config = self.get_hf_config().vit_config

151 + return self.ctx.get_hf_processor(

152 + Ovis2_5Processor,

153 + image_pad_token=self.get_image_pad_token(),

154 + patch_size=vit_config.patch_size,

155 + hidden_stride=vit_config.hidden_stride,

156 + temporal_patch_size=vit_config.temporal_patch_size,

157 + )

158 +

159 + def get_image_pad_token(self) -> str:

160 + hf_text_config = self.get_hf_config().get_text_config()

161 + text_model_type = hf_text_config.model_type

162 + return IMAGE_PAD_TOKEN_MAP.get(text_model_type)

163 +

164 + def get_image_processor(self) -> BaseImageProcessor:

165 + return self.get_hf_processor().image_processor # type: ignore

166 +

167 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:

168 + return {"image": None, "video": 1}

169 +

170 + def get_image_size_with_most_features(self) -> ImageSize:

171 + # NOTE(myselvess): max_pixels 1792 * 1792 hardcoded in original code

172 + # TODO(myselvess): Be adjusted based on the max_pixels

173 + return ImageSize(width=1792, height=1792)

174 +

175 + def get_num_image_tokens(

176 + self,

177 + *,

178 + image_width: int,

179 + image_height: int,

180 + num_frames: int = 1,

181 + ) -> tuple[ImageSize, int]:

182 + hf_config = self.get_hf_config()

183 + vit_config = hf_config.vit_config

184 + patch_size = vit_config.patch_size

185 + temporal_patch_size = vit_config.temporal_patch_size

186 + # NOTE: Frames are padded to be divisible by `temporal_patch_size`

187 + # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294

188 + padded_num_frames = num_frames + (-num_frames % temporal_patch_size)

189 + grid_t = max(padded_num_frames // temporal_patch_size, 1)

190 + grid_h = image_height // patch_size

191 + grid_w = image_width // patch_size

192 + num_patches = grid_t * grid_h * grid_w

193 + num_vision_tokens = num_patches

194 + return num_vision_tokens

195 +

196 + def get_max_image_tokens(self) -> int:

197 + target_width, target_height = self.get_image_size_with_most_features()

198 + return self.get_num_image_tokens(image_width=target_width,

199 + image_height=target_height)

200 +

201 + def _get_max_video_frames(self, max_tokens: int) -> int:

202 + target_width, target_height = self.get_image_size_with_most_features()

203 + num_frames = 0

204 + while True:

205 + next_num_frames = num_frames + 1

206 + next_max_tokens = self.get_num_video_tokens(

207 + image_width=target_width,

208 + image_height=target_height,

209 + num_frames=next_num_frames,

210 + image_processor=None,

211 + )

212 + if next_max_tokens > max_tokens:

213 + break

214 + num_frames = next_num_frames

215 + return num_frames

216 +

217 + def get_num_frames_with_most_features(

218 + self,

219 + seq_len: int,

220 + mm_counts: Mapping[str, int],

221 + ) -> int:

222 + max_images = mm_counts.get("image", 0)

223 + max_videos = mm_counts.get("video", 0)

224 + max_image_tokens = self.get_max_image_tokens() * max_images

225 + max_total_frames = self._get_max_video_frames(seq_len -

226 + max_image_tokens)

227 + max_frames_per_video = max_total_frames // max(max_videos, 1)

228 + return max(max_frames_per_video, 1)

229 +

230 + def get_num_video_tokens(

231 + self,

232 + *,

233 + image_width: int,

234 + image_height: int,

235 + num_frames: int,

236 + image_processor: Optional[BaseImageProcessor],

237 + ) -> int:

238 + num_video_tokens = self.get_num_image_tokens(image_width=image_width,

239 + image_height=image_height,

240 + num_frames=num_frames)

241 + return num_video_tokens

242 +

243 + def get_max_video_tokens(

244 + self,

245 + seq_len: int,

246 + mm_counts: Mapping[str, int],

247 + ) -> int:

248 + target_width, target_height = self.get_image_size_with_most_features()

249 + return self.get_num_video_tokens(

250 + image_width=target_width,

251 + image_height=target_height,

252 + num_frames=self.get_num_frames_with_most_features(

253 + seq_len, mm_counts),

254 + image_processor=None,

255 + )

256 +

257 +

258 +class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):

259 +

260 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

261 + num_images = mm_counts.get("image", 0)

262 + num_videos = mm_counts.get("video", 0)

263 + return IMAGE_TOKEN * num_images + VIDEO_TOKEN * num_videos

264 +

265 + def get_dummy_mm_data(

266 + self,

267 + seq_len: int,

268 + mm_counts: Mapping[str, int],

269 + ) -> MultiModalDataDict:

270 + num_images = mm_counts.get("image", 0)

271 + num_videos = mm_counts.get("video", 0)

272 +

273 + target_width, target_height = \

274 + self.info.get_image_size_with_most_features()

275 + target_num_frames = \

276 + self.info.get_num_frames_with_most_features(seq_len, mm_counts)

277 + mm_data = {

278 + "image":

279 + self._get_dummy_images(width=target_width,

280 + height=target_height,

281 + num_images=num_images),

282 + "video":

283 + self._get_dummy_videos(

284 + width=target_width,

285 + height=target_height,

286 + num_frames=target_num_frames,

287 + num_videos=num_videos,

288 + )

289 + }

290 + return mm_data

291 +

292 +

293 +class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]

294 + ):

295 +

296 + def visual_indicators_to_visual_tokens(

297 + self,

298 + visual_indicators: list[int],

299 + ) -> list[int]:

300 + """

301 + Filter image indicators placeholders and convert them to corresponding

302 + tokens in visual tokenizer.

303 + """

304 + hf_config = self.info.get_hf_config()

305 + vte_vocab_size = hf_config.visual_vocab_size

306 + return [

307 + vte_vocab_size - len(INDICATOR_IDS) + abs(x + 300) - 1

308 + for x in visual_indicators if x < -300

309 + ]

310 +

311 + def _call_hf_processor(

312 + self,

313 + prompt: str,

314 + mm_data: Mapping[str, object],

315 + mm_kwargs: Mapping[str, object],

316 + tok_kwargs: Mapping[str, object],

317 + ) -> BatchFeature:

318 + if not mm_data:

319 + # Avoid warning from HF logger for text-only input

320 + tokenizer = self.info.get_tokenizer()

321 + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)

322 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")

323 +

324 + processed_outputs = super()._call_hf_processor(

325 + prompt=prompt,

326 + mm_data=mm_data,

327 + mm_kwargs=mm_kwargs,

328 + tok_kwargs=tok_kwargs,

329 + )

330 + hf_processor = self.info.get_hf_processor()

331 +

332 + if "videos" in mm_data:

333 + visual_indicators = [

334 + hf_processor.construct_visual_indicators((1, 1, 1), True)

335 + for grid in processed_outputs["video_grids"]

336 + ]

337 + indicator_tokens = [

338 + self.visual_indicators_to_visual_tokens(indicator)

339 + for indicator in visual_indicators

340 + ]

341 + processed_outputs["video_indicator_tokens"] = indicator_tokens

342 + if "images" in mm_data:

343 + visual_indicators = [

344 + hf_processor.construct_visual_indicators((1, 1, 1), False)

345 + for grid in processed_outputs["grids"]

346 + ]

347 + indicator_tokens = [

348 + self.visual_indicators_to_visual_tokens(indicator)

349 + for indicator in visual_indicators

350 + ]

351 +

352 + processed_outputs["indicator_tokens"] = indicator_tokens

353 + return processed_outputs

354 +

355 + def _apply_hf_processor_tokens_only(

356 + self,

357 + prompt_tokens: list[int],

358 + ) -> list[int]:

359 +

360 + return prompt_tokens

361 +

362 + def _get_mm_fields_config(

363 + self,

364 + hf_inputs: BatchFeature,

365 + hf_processor_mm_kwargs: Mapping[str, object],

366 + ) -> Mapping[str, MultiModalFieldConfig]:

367 + return _ovis2_5_field_config()

368 +

369 + def _get_prompt_updates(

370 + self,

371 + mm_items: MultiModalDataItems,

372 + hf_processor_mm_kwargs: Mapping[str, object],

373 + out_mm_kwargs: MultiModalKwargsItems,

374 + ) -> list[PromptReplacement]:

375 +

376 + def get_replacement_ovis(item_idx, modality: str):

377 + if modality == "image":

378 + out_item = out_mm_kwargs["image"][item_idx]

379 + grid = out_item["grids"].data

380 + elif modality == "video":

381 + out_item = out_mm_kwargs["video"][item_idx]

382 + grid = out_item["video_grids"].data

383 + hf_processor = self.info.get_hf_processor()

384 + return hf_processor.construct_visual_placeholders(grid[0], )

385 +

386 + return [

387 + PromptReplacement(

388 + modality=modality,

389 + target=IMAGE_TOKEN if modality == "image" else VIDEO_TOKEN,

390 + replacement=partial(get_replacement_ovis, modality=modality),

391 + ) for modality in ("image", "video")

392 + ]

393 +

394 +

395 +@MULTIMODAL_REGISTRY.register_processor(Ovis2_5MultiModalProcessor,

396 + info=Ovis2_5ProcessingInfo,

397 + dummy_inputs=Ovis2_5DummyInputsBuilder)

398 +class Ovis2_5(nn.Module, SupportsMultiModal):

399 +

400 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

401 + super().__init__()

402 + config = vllm_config.model_config.hf_config

403 + quant_config = vllm_config.quant_config

404 +

405 + self.config: PretrainedConfig = config

406 + self.llm = init_vllm_registered_model(

407 + vllm_config=vllm_config.with_hf_config(config.text_config),

408 + prefix=maybe_prefix(prefix, "llm"),

409 + )

410 +

411 + self.visual_tokenizer = VisualTokenizer(

412 + config=config.vit_config,

413 + visual_vocab_size=config.visual_vocab_size,

414 + quant_config=quant_config,

415 + prefix=f"{prefix}.visual_tokenizer",

416 + )

417 +

418 + self.vte = VisualEmbedding(config.visual_vocab_size,

419 + config.hidden_size)

420 +

421 + text_model_type = self.config.get_text_config().model_type

422 + self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]

423 +

424 + # TODO(Isotr0py): PP support

425 + # self.make_empty_intermediate_tensors = (

426 + # self.language_model.make_empty_intermediate_tensors)

427 +

428 + def _parse_and_validate_visual_input(

429 + self, is_video,

430 + **kwargs: object) -> Optional[OvisImagePatchInputs]:

431 + if is_video:

432 + pixel_values = kwargs.pop("video_pixel_values", None)

433 + indicator_tokens = kwargs.pop("video_indicator_tokens", None)

434 + grids = kwargs.pop("video_grids", None)

435 + else:

436 + pixel_values = kwargs.pop("pixel_values", None)

437 + indicator_tokens = kwargs.pop("indicator_tokens", None)

438 + grids = kwargs.pop("grids", None)

439 + if pixel_values is None and indicator_tokens is None:

440 + return None

441 +

442 + if pixel_values is not None and indicator_tokens is not None:

443 + if not isinstance(pixel_values, (torch.Tensor, list)):

444 + raise ValueError("Incorrect type of pixel values. "

445 + f"Got type: {type(pixel_values)}")

446 +

447 + if not isinstance(indicator_tokens, (torch.Tensor, list)):

448 + raise ValueError("Incorrect type of indicator_tokens. "

449 + f"Got type: {type(indicator_tokens)}")

450 +

451 + return OvisImagePatchInputs(

452 + type="image_patches",

453 + flat_data=flatten_bn(flatten_bn(pixel_values), concat=True),

454 + patches_per_image=[

455 + x.shape[0] // (self.config.vit_config.hidden_stride**2)

456 + for x in flatten_bn(pixel_values)

457 + ],

458 + indicator_tokens=flatten_bn(flatten_bn(indicator_tokens),

459 + concat=True),

460 + grids=flatten_bn(flatten_bn(grids), concat=True),

461 + )

462 +

463 + raise AssertionError("This line should be unreachable.")

464 +

465 + def _process_image_input(

466 + self, image_input: OvisImagePatchInputs) -> MultiModalEmbeddings:

467 + image_patches_flat = image_input["flat_data"]

468 + patches_per_image = image_input["patches_per_image"]

469 + indicator_tokens = image_input["indicator_tokens"]

470 + grid_thws = image_input["grids"]

471 +

472 + indicator_per_image = list(

473 + map(lambda x: 2 if x > 1 else x + 2, patches_per_image))

474 +

475 + target_dtype = self.visual_tokenizer.dtype

476 + visual_tokens = self.visual_tokenizer(

477 + image_patches_flat.to(target_dtype), grid_thws)

478 +

479 + visual_embeds = self.vte(visual_tokens) # 1:1 numeric eq.

480 + indicator_embeds = self.vte(indicator_tokens)

481 +

482 + visual_embeds_per_image = visual_embeds.split(patches_per_image, dim=0)

483 + indicator_embeds_per_image = indicator_embeds.split(

484 + indicator_per_image)

485 +

486 + vision_embeddings = []

487 + for indicator, visual in zip(indicator_embeds_per_image,

488 + visual_embeds_per_image):

489 + vision_embeddings_per_image = []

490 + visual = visual.unsqueeze(0)

491 + for i in range(visual.shape[0]):

492 + vision_embeddings_per_image.append(

493 + torch.cat([indicator[i:i + 1], visual[i]], dim=0))

494 + vision_embeddings_per_image.append(indicator[i + 1:])

495 + vision_embeddings.append(

496 + torch.cat(vision_embeddings_per_image, dim=0))

497 + return tuple(vision_embeddings)

498 +

499 + def get_multimodal_embeddings(

500 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

501 + embeddings = []

502 +

503 + # NOTE: _parse_and_validate_visual_input has side-effects and pops

504 + # keys from kwargs. We process images first, then videos.

505 + image_input = self._parse_and_validate_visual_input(False, **kwargs)

506 + if image_input:

507 + embeddings.extend(self._process_image_input(image_input))

508 +

509 + video_input = self._parse_and_validate_visual_input(True, **kwargs)

510 + if video_input:

511 + embeddings.extend(self._process_image_input(video_input))

512 +

513 + return tuple(embeddings) if embeddings else None

514 +

515 + def get_input_embeddings(

516 + self,

517 + input_ids: torch.Tensor,

518 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

519 + ) -> torch.Tensor:

520 + inputs_embeds = self.llm.get_input_embeddings(input_ids)

521 + if multimodal_embeddings is not None:

522 + tmp = torch.concat(multimodal_embeddings, dim=0)

523 + inputs_embeds[input_ids == self.image_pad_token_id] = tmp

524 + return inputs_embeds

525 +

526 + def forward(

527 + self,

528 + input_ids: torch.Tensor,

529 + positions: torch.Tensor,

530 + intermediate_tensors: Optional[IntermediateTensors] = None,

531 + inputs_embeds: Optional[torch.Tensor] = None,

532 + **kwargs: object,

533 + ) -> Union[torch.Tensor, IntermediateTensors]:

534 + if intermediate_tensors is not None:

535 + inputs_embeds = None

536 +

537 + # NOTE: In v1, inputs_embeds is always generated at model runner, this

538 + # condition is for v0 compatibility.

539 + elif inputs_embeds is None:

540 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)

541 +

542 + inputs_embeds = self.get_input_embeddings(input_ids,

543 + vision_embeddings)

544 + input_ids = None

545 +

546 + # up until here we have a inputs_embeds 100% numerical identity

547 + # between the OG HF Transformers implementation and ours

548 + hidden_states = self.llm(

549 + input_ids=input_ids,

550 + positions=positions,

551 + intermediate_tensors=intermediate_tensors,

552 + inputs_embeds=inputs_embeds,

553 + )

554 + return hidden_states

555 +

556 + def compute_logits(

557 + self,

558 + hidden_states: torch.Tensor,

559 + sampling_metadata: SamplingMetadata,

560 + ) -> Optional[torch.Tensor]:

561 + logits = self.llm.compute_logits(hidden_states, sampling_metadata)

562 + return logits

563 +

564 + def load_weights(self, weights: Iterable[tuple[str,

565 + torch.Tensor]]) -> set[str]:

566 + loader = AutoWeightsLoader(self)

567 + return loader.load_weights(weights)

568 +

569 + def get_language_model(self) -> torch.nn.Module:

570 + return self.llm

\ No newline at end of file

vllm/model_executor/models/registry.py

@@ -231,6 +231,7 @@ _MULTIMODAL_MODELS = {

231 "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),

232 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),

233 "Ovis": ("ovis", "Ovis"),

234 + "Ovis2_5": ("ovis2_5", "Ovis2_5"),

235 "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501

236 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),

237 "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),

vllm/model_executor/models/siglip2navit.py new

+607

@@ -0,0 +1,607 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Implementation of SiglipVisionModel intended to be only used

4 +within a vision language model."""

5 +

6 +from typing import Optional, Union

7 +

8 +import torch

9 +from einops import rearrange, repeat

10 +from torch import nn

11 +from torch.nn import functional as F

12 +from transformers.activations import ACT2FN

13 +from transformers.configuration_utils import PretrainedConfig

14 +from transformers.modeling_outputs import BaseModelOutputWithNoAttention

15 +

16 +from vllm.platforms import _Backend

17 +

18 +from .vision import get_vit_attn_backend

19 +

20 +

21 +class VisionRotaryEmbedding(nn.Module):

22 +

23 + def __init__(self, dim: int, theta: float = 10000.0) -> None:

24 + super().__init__()

25 + inv_freq = 1.0 / (theta

26 + **(torch.arange(0, dim, 2, dtype=torch.float) / dim))

27 + self.register_buffer("inv_freq", inv_freq, persistent=False)

28 +

29 + def forward(self, seqlen: int) -> torch.Tensor:

30 + seq = torch.arange(seqlen,

31 + device=self.inv_freq.device,

32 + dtype=self.inv_freq.dtype)

33 + freqs = torch.outer(seq, self.inv_freq)

34 + return freqs

35 +

36 +

37 +class Siglip2VisionEmbeddings(nn.Module):

38 +

39 + def __init__(self, config: PretrainedConfig):

40 + super().__init__()

41 + self.config = config

42 + self.embed_dim = config.hidden_size

43 + self.patch_size = config.patch_size

44 + self.image_size = config.image_size

45 + self.num_patches = config.num_patches

46 + self.preserve_original_pe = config.preserve_original_pe

47 + self.hidden_stride = config.hidden_stride

48 +

49 + # siglip2 naflex

50 + if self.num_patches > 0:

51 + self.patch_embedding = nn.Linear(

52 + in_features=config.num_channels * self.patch_size *

53 + self.patch_size,

54 + out_features=self.embed_dim,

55 + )

56 + if self.preserve_original_pe:

57 + self.position_embedding_size = int(self.num_patches**0.5)

58 + self.position_embedding = nn.Embedding(self.num_patches,

59 + self.embed_dim)

60 +

61 + else:

62 + self.patch_embedding = nn.Conv2d(

63 + in_channels=config.num_channels,

64 + out_channels=self.embed_dim,

65 + kernel_size=self.patch_size,

66 + stride=self.patch_size,

67 + padding="valid",

68 + )

69 + if self.preserve_original_pe:

70 + self.num_patches = (self.image_size // self.patch_size)**2

71 + self.position_embedding_size = (self.image_size //

72 + self.patch_size)

73 + self.position_embedding = nn.Embedding(self.num_patches,

74 + self.embed_dim)

75 +

76 + def forward(self,

77 + pixel_values: torch.FloatTensor,

78 + grid_thws: Optional[torch.LongTensor] = None) -> torch.Tensor:

79 + """

80 + Args:

81 + pixel_values (`torch.FloatTensor`):

82 + Pixel values of shape (

83 + num_patches,

84 + num_channels * temporal_patch_size * patch_size * patch_size

85 + )

86 + grid_thws: (`torch.LongTensor`):

87 + grid shape (num_patches, 3)

88 + """

89 +

90 + # Apply patch embeddings to already patchified pixel values

91 + target_dtype = self.patch_embedding.weight.dtype

92 + if isinstance(self.patch_embedding, nn.Linear):

93 + patch_embeds = self.patch_embedding(

94 + pixel_values.to(dtype=target_dtype))

95 + elif isinstance(self.patch_embedding, nn.Conv2d):

96 + pixel_values = pixel_values.view(

97 + -1, self.config.num_channels * self.config.temporal_patch_size,

98 + self.patch_size, self.patch_size)

99 + patch_embeds = self.patch_embedding(

100 + pixel_values.to(dtype=target_dtype))

101 + patch_embeds = patch_embeds.reshape(-1, self.embed_dim)

102 +

103 + if self.preserve_original_pe:

104 + assert grid_thws is not None

105 + pos_embed_new = torch.zeros_like(patch_embeds)

106 + positional_embeddings = self.position_embedding.weight.reshape(

107 + self.position_embedding_size, self.position_embedding_size,

108 + -1).unsqueeze(0).permute(0, 3, 1, 2)

109 + cnt = 0

110 + for t, h, w in grid_thws:

111 + volume = t * h * w

112 + pe = F.interpolate(positional_embeddings,

113 + size=(h, w),

114 + mode='bicubic',

115 + align_corners=False)

116 + pe = pe.permute(0, 2, 3, 1).reshape(1, h * w, -1)

117 + pe = pe[0].repeat(t, 1)

118 + pe = pe.reshape(t, h // self.hidden_stride, self.hidden_stride,

119 + w // self.hidden_stride, self.hidden_stride,

120 + -1)

121 + pe = pe.permute(0, 1, 3, 2, 4, 5).reshape(volume, -1)

122 + pos_embed_new[cnt:cnt + volume] = pe

123 + cnt += volume

124 + patch_embeds = patch_embeds + pos_embed_new

125 +

126 + return patch_embeds

127 +

128 +

129 +# copy from flash_attn/layers/rotary.py

130 +def rotate_half(x, interleaved=False):

131 + if not interleaved:

132 + x1, x2 = x.chunk(2, dim=-1)

133 + return torch.cat((-x2, x1), dim=-1)

134 + else:

135 + x1, x2 = x[..., ::2], x[..., 1::2]

136 + return rearrange(torch.stack((-x2, x1), dim=-1),

137 + "... d two -> ... (d two)",

138 + two=2)

139 +

140 +

141 +def apply_rotary_emb_torch(x, cos, sin, interleaved=False):

142 + """

143 + x: (batch_size, seqlen, nheads, headdim)

144 + cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2)

145 + """

146 + ro_dim = cos.shape[-1] * 2

147 + assert ro_dim <= x.shape[-1]

148 + cos = repeat(

149 + cos,

150 + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")

151 + sin = repeat(

152 + sin,

153 + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")

154 + return torch.cat(

155 + [

156 + x[..., :ro_dim] * cos +

157 + rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:]

158 + ],

159 + dim=-1,

160 + )

161 +

162 +

163 +def apply_rotary_pos_emb(

164 + q: torch.Tensor,

165 + k: torch.Tensor,

166 + cos: torch.Tensor,

167 + sin: torch.Tensor,

168 + is_flash_attn_backend: bool = False,

169 +) -> tuple[torch.Tensor, torch.Tensor]:

170 + cos = cos.chunk(2, dim=-1)[0].contiguous()

171 + sin = sin.chunk(2, dim=-1)[0].contiguous()

172 + if is_flash_attn_backend:

173 + from flash_attn.layers.rotary import apply_rotary_emb

174 + apply_rotary_emb_func = apply_rotary_emb

175 + else:

176 + apply_rotary_emb_func = apply_rotary_emb_torch

177 + q_embed = apply_rotary_emb_func(q.float(), cos.float(),

178 + sin.float()).type_as(q)

179 + k_embed = apply_rotary_emb_func(k.float(), cos.float(),

180 + sin.float()).type_as(k)

181 + return q_embed, k_embed

182 +

183 +

184 +class Siglip2Attention(nn.Module):

185 + """Multi-headed attention from 'Attention Is All You Need' paper"""

186 +

187 + def __init__(self, config):

188 + super().__init__()

189 + self.config = config

190 + self.embed_dim = config.hidden_size

191 + self.num_heads = config.num_attention_heads

192 + self.head_dim = self.embed_dim // self.num_heads

193 + if self.head_dim * self.num_heads != self.embed_dim:

194 + raise ValueError(

195 + f"embed_dim must be divisible by num_heads "

196 + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"

197 + f" {self.num_heads}).")

198 + self.scale = self.head_dim**-0.5

199 + self.dropout = config.attention_dropout

200 + self.is_causal = False

201 +

202 + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)

203 + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)

204 + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)

205 + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)

206 +

207 + self.use_rope = config.use_rope

208 +

209 + # Detect attention implementation.

210 + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)

211 + if self.attn_backend not in {

212 + _Backend.FLASH_ATTN, _Backend.TORCH_SDPA,

213 + _Backend.ROCM_AITER_FA

214 + }:

215 + self.attn_backend = _Backend.TORCH_SDPA

216 + self.is_flash_attn_backend = self.attn_backend in {

217 + _Backend.FLASH_ATTN, _Backend.ROCM_AITER_FA

218 + }

219 +

220 + def forward(

221 + self,

222 + hidden_states: torch.Tensor,

223 + cu_seqlens: torch.Tensor,

224 + position_embeddings: Optional[tuple[torch.Tensor,

225 + torch.Tensor]] = None,

226 + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:

227 + """Input shape: Batch x Time x Channel"""

228 +

229 + seq_length, embed_dim = hidden_states.shape

230 +

231 + queries = self.q_proj(hidden_states)

232 + keys = self.k_proj(hidden_states)

233 + values = self.v_proj(hidden_states)

234 +

235 + queries = queries.view(seq_length, self.num_heads, self.head_dim)

236 + keys = keys.view(seq_length, self.num_heads, self.head_dim)

237 + values = values.view(seq_length, self.num_heads, self.head_dim)

238 +

239 + if self.use_rope:

240 + cos, sin = position_embeddings

241 + queries, keys = apply_rotary_pos_emb(queries.unsqueeze(0),

242 + keys.unsqueeze(0), cos, sin,

243 + self.is_flash_attn_backend)

244 + queries = queries.squeeze(0)

245 + keys = keys.squeeze(0)

246 +

247 + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()

248 + if self.is_flash_attn_backend:

249 + if self.attn_backend == _Backend.ROCM_AITER_FA:

250 + from aiter import flash_attn_varlen_func

251 + else:

252 + from flash_attn import flash_attn_varlen_func

253 + attn_output = flash_attn_varlen_func(

254 + queries, keys, values, cu_seqlens, cu_seqlens, max_seqlen,

255 + max_seqlen).reshape(seq_length, -1)

256 + elif self.attn_backend == _Backend.TORCH_SDPA:

257 + # Execute attention entry by entry for speed & less VRAM.

258 + batch_size = cu_seqlens.shape[0] - 1

259 + outputs = []

260 + cu = cu_seqlens.tolist()

261 + for i in range(batch_size):

262 + start_idx = cu[i]

263 + end_idx = cu[i + 1]

264 +

265 + # Each sequence is processed independently.

266 + q_i = queries[start_idx:end_idx].unsqueeze(0)

267 + k_i = keys[start_idx:end_idx].unsqueeze(0)

268 + v_i = values[start_idx:end_idx].unsqueeze(0)

269 +

270 + # (1, seq_len, num_heads, head_dim) ->

271 + # (1, num_heads, seq_len, head_dim)

272 + q_i, k_i, v_i = [x.transpose(1, 2) for x in (q_i, k_i, v_i)]

273 +

274 + output_i = F.scaled_dot_product_attention(q_i,

275 + k_i,

276 + v_i,

277 + dropout_p=0.0)

278 + # (1, num_heads, seq_len, head_dim) -> (seq_len, embed_dim)

279 + output_i = output_i.transpose(1, 2).reshape(-1, self.embed_dim)

280 + outputs.append(output_i)

281 +

282 + attn_output = torch.cat(outputs, dim=0)

283 + attn_output = self.out_proj(attn_output)

284 + return attn_output

285 +

286 +

287 +class Siglip2MLP(nn.Module):

288 +

289 + def __init__(self, config):

290 + super().__init__()

291 + self.config = config

292 + self.activation_fn = ACT2FN[config.hidden_act]

293 + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)

294 + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)

295 +

296 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

297 + hidden_states = self.fc1(hidden_states)

298 + hidden_states = self.activation_fn(hidden_states)

299 + hidden_states = self.fc2(hidden_states)

300 + return hidden_states

301 +

302 +

303 +class Siglip2EncoderLayer(nn.Module):

304 +

305 + def __init__(self, config: PretrainedConfig):

306 + super().__init__()

307 + self.embed_dim = config.hidden_size

308 + self.layer_norm1 = nn.LayerNorm(self.embed_dim,

309 + eps=config.layer_norm_eps)

310 + self.self_attn = Siglip2Attention(config)

311 + self.layer_norm2 = nn.LayerNorm(self.embed_dim,

312 + eps=config.layer_norm_eps)

313 + self.mlp = Siglip2MLP(config)

314 +

315 + def forward(self, hidden_states: torch.Tensor, cu_seqlens: torch.Tensor,

316 + position_embeddings: torch.Tensor) -> tuple[torch.FloatTensor]:

317 + """

318 + Args:

319 + hidden_states (`torch.FloatTensor`):

320 + Input to the layer of shape `(batch, seq_len, embed_dim)`.

321 + output_attentions (`bool`, *optional*, defaults to `False`):

322 + Whether or not to return the attentions tensors of all

323 + attention layers. See `attentions` under

324 + returned tensors for more detail.

325 + """

326 + residual = hidden_states

327 +

328 + hidden_states = self.layer_norm1(hidden_states)

329 + hidden_states = self.self_attn(hidden_states=hidden_states,

330 + cu_seqlens=cu_seqlens,

331 + position_embeddings=position_embeddings)

332 + hidden_states = residual + hidden_states

333 +

334 + residual = hidden_states

335 + hidden_states = self.layer_norm2(hidden_states)

336 + hidden_states = self.mlp(hidden_states)

337 + hidden_states = residual + hidden_states

338 + return hidden_states

339 +

340 +

341 +class Siglip2Encoder(nn.Module):

342 + """

343 + Transformer encoder consisting of `config.num_hidden_layers`

344 + self attention layers. Each layer is a [`Siglip2EncoderLayer`].

345 +

346 + Args:

347 + config: PretrainedConfig

348 + """

349 +

350 + def __init__(self, config: PretrainedConfig):

351 + super().__init__()

352 + self.config = config

353 + self.layers = nn.ModuleList([

354 + Siglip2EncoderLayer(config)

355 + for _ in range(config.num_hidden_layers)

356 + ])

357 + self.gradient_checkpointing = False

358 +

359 + self.rotary_pos_emb = VisionRotaryEmbedding(

360 + config.hidden_size // config.num_attention_heads // 2)

361 + self.patch_size = config.patch_size

362 + self.hidden_stride = config.hidden_stride

363 + self.window_size = config.window_size

364 + self.spatial_merge_unit = config.hidden_stride * config.hidden_stride

365 + if config.fullatt_block_indexes is None:

366 + self.fullatt_block_indexes = None

367 + else:

368 + self.fullatt_block_indexes = [

369 + int(i) for i in config.fullatt_block_indexes.split('|')

370 + ]

371 +

372 + # copied from qwen2.5_vl

373 + def rot_pos_emb(self, grid_thw):

374 + pos_ids = []

375 + for t, h, w in grid_thw:

376 + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)

377 + hpos_ids = hpos_ids.reshape(

378 + h // self.hidden_stride,

379 + self.hidden_stride,

380 + w // self.hidden_stride,

381 + self.hidden_stride,

382 + )

383 + hpos_ids = hpos_ids.permute(0, 2, 1, 3)

384 + hpos_ids = hpos_ids.flatten()

385 +

386 + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)

387 + wpos_ids = wpos_ids.reshape(

388 + h // self.hidden_stride,

389 + self.hidden_stride,

390 + w // self.hidden_stride,

391 + self.hidden_stride,

392 + )

393 + wpos_ids = wpos_ids.permute(0, 2, 1, 3)

394 + wpos_ids = wpos_ids.flatten()

395 + pos_ids.append(

396 + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))

397 + pos_ids = torch.cat(pos_ids, dim=0)

398 + max_grid_size = grid_thw[:, 1:].max()

399 + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)

400 + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)

401 + return rotary_pos_emb

402 +

403 + def get_window_index(self, grid_thw):

404 + window_index: list = []

405 + cu_window_seqlens: list = [0]

406 + window_index_id = 0

407 + # patch (after merge) number in each window

408 + vit_merger_window_size = (self.window_size // self.hidden_stride //

409 + self.patch_size)

410 +

411 + for grid_t, grid_h, grid_w in grid_thw:

412 + llm_grid_h, llm_grid_w = (

413 + grid_h // self.hidden_stride, # number of patch after merge

414 + grid_w // self.hidden_stride,

415 + )

416 + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(

417 + grid_t, llm_grid_h, llm_grid_w)

418 + pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size

419 + pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size

420 + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size

421 + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size

422 + index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)

423 + index_padded = index_padded.reshape(

424 + grid_t,

425 + num_windows_h,

426 + vit_merger_window_size,

427 + num_windows_w,

428 + vit_merger_window_size,

429 + )

430 + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(

431 + grid_t,

432 + num_windows_h * num_windows_w,

433 + vit_merger_window_size,

434 + vit_merger_window_size,

435 + )

436 + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)

437 + index_padded = index_padded.reshape(-1)

438 + index_new = index_padded[index_padded != -100]

439 + window_index.append(index_new + window_index_id)

440 + cu_seqlens_tmp = seqlens.cumsum(

441 + 0) * self.spatial_merge_unit + cu_window_seqlens[-1]

442 + cu_window_seqlens.extend(cu_seqlens_tmp.tolist())

443 + window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()

444 + window_index = torch.cat(window_index, dim=0)

445 +

446 + return window_index, cu_window_seqlens

447 +

448 + # Ignore copy

449 + def forward(

450 + self,

451 + inputs_embeds,

452 + grid_thws: torch.Tensor,

453 + output_hidden_states: bool = False,

454 + ) -> tuple[torch.Tensor, Optional[tuple[torch.Tensor, ...]]]:

455 + r"""

456 + Args:

457 + inputs_embeds (`torch.FloatTensor` of shape

458 + `(batch_size, sequence_length, hidden_size)`):

459 + Optionally, instead of passing `input_ids` you can choose to

460 + directly pass an embedded representation. This is useful if

461 + you want more control over how to convert `input_ids` indices

462 + into associated vectors than the model's internal embedding

463 + lookup matrix.

464 + grid_thws (`torch.LongTensor`):

465 + grid shape (num_patches, 3)

466 + output_hidden_states (`bool`, *optional*):

467 + Whether or not to return the hidden states of all layers. See

468 + `hidden_states` under returned tensors for more detail.

469 + return_dict (`bool`, *optional*):

470 + Whether or not to return a [`~utils.ModelOutput`] instead of

471 + a plain tuple.

472 + """

473 + rotary_pos_emb = self.rot_pos_emb(grid_thws)

474 + window_index, cu_window_seqlens = self.get_window_index(grid_thws)

475 + cu_window_seqlens = torch.tensor(

476 + cu_window_seqlens,

477 + device=inputs_embeds.device,

478 + dtype=grid_thws.dtype if torch.jit.is_tracing() else torch.int32,

479 + )

480 + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)

481 +

482 + seq_len, _ = inputs_embeds.size()

483 + inputs_embeds = inputs_embeds.reshape(

484 + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)

485 + inputs_embeds = inputs_embeds[window_index, :, :]

486 + inputs_embeds = inputs_embeds.reshape(seq_len, -1)

487 + rotary_pos_emb = rotary_pos_emb.reshape(

488 + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)

489 + rotary_pos_emb = rotary_pos_emb[window_index, :, :]

490 + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)

491 + emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)

492 + position_embeddings = (emb.cos(), emb.sin())

493 +

494 + cu_seqlens = torch.repeat_interleave(

495 + grid_thws[:, 1] * grid_thws[:, 2], grid_thws[:, 0]

496 + ).cumsum(

497 + dim=0,

498 + # Select dtype based on the following factors:

499 + # - FA2 requires that cu_seqlens_q must have dtype int32

500 + # - torch.onnx.export requires that cu_seqlens_q must have

501 + # same dtype as grid_thw

502 + # See https://github.com/huggingface/transformers/pull/34852

503 + # for more information

504 + dtype=grid_thws.dtype if torch.jit.is_tracing() else torch.int32,

505 + )

506 + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)

507 +

508 + reverse_indices = torch.argsort(window_index)

509 + encoder_states = () if output_hidden_states else None

510 +

511 + hidden_states = inputs_embeds

512 + for index, block in enumerate(self.layers):

513 + if (not self.fullatt_block_indexes

514 + or index in self.fullatt_block_indexes):

515 + cu_seqlens_tmp = cu_seqlens

516 + else:

517 + cu_seqlens_tmp = cu_window_seqlens

518 + hidden_states = block(hidden_states, cu_seqlens_tmp,

519 + position_embeddings)

520 + if output_hidden_states:

521 + hidden_states_ = hidden_states.reshape(

522 + seq_len // self.spatial_merge_unit,

523 + self.spatial_merge_unit, -1)

524 + encoder_states += (hidden_states_[reverse_indices, :].reshape(

525 + seq_len, -1), )

526 + # tokens = self.post_trunk_norm(tokens)

527 + hidden_states = hidden_states.reshape(

528 + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)

529 + hidden_states = hidden_states[reverse_indices, :].reshape(seq_len, -1)

530 +

531 + return hidden_states, encoder_states

532 +

533 +

534 +class Siglip2VisionTransformer(nn.Module):

535 +

536 + def __init__(self, config: PretrainedConfig):

537 + super().__init__()

538 + self.config = config

539 + embed_dim = config.hidden_size

540 +

541 + self.embeddings = Siglip2VisionEmbeddings(config)

542 + self.encoder = Siglip2Encoder(config)

543 + self.post_layernorm = nn.LayerNorm(embed_dim,

544 + eps=config.layer_norm_eps)

545 + self._use_flash_attention_2 = \

546 + (config._attn_implementation == "flash_attention_2")

547 +

548 + def forward(

549 + self,

550 + pixel_values: torch.FloatTensor,

551 + grid_thws: torch.LongTensor,

552 + output_hidden_states: Optional[bool] = True,

553 + return_dict: Optional[bool] = True,

554 + ) -> Union[

555 + tuple[torch.Tensor],

556 + tuple[torch.Tensor, tuple[torch.Tensor, ...]],

557 + BaseModelOutputWithNoAttention,

558 + ]:

559 + r"""

560 + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):

561 + Tensor containing the spatial dimensions (height, width)

562 + of the input images.

563 + """

564 + hidden_states = self.embeddings(pixel_values, grid_thws)

565 +

566 + last_hidden_state, hidden_states = self.encoder(

567 + hidden_states, grid_thws, output_hidden_states)

568 + last_hidden_state = self.post_layernorm(last_hidden_state)

569 +

570 + if not return_dict:

571 + output = (last_hidden_state, )

572 + output += (hidden_states, ) if output_hidden_states else ()

573 + return output

574 +

575 + return last_hidden_state

576 +

577 +

578 +class Siglip2NavitModel(torch.nn.Module):

579 +

580 + def __init__(self, config: PretrainedConfig):

581 + super().__init__()

582 +

583 + self.vision_model = Siglip2VisionTransformer(config)

584 +

585 + def forward(

586 + self,

587 + pixel_values: torch.FloatTensor,

588 + grid_thws: torch.LongTensor,

589 + output_hidden_states: Optional[bool] = None,

590 + return_dict: Optional[bool] = None,

591 + ) -> Union[

592 + tuple[torch.Tensor],

593 + tuple[torch.Tensor, tuple[torch.Tensor, ...]],

594 + BaseModelOutputWithNoAttention,

595 + ]:

596 +

597 + if output_hidden_states is None:

598 + output_hidden_states = self.config.output_hidden_states

599 + if return_dict is None:

600 + return_dict = self.config.use_return_dict

601 +

602 + return self.vision_model(

603 + pixel_values=pixel_values,

604 + grid_thws=grid_thws,

605 + output_hidden_states=output_hidden_states,

606 + return_dict=return_dict,

607 + )

vllm/transformers_utils/processors/__init__.py

+2 -1

@@ -11,5 +11,6 @@ reasons:

11 from vllm.transformers_utils.processors.deepseek_vl2 import (

12 DeepseekVLV2Processor)

13 from vllm.transformers_utils.processors.ovis import OvisProcessor

14 +from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor

15 -__all__ = ["DeepseekVLV2Processor", "OvisProcessor"]

16 +__all__ = ["DeepseekVLV2Processor", "OvisProcessor", "Ovis2_5Processor"]

vllm/transformers_utils/processors/ovis2_5.py new

+458

@@ -0,0 +1,458 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import math

4 +from functools import cached_property

5 +from typing import Optional, Union

6 +

7 +import numpy as np

8 +import PIL

9 +import torch

10 +from transformers import AutoProcessor, BatchFeature

11 +from transformers.image_utils import ImageInput

12 +from transformers.processing_utils import (ProcessingKwargs, ProcessorMixin,

13 + Unpack)

14 +from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

15 +

16 +__all__ = ['Ovis2_5Processor']

17 +IMAGE_TOKEN = "<image>"

18 +VIDEO_TOKEN = "<video>"

19 +MIN_PIXELS = 448 * 448

20 +MAX_PIXELS = 1792 * 1792

21 +

22 +

23 +class Ovis2_5ProcessorKwargs(ProcessingKwargs,

24 + total=False): # type: ignore[call-arg]

25 + _defaults = {

26 + "text_kwargs": {

27 + "padding": False,

28 + },

29 + "images_kwargs": {

30 + 'convert_to_rgb': True,

31 + 'min_pixels': MIN_PIXELS,

32 + 'max_pixels': MAX_PIXELS,

33 + },

34 + "videos_kwargs": {

35 + 'convert_to_rgb': True,

36 + 'min_pixels': MIN_PIXELS,

37 + 'max_pixels': MAX_PIXELS,

38 + }

39 + }

40 +

41 +

42 +class Ovis2_5Processor(ProcessorMixin):

43 + r"""

44 + Constructs a Ovis processor which wraps a Ovis image processor

45 + and a Qwen2 tokenizer into a single processor.

46 + [`OvisProcessor`] offers all the functionalities of

47 + [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`].

48 + See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`]

49 + for more information.

50 + Args:

51 + image_processor ([`Qwen2VLImageProcessor`], *optional*):

52 + The image processor is a required input.

53 + tokenizer ([`Qwen2TokenizerFast`], *optional*):

54 + The tokenizer is a required input.

55 + chat_template (`str`, *optional*): A Jinja template which will

56 + be used to convert lists of messages in a chat into

57 + a tokenizable string.

58 + """

59 +

60 + attributes = ["image_processor", "tokenizer"]

61 + valid_kwargs = ["chat_template", "image_pad_token"]

62 +

63 + image_processor_class = "AutoImageProcessor"

64 + tokenizer_class = "AutoTokenizer"

65 +

66 + def __init__(

67 + self,

68 + image_processor=None,

69 + tokenizer=None,

70 + chat_template=None,

71 + image_pad_token=None,

72 + patch_size=16,

73 + hidden_stride=2,

74 + temporal_patch_size=1,

75 + **kwargs,

76 + ):

77 + self.image_token = IMAGE_TOKEN

78 + self.video_token = VIDEO_TOKEN

79 + self.image_pad_token = "<|image_pad|>"

80 +

81 + self.patch_size = patch_size

82 + self.hidden_stride = hidden_stride

83 + self.temporal_patch_size = temporal_patch_size

84 + super().__init__(image_processor,

85 + tokenizer,

86 + chat_template=chat_template)

87 +

88 + @cached_property

89 + def extra_special_tokens(self):

90 + image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]

91 + extra_special_tokens = {

92 + "image_token": -200,

93 + "video_token": -201,

94 + "visual_atom": -300,

95 + "image_start": -301,

96 + "image_end": -302,

97 + "video_start": -303,

98 + "video_end": -304,

99 + 'image_pad': image_pad_token_id,

100 + }

101 + return extra_special_tokens

102 +

103 + def __call__(

104 + self,

105 + images: ImageInput = None,

106 + videos: Union[np.ndarray, list[ImageInput]] = None,

107 + text: Union[TextInput, PreTokenizedInput, list[TextInput],

108 + list[PreTokenizedInput]] = None,

109 + **kwargs: Unpack[Ovis2_5ProcessorKwargs],

110 + ) -> BatchFeature:

111 + """

112 + Main method to prepare for the model one or several sequences(s)

113 + and image(s). This method forwards the `text`and `kwargs` arguments

114 + to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text`

115 + is not `None` to encode the text. To prepare the vision inputs,

116 + this method forwards the `vision_infos` and `kwrags` arguments to

117 + Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`]

118 + if `vision_infos` is not `None`.

119 + Args:

120 + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`,

121 + `list[PIL.Image.Image]`, `list[np.ndarray]`,

122 + `list[torch.Tensor]`):

123 + The image or batch of images to be prepared.

124 + Each image can be a PIL image, NumPy array or PyTorch

125 + tensor. Both channels-first and channels-last formats

126 + are supported.

127 + text (`str`, `list[str]`, `list[list[str]]`):

128 + The sequence or batch of sequences to be encoded.

129 + Each sequence can be a string or a list of strings

130 + (pretokenized string). If the sequences are provided as

131 + list of strings (pretokenized), you must set

132 + `is_split_into_words=True` (to lift the ambiguity with

133 + a batch of sequences).

134 + videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`,

135 + `list[torch.Tensor]`):

136 + The image or batch of videos to be prepared. Each video

137 + can be a 4D NumPy array or PyTorch tensor, or a nested

138 + list of 3D frames. Both channels-first and channels-last

139 + formats are supported.

140 + return_tensors (`str` or [`~utils.TensorType`], *optional*):

141 + If set, will return tensors of a particular framework.

142 + Acceptable values are:

143 + - `'tf'`: Return TensorFlow `tf.constant` objects.

144 + - `'pt'`: Return PyTorch `torch.Tensor` objects.

145 + - `'np'`: Return NumPy `np.ndarray` objects.

146 + - `'jax'`: Return JAX `jnp.ndarray` objects.

147 + Returns:

148 + [`BatchFeature`]: A [`BatchFeature`] with the following fields:

149 + - **input_ids** -- list of token ids to be fed to a model.

150 + Returned when `text` is not `None`.

151 + - **attention_mask** -- list of indices specifying which tokens

152 + should be attended to by the model (when

153 + `return_attention_mask=True` or if *"attention_mask"*

154 + is in `self.model_input_names` and if `text` is not `None`).

155 + - **pixel_values** -- Pixel values to be fed to a model.

156 + Returned when `images` is not `None`.

157 + - **pixel_values_videos** -- Pixel values of videos to be fed to

158 + a model. Returned when `videos` is not `None`.

159 + - **image_grid_thw** -- list of image 3D grid in LLM. Returned

160 + when `images` is not `None`.

161 + - **video_grid_thw** -- list of video 3D grid in LLM. Returned

162 + when `videos` is not `None`.

163 + - **second_per_grid_ts** -- list of video seconds per time grid.

164 + Returned when `videos` is not `None`.

165 + """

166 + output_kwargs = self._merge_kwargs(

167 + Ovis2_5ProcessorKwargs,

168 + tokenizer_init_kwargs=self.tokenizer.init_kwargs,

169 + **kwargs,

170 + )

171 + # Process all images first

172 + visual_features = {}

173 + output = BatchFeature()

174 + if images is not None:

175 + processed_images = []

176 + image_placeholders_list = []

177 + grids = []

178 + # Process each image

179 + for image in images if isinstance(images, list) else [images]:

180 + pixel_values, image_placeholders, grid = (

181 + self.preprocess_multidata(

182 + images=image, **output_kwargs["images_kwargs"]))

183 + processed_images.append(pixel_values)

184 + image_placeholders_list.append(image_placeholders)

185 + grids.append(grid)

186 +

187 + # assign all processed images

188 + if processed_images:

189 + visual_features["image_placeholders"] = image_placeholders_list

190 + output["pixel_values"] = processed_images

191 + output["grids"] = grids

192 +

193 + if videos is not None:

194 + processed_videos = []

195 + videos_placeholders_list = []

196 + grids = []

197 + # Process each video

198 + for video in videos if isinstance(videos, list) else [videos]:

199 + pixel_values, video_placeholders, grid = (

200 + self.preprocess_multidata(

201 + video=video, **output_kwargs["videos_kwargs"]))

202 + processed_videos.append(pixel_values)

203 + videos_placeholders_list.append(video_placeholders)

204 + grids.append(grid)

205 + # assign all processed videos

206 + if processed_videos:

207 + visual_features[

208 + "video_placeholders"] = videos_placeholders_list

209 + output["video_pixel_values"] = processed_videos

210 + output["video_grids"] = grids

211 +

212 + # Process text input

213 + if text is not None:

214 + if not isinstance(text, list):

215 + text = [text]

216 + tokenized_batched_text = self._tokenize_with_visual_symbol(text)

217 + image_token_id = self.get_token_value("image_token")

218 + video_token_id = self.get_token_value("video_token")

219 + replaced_ids_list = []

220 + image_idx = 0

221 + video_idx = 0

222 + for ids_tensor in tokenized_batched_text:

223 + has_image_tokens = (image_token_id in ids_tensor

224 + and "image_placeholders" in visual_features

225 + and image_idx < len(

226 + visual_features["image_placeholders"]))

227 + has_video_tokens = (video_token_id in ids_tensor

228 + and "video_placeholders" in visual_features

229 + and video_idx < len(

230 + visual_features["video_placeholders"]))

231 + if has_image_tokens or has_video_tokens:

232 + # Convert to list for easier manipulation

233 + ids_list = ids_tensor.tolist()

234 + new_ids = []

235 +

236 + # Replace placeholders

237 + for token_id in ids_list:

238 + if token_id == image_token_id:

239 + new_ids.extend(

240 + visual_features["image_placeholders"]

241 + [image_idx])

242 + image_idx += 1

243 + elif token_id == video_token_id:

244 + new_ids.extend(

245 + visual_features["video_placeholders"]

246 + [video_idx])

247 + video_idx += 1

248 + else:

249 + new_ids.append(token_id)

250 + # Convert back to tensor

251 + ids_tensor = torch.tensor(new_ids, dtype=torch.long)

252 + replaced_ids_list.append(ids_tensor)

253 + if replaced_ids_list:

254 + replaced_and_tokenized_ids = torch.stack(replaced_ids_list)

255 + else:

256 + replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long)

257 + output["input_ids"] = replaced_and_tokenized_ids

258 +

259 + return output

260 + # If only images were provided

261 + return BatchFeature(data=visual_features)

262 +

263 + def _tokenize_with_visual_symbol(self,

264 + text_list: list[str]) -> torch.LongTensor:

265 + batch_token_ids = []

266 + for text in text_list:

267 + token_ids = []

268 + video_token_id = self.get_token_value("video_token")

269 + image_token_id = self.get_token_value("image_token")

270 + video_split_texts = text.split(self.video_token)

271 +

272 + for j, video_segment in enumerate(video_split_texts):

273 + image_split_texts = video_segment.split(self.image_token)

274 + text_chunks = [

275 + self.tokenizer(chunk, add_special_tokens=False).input_ids

276 + for chunk in image_split_texts

277 + ]

278 + segment_tokens = []

279 + for i, chunk in enumerate(text_chunks):

280 + segment_tokens.extend(chunk)

281 + if i < len(text_chunks) - 1:

282 + segment_tokens.append(image_token_id)

283 + token_ids.extend(segment_tokens)

284 + if j < len(video_split_texts) - 1:

285 + token_ids.append(video_token_id)

286 +

287 + batch_token_ids.append(token_ids)

288 + return torch.tensor(batch_token_ids, dtype=torch.long)

289 +

290 + # Copied from qwen2_vl

291 + def smart_resize(self,

292 + height: int,

293 + width: int,

294 + factor: int = 28,

295 + min_pixels: int = MIN_PIXELS,

296 + max_pixels: int = MAX_PIXELS):

297 + """Rescales the image so that the following conditions are met:

298 + 1. Both dimensions (height and width) are divisible by 'factor'.

299 + 2. The total number of pixels is within the range

300 + ['min_pixels', 'max_pixels'].

301 + 3. The aspect ratio of the image is maintained as closely as possible.

302 + """

303 + if height < factor or width < factor:

304 + print(f"height:{height} or width:{width} must be "

305 + f"larger than factor:{factor}")

306 + if height < width:

307 + width = round(factor / height * width)

308 + height = factor

309 + else:

310 + height = round(factor / width * height)

311 + width = factor

312 +

313 + elif max(height, width) / min(height, width) > 200:

314 + print(f"absolute aspect ratio must be smaller than 200, "

315 + f"got {max(height, width) / min(height, width)}")

316 + if height > width:

317 + height = 200 * width

318 + else:

319 + width = 200 * height

320 +

321 + h_bar = round(height / factor) * factor

322 + w_bar = round(width / factor) * factor

323 + if h_bar * w_bar > max_pixels:

324 + beta = math.sqrt((height * width) / max_pixels)

325 + h_bar = math.floor(height / beta / factor) * factor

326 + w_bar = math.floor(width / beta / factor) * factor

327 + elif h_bar * w_bar < min_pixels:

328 + beta = math.sqrt(min_pixels / (height * width))

329 + h_bar = math.ceil(height * beta / factor) * factor

330 + w_bar = math.ceil(width * beta / factor) * factor

331 + return h_bar, w_bar

332 +

333 + def get_token_value(self, tok):

334 + return self.extra_special_tokens[tok]

335 +

336 + def construct_visual_indicators(self, grid, is_video: bool = False):

337 + if is_video:

338 + start_token = self.get_token_value('video_start')

339 + end_token = self.get_token_value('video_end')

340 + else:

341 + start_token = self.get_token_value('image_start')

342 + end_token = self.get_token_value('image_end')

343 +

344 + image_placeholders = [start_token, self.get_token_value('visual_atom')]

345 + if grid[0] * grid[1] > 1:

346 + for r in range(grid[0]):

347 + for c in range(grid[1]):

348 + image_placeholders.append(

349 + self.get_token_value('visual_atom'))

350 +

351 + image_placeholders.append(end_token)

352 + return image_placeholders

353 +

354 + def construct_visual_placeholders(self, grid, is_video: bool = False):

355 + visual_placeholders = self.construct_visual_indicators((1, 1),

356 + is_video)

357 +

358 + image_atom_token_id = self.get_token_value('visual_atom')

359 + # Extract the padding token ID from tokenizer

360 + image_padding_token_id = self.get_token_value('image_pad')

361 +

362 + num_image_atoms = grid[0] * grid[1] * grid[2]

363 + num_image_atoms //= self.hidden_stride**2

364 + num_image_atoms //= self.temporal_patch_size

365 +

366 + # Create a new list with padding tokens inserted

367 + padded_placeholder_tokens = []

368 + for token in visual_placeholders:

369 + if token == image_atom_token_id:

370 + padded_placeholder_tokens.extend([image_padding_token_id] *

371 + num_image_atoms)

372 + else:

373 + padded_placeholder_tokens.append(image_padding_token_id)

374 + return padded_placeholder_tokens

375 +

376 + def preprocess_multidata(

377 + self,

378 + images: Optional[Union[PIL.Image.Image, list[PIL.Image.Image]]] = None,

379 + video: Optional[Union[list[PIL.Image.Image], np.ndarray]] = None,

380 + convert_to_rgb: Optional[bool] = True,

381 + min_pixels: int = MIN_PIXELS,

382 + max_pixels: int = MAX_PIXELS,

383 + return_tensors: Optional[str] = 'pt',

384 + ):

385 + is_video = False

386 + if images is not None:

387 + if not isinstance(images, list):

388 + images = [images]

389 + elif video is not None:

390 + is_video = True

391 + # type of vidoe in dummy_mm_data is np.ndarray

392 + if isinstance(video, np.ndarray):

393 + images = []

394 + for i in range(video.shape[0]):

395 + image = PIL.Image.fromarray(video[i].astype(np.uint8))

396 + images.append(image)

397 + elif isinstance(video, list):

398 + images = video

399 + min_pixels = min(max_pixels if max_pixels is not None else MAX_PIXELS,

400 + min_pixels if min_pixels is not None else MIN_PIXELS)

401 + images = [

402 + image.convert("RGB")

403 + if convert_to_rgb and image.mode != 'RGB' else image

404 + for image in images

405 + ]

406 +

407 + width, height = images[0].size

408 + resized_height, resized_width = height, width

409 + processed_images = []

410 + for image in images:

411 + resized_height, resized_width = self.smart_resize(

412 + height,

413 + width,

414 + factor=self.patch_size * self.hidden_stride,

415 + min_pixels=min_pixels,

416 + max_pixels=max_pixels,

417 + )

418 + new_size = dict(height=resized_height, width=resized_width)

419 + image_pt = self.image_processor.preprocess(

420 + image, size=new_size, return_tensors="np")['pixel_values'][0]

421 +

422 + processed_images.append(image_pt)

423 +

424 + patches = np.array(processed_images)

425 + if patches.shape[0] % self.temporal_patch_size != 0:

426 + num_to_pad = self.temporal_patch_size - (patches.shape[0] %

427 + self.temporal_patch_size)

428 + repeats = np.repeat(patches[-1][np.newaxis], num_to_pad, axis=0)

429 + patches = np.concatenate([patches, repeats], axis=0)

430 + channel = patches.shape[1]

431 + grid_t = patches.shape[0] // self.temporal_patch_size

432 + grid_h = resized_height // self.patch_size

433 + grid_w = resized_width // self.patch_size

434 +

435 + patches = patches.reshape(

436 + grid_t,

437 + self.temporal_patch_size,

438 + channel,

439 + grid_h // self.hidden_stride,

440 + self.hidden_stride,

441 + self.patch_size,

442 + grid_w // self.hidden_stride,

443 + self.hidden_stride,

444 + self.patch_size,

445 + )

446 + patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)

447 + flatten_patches = patches.reshape(

448 + grid_t * grid_h * grid_w, channel * self.temporal_patch_size *

449 + self.patch_size * self.patch_size)

450 +

451 + visual_placeholders = self.construct_visual_placeholders(

452 + [grid_t, grid_h, grid_w], is_video)

453 + return torch.tensor(

454 + flatten_patches), visual_placeholders, torch.tensor(

455 + [[grid_t, grid_h, grid_w]])

456 +

457 +

458 +AutoProcessor.register("Ovis2_5Processor", Ovis2_5Processor)