@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 29c6fbe58c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 23:45 UTC

[MODEL] New model support for naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B (#20931)

Signed-off-by: bigshanedogg <bigshane319@gmail.com>

bigshanedogg committed Jul 25, 2025 at 22:05 UTC 29c6fbe58cfa705c26ed1b38f262d5ade0b4f9ba
7 files changed +1365
docs/models/supported_models.md
+1
@@ -365,6 +365,7 @@ th {
365 | `Grok1ModelForCausalLM` | Grok1 | `hpcai-tech/grok-1`. | ✅︎ | ✅︎ | ✅︎ |
366 | `HunYuanDenseV1ForCausalLM` | Hunyuan-7B-Instruct-0124 | `tencent/Hunyuan-7B-Instruct-0124` | ✅︎ | | ✅︎ |
367 | `HunYuanMoEV1ForCausalLM` | Hunyuan-80B-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | | ✅︎ |
368 +| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | | ✅︎ |
369 | `InternLMForCausalLM` | InternLM | `internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc. | ✅︎ | ✅︎ | ✅︎ |
370 | `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ | ✅︎ |
371 | `InternLM3ForCausalLM` | InternLM3 | `internlm/internlm3-8b-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+80
@@ -316,6 +316,85 @@ def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
316 )
317
318
319 +# naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B
320 +def run_hyperclovax_seed_vision(
321 + questions: list[str], modality: str
322 +) -> ModelRequestData:
323 + model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
324 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
325 +
326 + engine_args = EngineArgs(
327 + model=model_name,
328 + trust_remote_code=True,
329 + max_model_len=8192 if modality == "image" else 16384,
330 + limit_mm_per_prompt={modality: 1},
331 + )
332 +
333 + messages = list()
334 + for question in questions:
335 + if modality == "image":
336 + """
337 + ocr: List the words in the image in raster order.
338 + Even if the word order feels unnatural for reading,
339 + the model will handle it as long as it follows raster order.
340 + e.g. "Naver, CLOVA, bigshane"
341 + lens_keywords: List the entity names in the image.
342 + e.g. "iPhone"
343 + lens_local_keywords: List the entity names with quads in the image.
344 + e.g. "[0.07, 0.21, 0.92, 0.90] iPhone"
345 + """
346 + messages.append(
347 + [
348 + {
349 + "role": "user",
350 + "content": [
351 + {
352 + "type": "image",
353 + "ocr": "",
354 + "lens_keywords": "",
355 + "lens_local_keywords": "",
356 + },
357 + {
358 + "type": "text",
359 + "text": question,
360 + },
361 + ],
362 + }
363 + ]
364 + )
365 + elif modality == "video":
366 + messages.append(
367 + [
368 + {
369 + "role": "user",
370 + "content": [
371 + {
372 + "type": "video",
373 + },
374 + {
375 + "type": "text",
376 + "text": question,
377 + },
378 + ],
379 + }
380 + ]
381 + )
382 + else:
383 + raise ValueError(f"Unsupported modality: {modality}")
384 +
385 + prompts = tokenizer.apply_chat_template(
386 + messages,
387 + tokenize=False,
388 + add_generation_prompt=True,
389 + )
390 +
391 + return ModelRequestData(
392 + engine_args=engine_args,
393 + prompts=prompts,
394 + stop_token_ids=None,
395 + )
396 +
397 +
398 # Idefics3-8B-Llama3
399 def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
400 assert modality == "image"
@@ -1222,6 +1301,7 @@ model_example_map = {
1301 "glm4v": run_glm4v,
1302 "glm4_1v": run_glm4_1v,
1303 "h2ovl_chat": run_h2ovl,
1304 + "hyperclovax_seed_vision": run_hyperclovax_seed_vision,
1305 "idefics3": run_idefics3,
1306 "internvl_chat": run_internvl,
1307 "nemotron_vl": run_nemotron_vl,
examples/offline_inference/vision_language_multi_image.py
+48
@@ -289,6 +289,53 @@ def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
289 )
290
291
292 +def load_hyperclovax_seed_vision(
293 + question: str, image_urls: list[str]
294 +) -> ModelRequestData:
295 + model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
296 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
297 +
298 + engine_args = EngineArgs(
299 + model=model_name,
300 + trust_remote_code=True,
301 + max_model_len=16384,
302 + limit_mm_per_prompt={"image": len(image_urls)},
303 + )
304 +
305 + message = {"role": "user", "content": list()}
306 + for _image_url in image_urls:
307 + message["content"].append(
308 + {
309 + "type": "image",
310 + "image": _image_url,
311 + "ocr": "",
312 + "lens_keywords": "",
313 + "lens_local_keywords": "",
314 + }
315 + )
316 + message["content"].append(
317 + {
318 + "type": "text",
319 + "text": question,
320 + }
321 + )
322 +
323 + prompt = tokenizer.apply_chat_template(
324 + [
325 + message,
326 + ],
327 + tokenize=False,
328 + add_generation_prompt=True,
329 + )
330 +
331 + return ModelRequestData(
332 + engine_args=engine_args,
333 + prompt=prompt,
334 + stop_token_ids=None,
335 + image_data=[fetch_image(url) for url in image_urls],
336 + )
337 +
338 +
339 def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
340 # NOTE: CAUTION! Original Llava models wasn't really trained on multi-image inputs,
341 # it will generate poor response for multi-image inputs!
@@ -900,6 +947,7 @@ model_example_map = {
947 "h2ovl_chat": load_h2ovl,
948 "idefics3": load_idefics3,
949 "internvl_chat": load_internvl,
950 + "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
951 "keye_vl": load_keye_vl,
952 "kimi_vl": load_kimi_vl,
953 "llava": load_llava,
tests/models/multimodal/processing/test_common.py
+1
@@ -278,6 +278,7 @@ def _test_processing_correctness_one(
278 "HuggingFaceTB/SmolVLM2-2.2B-Instruct",
279 "moonshotai/Kimi-VL-A3B-Instruct",
280 "meta-llama/Llama-4-Scout-17B-16E-Instruct",
281 + "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
282 "llava-hf/llava-1.5-7b-hf",
283 "llava-hf/llava-v1.6-mistral-7b-hf",
284 "llava-hf/LLaVA-NeXT-Video-7B-hf",
tests/models/registry.py
+3
@@ -201,6 +201,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
201 trust_remote_code=True),
202 "HunYuanDenseV1ForCausalLM":_HfExamplesInfo("tencent/Hunyuan-7B-Instruct-0124",
203 trust_remote_code=True),
204 + "HCXVisionForCausalLM": _HfExamplesInfo(
205 + "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
206 + trust_remote_code=True),
207 "InternLMForCausalLM": _HfExamplesInfo("internlm/internlm-chat-7b",
208 trust_remote_code=True),
209 "InternLM2ForCausalLM": _HfExamplesInfo("internlm/internlm2-chat-7b",
vllm/model_executor/models/hyperclovax_vision.py new
+1231
@@ -0,0 +1,1231 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# copied from : https://github.com/huggingface/transformers
4 +import ast
5 +import sys
6 +from collections import defaultdict
7 +from collections.abc import Iterable, Mapping, Sequence
8 +from functools import partial
9 +from itertools import chain
10 +from typing import Any, Literal, Optional, TypedDict, Union
11 +
12 +import numpy as np
13 +import PIL
14 +from einops import rearrange
15 +from PIL import Image
16 +
17 +if sys.version_info >= (3, 11):
18 + import typing
19 + Unpack = typing.Unpack
20 +else:
21 + import typing_extensions
22 + Unpack = typing_extensions.Unpack
23 +
24 +import torch
25 +import torch.nn as nn
26 +from timm.layers import LayerNorm, LayerNorm2d
27 +from timm.models.regnet import RegStage
28 +from transformers import (AutoProcessor, BatchFeature, CLIPVisionConfig,
29 + SiglipVisionConfig)
30 +from transformers.modeling_utils import no_init_weights
31 +
32 +from vllm.config import VllmConfig
33 +from vllm.inputs import InputProcessingContext
34 +from vllm.model_executor.layers.quantization import QuantizationConfig
35 +from vllm.model_executor.sampling_metadata import SamplingMetadata
36 +from vllm.multimodal import MULTIMODAL_REGISTRY
37 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
38 + MultiModalKwargs)
39 +from vllm.multimodal.parse import ImageSize, MultiModalDataItems
40 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 + BaseProcessingInfo, ProcessingCache,
42 + PromptReplacement, PromptUpdate)
43 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
44 +from vllm.sequence import IntermediateTensors
45 +
46 +from .clip import CLIPVisionModel
47 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
48 +from .siglip import SiglipVisionModel
49 +from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
50 +from .vision import get_vision_encoder_info
51 +
52 +EOT = "<|endofturn|>"
53 +IMAGE_TOKEN: str = "<|dummy3|>"
54 +VIDEO_TOKEN: str = "<|_unuse_missing_100270|>"
55 +
56 +
57 +class HCXVisionMultimodalPixelInputs(TypedDict):
58 + type: Literal["pixel_values"]
59 + pixel_values_images: list[torch.Tensor]
60 + """
61 + Shape: `[(num_grids, num_channels, height, width), ...]` if anyres
62 +
63 + Note that `height` or `width` may be different per batch and image,
64 + in which case the data is passed as a list instead of a batched tensor.
65 + """
66 + image_sizes_images: list[tuple[Union[int, float]]]
67 + """
68 + Shape: `[(height, width), ...]`
69 + """
70 + vision_query_lengths_images: list[Union[int, float]]
71 + pixel_values_videos: list[tuple[Union[int, float]]]
72 + """
73 + Shape: `[(num_grids, num_channels, height, width), ...]` if anyres
74 + """
75 + vision_query_lengths_videos: list[Union[int, float]]
76 +
77 +
78 +HCXVisionMultimodalInputs = Union[HCXVisionMultimodalPixelInputs]
79 +
80 +
81 +class HCXVisionProcessingInfo(BaseProcessingInfo):
82 +
83 + def get_hf_config(self):
84 + return self.ctx.get_hf_config()
85 +
86 + def get_vision_encoder_info(self):
87 + return get_vision_encoder_info(self.get_hf_config())
88 +
89 + def get_hf_processor(
90 + self,
91 + **kwargs: object,
92 + ):
93 + processor_cls = type(
94 + AutoProcessor.from_pretrained(
95 + self.ctx.model_config.model,
96 + trust_remote_code=self.ctx.model_config.trust_remote_code,
97 + ))
98 + return self.ctx.get_hf_processor(
99 + processor_cls,
100 + **kwargs,
101 + )
102 +
103 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104 + return {"image": None, "video": None}
105 +
106 + def get_num_image_tokens(
107 + self,
108 + *,
109 + vision_query_length: Union[int, list[int]],
110 + ) -> int:
111 + if isinstance(vision_query_length, int):
112 + return vision_query_length
113 + else:
114 + return sum(vision_query_length)
115 +
116 + def get_num_video_tokens(
117 + self,
118 + *,
119 + vision_query_length: Union[int, list[int]],
120 + ) -> int:
121 + if isinstance(vision_query_length, int):
122 + return vision_query_length
123 + else:
124 + return sum(vision_query_length)
125 +
126 + def get_image_size_with_most_features(self) -> ImageSize:
127 + vision_encoder_info = self.get_vision_encoder_info()
128 + width = height = vision_encoder_info.get_image_size()
129 + return ImageSize(width=width, height=height)
130 +
131 + def get_max_image_tokens(self) -> int:
132 + target_width, target_height = self.get_image_size_with_most_features()
133 +
134 + return self.get_num_image_tokens(
135 + image_width=target_width,
136 + image_height=target_height,
137 + )
138 +
139 +
140 +class HCXVisionDummyInputsBuilder(
141 + BaseDummyInputsBuilder[HCXVisionProcessingInfo]):
142 +
143 + def get_dummy_text(
144 + self,
145 + mm_counts: Mapping[str, int],
146 + ) -> str:
147 + dummy_text = IMAGE_TOKEN * mm_counts.get(
148 + "image", 0) + VIDEO_TOKEN * mm_counts.get("video", 0)
149 + return dummy_text
150 +
151 + def get_dummy_mm_data(
152 + self,
153 + seq_len: int,
154 + mm_counts: Mapping[str, int],
155 + ) -> MultiModalDataDict:
156 + num_images = mm_counts.get("image", 0)
157 + num_videos = mm_counts.get("video", 0)
158 +
159 + target_width, target_height = \
160 + self.info.get_image_size_with_most_features()
161 + target_num_frames = 32
162 + return {
163 + "image":
164 + self._get_dummy_images(
165 + width=target_width,
166 + height=target_height,
167 + num_images=num_images,
168 + ),
169 + "video":
170 + self._get_dummy_videos(
171 + width=target_width - 1,
172 + height=target_height - 1,
173 + num_frames=target_num_frames,
174 + num_videos=num_videos,
175 + )
176 + }
177 +
178 +
179 +class HCXVisionMultiModalProcessor(
180 + BaseMultiModalProcessor[HCXVisionProcessingInfo]):
181 +
182 + def _call_hf_processor(
183 + self,
184 + prompt: str,
185 + mm_data: Mapping[str, object],
186 + mm_kwargs: Mapping[str, object],
187 + tok_kwargs: Mapping[str, object],
188 + ) -> BatchFeature:
189 +
190 + def replace_multimodal_token(
191 + token_ids: torch.Tensor,
192 + target_token: int,
193 + repeats: list,
194 + ):
195 + output = list()
196 + _repeats_idx = 0
197 + for token_id in token_ids:
198 + if token_id == target_token:
199 + output += [
200 + token_id.item(),
201 + ] * repeats[_repeats_idx]
202 + _repeats_idx += 1
203 + else:
204 + output += [
205 + token_id.item(),
206 + ]
207 + return torch.tensor(output, device=token_ids.device)
208 +
209 + for video_idx, video_arr in enumerate(mm_data.get("videos", list())):
210 + if video_arr.dtype == np.uint8:
211 + continue
212 + mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
213 +
214 + processed_outputs = self.info.ctx.call_hf_processor(
215 + hf_processor=self.info.get_hf_processor(**mm_kwargs),
216 + data=dict(
217 + text=prompt,
218 + images=None,
219 + videos=None,
220 + ),
221 + ) # text-only
222 +
223 + if len(mm_data) > 0:
224 + # batchify input as a single item
225 + images = mm_data.get("images", None)
226 + num_images = 0
227 + if images is not None:
228 + num_images = len(images)
229 + images = [
230 + images,
231 + ] # batchify
232 +
233 + videos = mm_data.get("videos",
234 + None) # list of video in single conversation
235 + num_videos = 0
236 + if videos is not None:
237 + num_videos = len(videos)
238 + videos = [
239 + videos,
240 + ] # batchify
241 +
242 + _processed_outputs = self.info.ctx.call_hf_processor(
243 + hf_processor=self.info.get_hf_processor(**mm_kwargs),
244 + data=dict(
245 + text=None,
246 + images=images,
247 + videos=videos,
248 + ),
249 + ) # mm-only
250 +
251 + for k, v in _processed_outputs.items():
252 + if len(v) < 1:
253 + continue
254 + elif k.endswith("_images"):
255 + # list of list of 4D tensor -> list of 4D tensor
256 + _processed_outputs[k] = v[0]
257 + elif k.endswith("_videos"):
258 + # list of list of 4D tensor -> list of 4D tensor
259 + v = v[0]
260 + if k == "pixel_values_videos":
261 + v = torch.cat(v, dim=0)
262 + _c, _w, _h = v.shape[-3:]
263 + v = v.reshape(num_videos, -1, _c, _w, _h)
264 + v = list(torch.unbind(v, dim=0))
265 + _processed_outputs[k] = v
266 +
267 + if num_images > 0:
268 + tokenizer = self.info.get_tokenizer()
269 + processed_outputs["input_ids"] = torch.stack([
270 + replace_multimodal_token(
271 + token_ids=_input_ids,
272 + target_token=tokenizer.convert_tokens_to_ids(
273 + IMAGE_TOKEN),
274 + repeats=_processed_outputs[
275 + "vision_query_lengths_images"],
276 + ) for _input_ids in processed_outputs["input_ids"]
277 + ],
278 + dim=0)
279 +
280 + if num_videos > 0:
281 + tokenizer = self.info.get_tokenizer()
282 + processed_outputs["input_ids"] = torch.stack([
283 + replace_multimodal_token(
284 + token_ids=_input_ids,
285 + target_token=tokenizer.convert_tokens_to_ids(
286 + VIDEO_TOKEN),
287 + repeats=_processed_outputs[
288 + "vision_query_lengths_videos"],
289 + ) for _input_ids in processed_outputs["input_ids"]
290 + ],
291 + dim=0)
292 +
293 + _ratios = [
294 + len(_pixel_values) for _pixel_values in
295 + _processed_outputs["pixel_values_videos"]
296 + ]
297 + _num_per_videos = [
298 + int(_e / sum(_ratios) *
299 + len(_processed_outputs["vision_query_lengths_videos"]))
300 + for _e in _ratios
301 + ]
302 + _processed_outputs["vision_query_lengths_videos"] = [
303 + _processed_outputs["vision_query_lengths_videos"]
304 + [sum(_num_per_videos[:_i]):sum(_num_per_videos[:_i + 1])]
305 + for _i in range(0, num_videos)
306 + ]
307 +
308 + processed_outputs.update(_processed_outputs)
309 +
310 + return processed_outputs
311 +
312 + def _get_prompt_updates(
313 + self,
314 + mm_items: MultiModalDataItems,
315 + hf_processor_mm_kwargs: Mapping[str, object],
316 + out_mm_kwargs: MultiModalKwargs,
317 + ) -> Sequence[PromptUpdate]:
318 + hf_config = self.info.get_hf_config()
319 + placeholder = {
320 + "image": hf_config.image_token_id,
321 + "video": hf_config.video_token_id,
322 + }
323 +
324 + def get_replacement_hyperclovax(
325 + item_idx: int,
326 + modality: str,
327 + out_mm_kwargs: MultiModalKwargs,
328 + ):
329 + num_tokens = None
330 + if modality == "image":
331 + num_tokens = self.info.get_num_image_tokens(
332 + vision_query_length=out_mm_kwargs[
333 + "vision_query_lengths_images"][item_idx], )
334 + if modality == "video":
335 + num_tokens = self.info.get_num_video_tokens(
336 + vision_query_length=out_mm_kwargs[
337 + "vision_query_lengths_videos"][item_idx], )
338 + assert isinstance(num_tokens, int)
339 + return [
340 + placeholder[modality],
341 + ] * num_tokens
342 +
343 + return [
344 + PromptReplacement(
345 + modality=modality,
346 + target=[
347 + placeholder[modality],
348 + ],
349 + replacement=partial(
350 + get_replacement_hyperclovax,
351 + modality=modality,
352 + out_mm_kwargs=out_mm_kwargs,
353 + ),
354 + ) for modality in ("image", "video")
355 + ]
356 +
357 + def _get_mm_fields_config(
358 + self,
359 + hf_inputs: BatchFeature,
360 + hf_processor_mm_kwargs: Mapping[str, object],
361 + ) -> Mapping[str, MultiModalFieldConfig]:
362 + return dict(
363 + # image
364 + pixel_values_images=MultiModalFieldConfig.batched("image"),
365 + image_sizes_images=MultiModalFieldConfig.batched("image"),
366 + vision_query_lengths_images=MultiModalFieldConfig.batched("image"),
367 + num_queries_vis_abstractors_images=MultiModalFieldConfig.batched(
368 + "image"),
369 + num_queries_vis_abstractors_slow_images=MultiModalFieldConfig.
370 + batched("image"),
371 + first_last_frames_slows_images=MultiModalFieldConfig.batched(
372 + "image"),
373 + # video
374 + pixel_values_videos=MultiModalFieldConfig.batched("video"),
375 + image_sizes_videos=MultiModalFieldConfig.batched("video"),
376 + vision_query_lengths_videos=MultiModalFieldConfig.batched("video"),
377 + num_queries_vis_abstractors_videos=MultiModalFieldConfig.batched(
378 + "video"),
379 + num_queries_vis_abstractors_slow_videos=MultiModalFieldConfig.
380 + batched("video"),
381 + first_last_frames_slows_videos=MultiModalFieldConfig.batched(
382 + "video"),
383 + )
384 +
385 +
386 +def _build_hcxvision_hf_info(
387 + ctx: InputProcessingContext, ) -> HCXVisionProcessingInfo:
388 + return HCXVisionProcessingInfo(ctx)
389 +
390 +
391 +def _build_hcxvision_hf_processor(
392 + info: HCXVisionProcessingInfo,
393 + dummy_inputs: BaseDummyInputsBuilder[HCXVisionProcessingInfo],
394 + *,
395 + cache: Optional[ProcessingCache] = None,
396 +) -> BaseMultiModalProcessor:
397 + if isinstance(info, HCXVisionProcessingInfo):
398 + return HCXVisionMultiModalProcessor(
399 + info,
400 + dummy_inputs, # type: ignore
401 + cache=cache,
402 + )
403 +
404 + raise NotImplementedError(type(info))
405 +
406 +
407 +def init_vision_tower_for_hcxvision(
408 + vision_config,
409 + quant_config: Optional[QuantizationConfig],
410 + *,
411 + use_nth_layer: Optional[int] = None,
412 + require_post_norm: Optional[bool] = None,
413 + prefix: str = "",
414 +) -> Union[CLIPVisionModel, SiglipVisionModel]:
415 + num_hidden_layers = vision_config.num_hidden_layers
416 + if not isinstance(use_nth_layer, int):
417 + pass
418 + elif use_nth_layer >= 0:
419 + num_hidden_layers = use_nth_layer + 1
420 + else:
421 + num_hidden_layers = num_hidden_layers + use_nth_layer + 1
422 +
423 + if isinstance(vision_config, CLIPVisionConfig):
424 + return CLIPVisionModel(
425 + vision_config,
426 + quant_config=quant_config,
427 + num_hidden_layers_override=num_hidden_layers,
428 + require_post_norm=require_post_norm,
429 + prefix=prefix,
430 + )
431 + elif isinstance(vision_config, SiglipVisionConfig):
432 + return SiglipVisionModel(
433 + vision_config,
434 + quant_config=quant_config,
435 + num_hidden_layers_override=num_hidden_layers,
436 + require_post_norm=require_post_norm,
437 + prefix=prefix,
438 + )
439 +
440 + msg = f"Unsupported vision config: {type(vision_config)}"
441 + raise NotImplementedError(msg)
442 +
443 +
444 +class HCXVisionMlp(nn.Module):
445 +
446 + def __init__(
447 + self,
448 + mm_projector_type,
449 + in_features,
450 + hidden_features=None,
451 + out_features=None,
452 + act_layer=nn.GELU,
453 + ):
454 + super().__init__()
455 + out_features = out_features or in_features
456 + hidden_features = hidden_features or in_features
457 + self.mm_projector_type = mm_projector_type
458 + if self.mm_projector_type == "mlp":
459 + self.fc1 = nn.Linear(in_features, hidden_features)
460 + self.act = act_layer()
461 + self.fc2 = nn.Linear(hidden_features, out_features)
462 + elif self.mm_projector_type == "inverted_mlp":
463 + self.fc1 = nn.Linear(in_features, 2 * hidden_features)
464 + self.act = act_layer()
465 + self.fc2 = nn.Linear(2 * hidden_features, out_features)
466 + else:
467 + raise NotImplementedError("{} is not implemented".format(
468 + self.mm_projector_type))
469 +
470 + def forward(self, x):
471 + x = self.fc1(x)
472 + x = self.act(x)
473 + x = self.fc2(x)
474 + return x
475 +
476 +
477 +class HCXVisionCAbstractor(nn.Module):
478 + """
479 + This module is based on C-Abstractor, whose license is under apache-2.0.
480 + You can check the original code at
481 + https://github.com/khanrc/honeybee/blob/main/honeybee/projectors/projectors.py
482 + and we made necessary modifications.
483 + """
484 +
485 + def __init__(
486 + self,
487 + num_queries: int,
488 + num_input_tokens: int,
489 + encoder_hidden_size: int,
490 + hidden_size: int,
491 + output_hidden_size: int,
492 + pos_emb: bool = True,
493 + prenorm: bool = False,
494 + ):
495 + super().__init__()
496 + self.num_input_tokens = num_input_tokens
497 + self.output_hidden_size = output_hidden_size
498 +
499 + # Positional embedding
500 + if pos_emb:
501 + self.pos_emb = torch.nn.Parameter(
502 + torch.zeros(1, num_input_tokens, encoder_hidden_size))
503 + self.pos_emb.data.normal_(mean=0.0, std=0.02)
504 + else:
505 + self.pos_emb = None
506 +
507 + # (Optional) Pre-normalization layer
508 + if prenorm:
509 + self.prenorm = LayerNorm(encoder_hidden_size)
510 + else:
511 + self.prenorm = None
512 +
513 + self.build_net(num_queries, encoder_hidden_size, hidden_size,
514 + output_hidden_size)
515 + self.dtype = next(self.parameters()).dtype
516 +
517 + def forward(
518 + self,
519 + x: torch.Tensor,
520 + num_queries_vis_abstractors: Optional[list[list[int]]] = None,
521 + num_grids: Optional[list[int]] = None,
522 + ) -> torch.Tensor:
523 + if self.prenorm is not None:
524 + x = self.prenorm(x)
525 +
526 + if self.pos_emb is not None:
527 + x = x + self.pos_emb
528 +
529 + x = self._forward(
530 + x,
531 + num_queries_vis_abstractors=num_queries_vis_abstractors,
532 + num_grids=num_grids,
533 + ) # (B, L, output_hidden_size)
534 +
535 + return x
536 +
537 + def _forward(
538 + self,
539 + x: torch.Tensor,
540 + num_queries_vis_abstractors: Optional[list[list[int]]] = None,
541 + num_grids: Optional[list[int]] = None,
542 + ) -> torch.Tensor:
543 + # x: [B, L, dim]
544 + B, L, dim = x.shape
545 + hw = int(L**0.5)
546 + x = rearrange(x, "b (h w) d -> b d h w", h=hw, w=hw)
547 +
548 + if num_queries_vis_abstractors is not None:
549 + assert num_grids is not None
550 + return self._forward_adaptive_num_query(
551 + x, num_queries_vis_abstractors, num_grids)
552 +
553 + x = self.net(x)
554 + x = rearrange(x, "b d h w -> b (h w) d")
555 + x = self.readout(x)
556 + return x
557 +
558 + def _forward_adaptive_num_query(
559 + self,
560 + x: torch.Tensor,
561 + num_queries_vis_abstractors: Optional[list[list[int]]] = None,
562 + num_grids: Optional[list[int]] = None,
563 + ) -> list[torch.Tensor]:
564 + # self.net is consisted by 3 layers (s1, sampler, s2)
565 + assert len(self.net) == 3
566 +
567 + x = self.net[0](x) # s1
568 + new_x = []
569 + for i, num_queries in enumerate(num_queries_vis_abstractors):
570 + hw = int(num_queries**0.5)
571 + sampler = nn.AdaptiveAvgPool2d((hw, hw))
572 + out = sampler(x[num_grids[i]:num_grids[i + 1], :])
573 + out = self.net[2](out) # s2
574 +
575 + out = rearrange(out, "b d h w -> b (h w) d")
576 + out = self.readout(out)
577 +
578 + new_x.append(out)
579 + return new_x
580 +
581 + def build_net(
582 + self,
583 + n_queries: int,
584 + encoder_hidden_size: int,
585 + hidden_size: int,
586 + output_hidden_size: int,
587 + depth: int = 3,
588 + mlp_depth: int = 2,
589 + ):
590 + assert (n_queries**0.5).is_integer(
591 + ), f"n_queries must be square number. n_queries: {n_queries}"
592 + hw = int(n_queries**0.5)
593 +
594 + # RegBlock = ResBlock + SE
595 + RegBlock = partial(
596 + RegStage,
597 + stride=1,
598 + dilation=1,
599 + act_layer=nn.SiLU,
600 + norm_layer=LayerNorm2d,
601 + )
602 +
603 + s1 = RegBlock(
604 + depth,
605 + encoder_hidden_size,
606 + hidden_size,
607 + )
608 + sampler = nn.AdaptiveAvgPool2d((hw, hw))
609 + s2 = RegBlock(
610 + depth,
611 + hidden_size,
612 + hidden_size,
613 + )
614 +
615 + self.net = nn.Sequential(s1, sampler, s2)
616 + self.readout = self.build_mlp(mlp_depth, hidden_size,
617 + output_hidden_size)
618 +
619 + def build_mlp(
620 + self,
621 + depth: int,
622 + hidden_size: int,
623 + output_hidden_size: int,
624 + ):
625 + layers = [nn.Linear(hidden_size, output_hidden_size)]
626 + for _ in range(1, depth):
627 + layers.append(nn.SiLU())
628 + layers.append(nn.Linear(output_hidden_size, output_hidden_size))
629 + return nn.Sequential(*layers)
630 +
631 +
632 +@MULTIMODAL_REGISTRY.register_processor(
633 + _build_hcxvision_hf_processor,
634 + info=_build_hcxvision_hf_info,
635 + dummy_inputs=HCXVisionDummyInputsBuilder)
636 +class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
637 +
638 + packed_modules_mapping = {
639 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
640 + "gate_up_proj": ["gate_proj", "up_proj"]
641 + }
642 +
643 + def __init__(
644 + self,
645 + *,
646 + vllm_config: VllmConfig,
647 + prefix: str = "",
648 + **kwargs: Optional[Any],
649 + ) -> None:
650 + super().__init__()
651 +
652 + # init configs
653 + config = vllm_config.model_config.hf_config
654 + quant_config = vllm_config.quant_config
655 + # text_config
656 + text_config = config.text_config
657 + if text_config.model_type in ["gpt2", "hyperclovax", "llama"]:
658 + text_config._attn_implementation = "sdpa"
659 + if text_config.model_type != "hyperclovax":
660 + text_config.logits_scaling = 1.0
661 + # vision_config
662 + vision_config = config.vision_config
663 + vision_config.auto_map = {}
664 + vision_config.anyres = config.anyres
665 + vision_config.max_num_grids = config.max_num_grids
666 + self.dtype = vllm_config.model_config.dtype
667 +
668 + ## possible_resolution should be matched with preprocessor_config.json
669 + config.possible_resolutions = self._init_possible_resolutions(
670 + config, vision_config)
671 +
672 + # init models & parameters
673 + with no_init_weights(): # weight will be loaded in from_pretrained
674 + self.vision_model = init_vision_tower_for_hcxvision(
675 + vision_config,
676 + quant_config,
677 + use_nth_layer=getattr(config, "use_nth_layer", -1),
678 + require_post_norm=False,
679 + prefix=maybe_prefix(prefix, "vision_model"),
680 + )
681 + self.mm_projector = self._init_mm_projector(config, text_config,
682 + vision_config)
683 +
684 + self.lm_head_vocab_size = getattr(text_config, "padded_vocab_size",
685 + text_config.vocab_size)
686 + self.language_model = init_vllm_registered_model(
687 + vllm_config=vllm_config,
688 + hf_config=text_config,
689 + prefix=maybe_prefix(prefix, "language_model"),
690 + )
691 +
692 + if config.anyres:
693 + self.image_newline = nn.Parameter(
694 + torch.empty(text_config.hidden_size, dtype=self.dtype))
695 +
696 + self.config = config
697 + self.vision_config = vision_config
698 + self.text_config = text_config
699 +
700 + # use_sum_loss = bool(kwargs.pop("use_sum_loss", False))
701 + # self.reduction = self._init_reduction_type(use_sum_loss)
702 +
703 + @classmethod
704 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
705 + if modality.startswith("image"):
706 + return IMAGE_TOKEN
707 + if modality.startswith("video"):
708 + return VIDEO_TOKEN
709 +
710 + raise ValueError("Only image or video modality is supported")
711 +
712 + def get_language_model(self) -> torch.nn.Module:
713 + return self.language_model
714 +
715 + def get_multimodal_embeddings(
716 + self,
717 + **kwargs: Unpack[HCXVisionMultimodalInputs],
718 + ) -> Optional[MultiModalEmbeddings]:
719 +
720 + multimodal_embeddings = list()
721 + if kwargs.get("pixel_values_images") is not None:
722 + for _pixel_values_images, _image_sizes_images in zip(
723 + kwargs["pixel_values_images"],
724 + kwargs["image_sizes_images"]):
725 + _pixel_values_images = _pixel_values_images.unsqueeze(dim=0)
726 + _image_sizes_images = _image_sizes_images.unsqueeze(dim=0)
727 + _len_pixel_values_images = [
728 + len(pixel_value) for pixel_value in _pixel_values_images
729 + ]
730 + if isinstance(_image_sizes_images, torch.Tensor):
731 + _image_sizes_images = _image_sizes_images.detach().cpu(
732 + ).tolist()
733 + _multimodal_embeddings_images = self.forward_images(
734 + pixel_values_images=_pixel_values_images,
735 + image_sizes_images=_image_sizes_images,
736 + len_pixel_values_images=_len_pixel_values_images,
737 + )
738 + _multimodal_embeddings_images = torch.cat(
739 + _multimodal_embeddings_images, dim=0)
740 + multimodal_embeddings.append(_multimodal_embeddings_images)
741 +
742 + if kwargs.get("pixel_values_videos") is not None:
743 + for _pixel_values_videos, _vision_query_lengths_videos in zip(
744 + kwargs["pixel_values_videos"],
745 + kwargs["vision_query_lengths_videos"]):
746 + _len_pixel_values_videos = [
747 + len(_vision_query_lengths)
748 + for _vision_query_lengths in _vision_query_lengths_videos
749 + ]
750 + _c, _w, _h = _pixel_values_videos.shape[-3:]
751 + _pixel_values_videos = _pixel_values_videos.reshape(
752 + sum(_len_pixel_values_videos), -1, _c, _w,
753 + _h).unsqueeze(dim=0)
754 + _multimodal_embeddings_videos = self.forward_videos(
755 + pixel_values_videos=_pixel_values_videos,
756 + len_pixel_values_videos=_len_pixel_values_videos,
757 + )
758 + _multimodal_embeddings_videos = torch.cat(
759 + _multimodal_embeddings_videos, dim=0)
760 + multimodal_embeddings.append(_multimodal_embeddings_videos)
761 + return multimodal_embeddings
762 +
763 + def get_input_embeddings(
764 + self,
765 + input_ids: torch.Tensor,
766 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
767 + **kwargs,
768 + ) -> torch.Tensor:
769 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
770 + if (kwargs.get("pixel_values_images") is not None
771 + or kwargs.get("pixel_values_videos")
772 + is not None): # v0 compatibility
773 + multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
774 + if multimodal_embeddings is not None:
775 + multimodal_embeddings = torch.cat(multimodal_embeddings, dim=0)
776 + _mask_image = input_ids == self.config.image_token_id
777 + _mask_video = input_ids == self.config.video_token_id
778 + assert _mask_image.sum() + _mask_video.sum() == len(
779 + multimodal_embeddings)
780 +
781 + if multimodal_embeddings.dtype != inputs_embeds.dtype:
782 + multimodal_embeddings = multimodal_embeddings.to(
783 + dtype=inputs_embeds.dtype)
784 + if multimodal_embeddings.device != inputs_embeds.device:
785 + multimodal_embeddings = multimodal_embeddings.to(
786 + device=inputs_embeds.device)
787 +
788 + if _mask_image.sum() > 0:
789 + inputs_embeds[
790 + _mask_image] = multimodal_embeddings[:sum(_mask_image)]
791 + if _mask_video.sum() > 0:
792 + inputs_embeds[_mask_video] = multimodal_embeddings[
793 + -sum(_mask_video):]
794 + return inputs_embeds
795 +
796 + def forward(
797 + self,
798 + input_ids: torch.Tensor,
799 + positions: torch.Tensor,
800 + intermediate_tensors: Optional[IntermediateTensors] = None,
801 + inputs_embeds: Optional[torch.Tensor] = None,
802 + **kwargs: object,
803 + ) -> Union[torch.Tensor, IntermediateTensors]:
804 + if intermediate_tensors is not None:
805 + inputs_embeds = None
806 +
807 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
808 + # condition is for v0 compatibility.
809 + elif inputs_embeds is None:
810 + inputs_embeds = self.get_input_embeddings(input_ids=input_ids,
811 + **kwargs)
812 + input_ids = None
813 + hidden_states = self.language_model.model(input_ids,
814 + positions,
815 + intermediate_tensors,
816 + inputs_embeds=inputs_embeds)
817 + return hidden_states
818 +
819 + def forward_images(
820 + self,
821 + pixel_values_images: list[list[torch.FloatTensor]],
822 + image_sizes_images: list[list[tuple[int, int]]],
823 + len_pixel_values_images: list[int],
824 + ) -> list[list[torch.Tensor]]:
825 + if sum(len_pixel_values_images) == 0:
826 + return None
827 +
828 + concat_pixel_values_images = torch.cat(list(
829 + chain(*pixel_values_images)),
830 + dim=0)
831 +
832 + visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
833 + image_forward_outs = self.vision_model(
834 + concat_pixel_values_images)[:, visual_token_idx:]
835 +
836 + image_forward_outs = image_forward_outs.to(
837 + dtype=self.mm_projector.dtype)
838 + image_forward_outs = self.mm_projector(image_forward_outs) # b (h w) d
839 +
840 + split_sizes = [
841 + pixel_value.shape[0] for pixel_value in chain(*pixel_values_images)
842 + ]
843 + image_forward_outs = torch.split(image_forward_outs,
844 + split_sizes,
845 + dim=0)
846 +
847 + # newline for anyres postprocessing
848 + image_features = anyres_postprocessing(
849 + image_forward_outs=image_forward_outs,
850 + image_sizes=[
851 + image_size for image_sizes in image_sizes_images
852 + for image_size in image_sizes
853 + ],
854 + num_queries_vis_abstractor=self.config.
855 + num_queries_vis_abstractor_image,
856 + unpad=self.config.unpad,
857 + patch_size=self.vision_config.patch_size,
858 + grid_size=self.vision_config.image_size,
859 + image_newline=self.image_newline,
860 + possible_resolutions=self.config.possible_resolutions,
861 + )
862 + return image_features
863 +
864 + def forward_videos(
865 + self,
866 + pixel_values_videos: list[list[torch.FloatTensor]],
867 + len_pixel_values_videos: list[int],
868 + ) -> list[torch.Tensor]:
869 +
870 + len_video_grids = sum(len_pixel_values_videos)
871 + if len_video_grids == 0:
872 + return None
873 +
874 + # Run Vision Model
875 + concat_pixel_values_videos = torch.cat(list(
876 + chain(*pixel_values_videos)),
877 + dim=0)
878 +
879 + visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
880 + video_forward_outs = self.vision_model(
881 + concat_pixel_values_videos)[:, visual_token_idx:]
882 +
883 + video_forward_outs = video_forward_outs.to(
884 + dtype=self.mm_projector.dtype)
885 +
886 + # Run MM-Projector
887 + # len(num_grids) == len(num_queries_vis_abstractors) + 1
888 + grid_idx = 0
889 + num_grids = [
890 + grid_idx
891 + ] # e.g. [0, 9, 18, 19, 27, 28, 36, 37, 45, 46, 54, 55, 56]
892 + num_queries_vis_abstractors = [
893 + ] # e.g. [81, 81, 81, 9, 81, 9, 81, 9, 81, 9, 81, 9]
894 + len_total_frames = video_forward_outs.shape[0]
895 +
896 + if self.config.first_last_frames_slow:
897 + # slowfast (first_last_frames_slow)
898 + assert len_total_frames != 0
899 + if len_total_frames <= 2:
900 + num_queries_vis_abstractors.append(
901 + self.config.num_queries_vis_abstractor_video_slow)
902 + grid_idx += len_total_frames
903 + num_grids.append(grid_idx)
904 + else:
905 + num_queries_vis_abstractors.append(
906 + self.config.num_queries_vis_abstractor_video_slow)
907 + grid_idx += 1
908 + num_grids.append(grid_idx)
909 +
910 + num_queries_vis_abstractors.append(
911 + self.config.num_queries_vis_abstractor_video_fast)
912 + grid_idx += len_total_frames - 2
913 + num_grids.append(grid_idx)
914 +
915 + num_queries_vis_abstractors.append(
916 + self.config.num_queries_vis_abstractor_video_slow)
917 + grid_idx += 1
918 + num_grids.append(grid_idx)
919 + else:
920 + # slowfast
921 + for pixel_values_frames in pixel_values_videos:
922 + for pixel_values_frame in pixel_values_frames:
923 + if len(pixel_values_frame) > 0:
924 + num_queries_vis_abstractors.append(
925 + self.config.num_queries_vis_abstractor_video_slow)
926 + grid_idx += 1
927 + num_grids.append(grid_idx)
928 + num_queries_vis_abstractors.append(
929 + self.config.num_queries_vis_abstractor_video_fast)
930 + grid_idx = grid_idx + len(pixel_values_frame) - 1
931 + num_grids.append(grid_idx)
932 +
933 + video_forward_outs = self.mm_projector(video_forward_outs,
934 + num_queries_vis_abstractors,
935 + num_grids)
936 +
937 + video_features = [] # what we want to return
938 + target_features = []
939 + target_group_size = 0
940 + group_counter = 0
941 + video_groups = [
942 + len(frame) for frames in pixel_values_videos for frame in frames
943 + ] # for concat video features after projector
944 +
945 + for forward_out in video_forward_outs:
946 + target_group_size += len(forward_out)
947 + target_features.append(forward_out.flatten(0, 1))
948 +
949 + video_group_size = video_groups[group_counter]
950 + if video_group_size == target_group_size:
951 + video_features.append(torch.cat(target_features, dim=0))
952 + target_features = []
953 + group_counter += 1
954 + target_group_size = 0
955 +
956 + elif video_group_size < target_group_size:
957 + raise RuntimeError(f"video_group_size < target_group_size!! \
958 + [{video_group_size} < {target_group_size}]")
959 +
960 + assert len(target_features
961 + ) == 0, f"target_features is not empty!! {target_features}"
962 + assert len(video_groups) == len(video_features)
963 +
964 + return video_features
965 +
966 + def _prepare_multimodal_kwargs(self, **kwargs: object):
967 + output = defaultdict(list)
968 + for k, v in kwargs.items():
969 + if len(v) < 1 or len(v[0]) < 1:
970 + continue # if empty batch of empty sample
971 +
972 + new_k, is_video = k, False
973 + if (not k.endswith("_images") and not k.endswith("_videos")):
974 + pass
975 + else:
976 + new_k, is_video = k.split("_")[:-1], k.split("_")[-1]
977 + new_k = "_".join(new_k)
978 + is_video = is_video == "videos"
979 +
980 + for _sample_idx, _v in enumerate(v): # batch -> sample
981 + if new_k not in ["pixel_values"]:
982 + if len(output[new_k]) < _sample_idx + 1:
983 + output[new_k].append(list())
984 + _v = _v.detach().cpu().numpy().tolist()
985 + output[new_k][_sample_idx] += _v
986 + elif isinstance(_v, torch.Tensor):
987 + if len(output[new_k]) < _sample_idx + 1:
988 + output[new_k].append(list())
989 + output["is_videos"].append(list())
990 + _v = list(torch.unbind(_v, dim=0))
991 + output[new_k][_sample_idx] += _v
992 + output["is_videos"][_sample_idx] += [
993 + is_video,
994 + ] * len(_v)
995 + return dict(output)
996 +
997 + def compute_logits(
998 + self,
999 + hidden_states: torch.Tensor,
1000 + sampling_metadata: SamplingMetadata,
1001 + ) -> Optional[torch.Tensor]:
1002 + return self.language_model.compute_logits(hidden_states,
1003 + sampling_metadata)
1004 +
1005 + def load_weights(
1006 + self,
1007 + weights: Iterable[tuple[str, torch.Tensor]],
1008 + ) -> set[str]:
1009 + loader = AutoWeightsLoader(self)
1010 + return loader.load_weights(weights)
1011 +
1012 + def _init_possible_resolutions(
1013 + self,
1014 + config,
1015 + vision_config,
1016 + ):
1017 + if not getattr(config, "possible_resolutions", []):
1018 + possible_resolutions = []
1019 + if config.anyres:
1020 + assert config.max_num_grids > 0
1021 + for i in range(1, config.max_num_grids + 1):
1022 + for j in range(1, config.max_num_grids + 1):
1023 + if i == 1 and j == 1 and not config.use_1x1_grid:
1024 + continue
1025 + if i * j <= config.max_num_grids:
1026 + possible_resolutions.append([i, j])
1027 +
1028 + possible_resolutions = [[
1029 + ys * vision_config.image_size,
1030 + xs * vision_config.image_size
1031 + ] for ys, xs in possible_resolutions]
1032 + return possible_resolutions
1033 + else:
1034 + return config.possible_resolutions
1035 +
1036 + def _init_mm_projector(
1037 + self,
1038 + config,
1039 + text_config,
1040 + vision_config,
1041 + ):
1042 + input_hidden_size = vision_config.hidden_size
1043 + if config.mm_projector_type == "linear":
1044 + mm_projector = nn.Linear(input_hidden_size,
1045 + text_config.hidden_size)
1046 + mm_projector.dtype = next(mm_projector.parameters()).dtype
1047 + elif config.mm_projector_type == "cabstractor":
1048 + mm_projector = HCXVisionCAbstractor(
1049 + num_queries=config.num_queries_vis_abstractor_image,
1050 + num_input_tokens=(vision_config.image_size //
1051 + vision_config.patch_size)**2,
1052 + encoder_hidden_size=input_hidden_size,
1053 + hidden_size=input_hidden_size,
1054 + output_hidden_size=text_config.hidden_size,
1055 + pos_emb=config.proj_pos_emb,
1056 + prenorm=config.proj_prenorm,
1057 + )
1058 + else:
1059 + mm_projector = HCXVisionMlp(
1060 + config.mm_projector_type,
1061 + input_hidden_size,
1062 + hidden_features=input_hidden_size,
1063 + out_features=self.text_config.hidden_size,
1064 + )
1065 + return mm_projector
1066 +
1067 +
1068 +def unpad_image(tensor: torch.Tensor,
1069 + original_size: tuple[int, int]) -> torch.Tensor:
1070 + original_width, original_height = original_size
1071 + current_height, current_width = tensor.shape[1:]
1072 +
1073 + original_aspect_ratio = original_width / original_height
1074 + current_aspect_ratio = current_width / current_height
1075 +
1076 + if original_aspect_ratio > current_aspect_ratio:
1077 + scale_factor = current_width / original_width
1078 + new_height = int(original_height * scale_factor)
1079 + padding = (current_height - new_height) // 2
1080 + unpadded_tensor = tensor[:, padding:current_height - padding, :]
1081 + else:
1082 + scale_factor = current_height / original_height
1083 + new_width = int(original_width * scale_factor)
1084 + padding = (current_width - new_width) // 2
1085 + unpadded_tensor = tensor[:, :, padding:current_width - padding]
1086 +
1087 + return unpadded_tensor
1088 +
1089 +
1090 +def select_best_resolution(original_size: tuple,
1091 + possible_resolutions: list) -> tuple:
1092 + original_height, original_width = original_size
1093 + best_fit = None
1094 + max_effective_resolution = 0
1095 + min_wasted_resolution = float("inf")
1096 +
1097 + for height, width in possible_resolutions:
1098 + scale = min(width / original_width, height / original_height)
1099 + downscaled_width, downscaled_height = int(original_width * scale), int(
1100 + original_height * scale)
1101 + effective_resolution = min(downscaled_width * downscaled_height,
1102 + original_width * original_height)
1103 + wasted_resolution = (width * height) - effective_resolution
1104 +
1105 + if effective_resolution > max_effective_resolution or (
1106 + effective_resolution == max_effective_resolution
1107 + and wasted_resolution < min_wasted_resolution):
1108 + max_effective_resolution = effective_resolution
1109 + min_wasted_resolution = wasted_resolution
1110 + best_fit = (height, width)
1111 +
1112 + return best_fit
1113 +
1114 +
1115 +def get_anyres_image_grid_shape(
1116 + image_size: tuple[int, int],
1117 + grid_pinpoints: Union[str, list[tuple[int, int]]],
1118 + patch_size: int,
1119 +) -> tuple[int, int]:
1120 + possible_resolutions = grid_pinpoints if isinstance(
1121 + grid_pinpoints, list) else ast.literal_eval(grid_pinpoints)
1122 +
1123 + original_width, original_height = image_size
1124 + height, width = select_best_resolution((original_height, original_width),
1125 + possible_resolutions)
1126 + return width // patch_size, height // patch_size
1127 +
1128 +
1129 +def reshape_and_unpad_image_features(
1130 + image_feature: torch.Tensor,
1131 + height: int,
1132 + width: int,
1133 + image_size: tuple[int, int],
1134 + possible_resolutions: list[tuple[int, int]],
1135 + grid_size: int,
1136 + unpad: bool,
1137 + image_newline: torch.Tensor,
1138 +) -> torch.Tensor:
1139 + base_image_feature = image_feature[0]
1140 + image_feature = image_feature[1:]
1141 +
1142 + assert (height * width == base_image_feature.shape[0]
1143 + ), f"height: {height}, width: {width}, \
1144 + base_image_feature.shape[0]: {base_image_feature.shape[0]}"
1145 +
1146 + num_patch_width, num_patch_height = get_anyres_image_grid_shape(
1147 + image_size, possible_resolutions, grid_size)
1148 + image_feature = image_feature.view(num_patch_height, num_patch_width,
1149 + height, width, -1)
1150 +
1151 + if unpad:
1152 + image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()
1153 + image_feature = image_feature.flatten(1, 2).flatten(2, 3)
1154 + image_feature = unpad_image(image_feature, image_size)
1155 + image_feature = torch.cat(
1156 + (
1157 + image_feature,
1158 + image_newline[:, None, None].expand(
1159 + *image_feature.shape[:-1], 1).to(image_feature.device),
1160 + ),
1161 + dim=-1,
1162 + )
1163 + image_feature = image_feature.flatten(1, 2).transpose(0, 1)
1164 + else:
1165 + image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()
1166 + image_feature = image_feature.flatten(0, 3)
1167 + image_feature = torch.cat((base_image_feature, image_feature), dim=0)
1168 +
1169 + return image_feature
1170 +
1171 +
1172 +def anyres_postprocessing(
1173 + image_forward_outs: list[torch.FloatTensor],
1174 + image_sizes: list[list[int]],
1175 + possible_resolutions: list[tuple[int, int]],
1176 + patch_size: int,
1177 + grid_size: int,
1178 + image_newline: torch.FloatTensor,
1179 + num_queries_vis_abstractor: int = -1,
1180 + unpad: bool = False,
1181 +) -> list[torch.FloatTensor]:
1182 + height = width = grid_size // patch_size
1183 +
1184 + if num_queries_vis_abstractor > 0:
1185 + assert (num_queries_vis_abstractor**0.5
1186 + ).is_integer(), "n_queries must be square number"
1187 + height = width = int(num_queries_vis_abstractor**0.5)
1188 +
1189 + # post-processing (unpad, add newline)
1190 + new_image_features = []
1191 + for image_idx, image_feature in enumerate(image_forward_outs):
1192 + if image_feature.shape[0] > 1:
1193 + image_feature = reshape_and_unpad_image_features(
1194 + image_feature=image_feature,
1195 + height=height,
1196 + width=width,
1197 + image_size=image_sizes[image_idx],
1198 + possible_resolutions=possible_resolutions,
1199 + grid_size=grid_size, # Pass grid info if needed by helper
1200 + unpad=unpad,
1201 + image_newline=image_newline,
1202 + )
1203 + else:
1204 + image_feature = image_feature[0]
1205 + image_feature = torch.cat(
1206 + (image_feature, image_newline[None].to(image_feature.device)),
1207 + dim=0)
1208 + new_image_features.append(image_feature)
1209 + image_features = new_image_features
1210 + return image_features
1211 +
1212 +
1213 +def resize_image(
1214 + image: Union[np.ndarray, PIL.Image.Image],
1215 + max_side: int = 378,
1216 +) -> np.ndarray:
1217 + image_arr = image
1218 + if isinstance(image, np.ndarray):
1219 + image = Image.fromarray(image)
1220 +
1221 + width, height = image.size
1222 + cur_max_size = max(width, height)
1223 + if cur_max_size <= max_side:
1224 + return image_arr
1225 +
1226 + scale = max_side / cur_max_size
1227 + width = int(width * scale)
1228 + height = int(height * scale)
1229 + image = image.resize((width, height), Image.LANCZOS)
1230 + image_arr = np.array(image)
1231 + return image_arr
vllm/model_executor/models/registry.py
+1
@@ -81,6 +81,7 @@ _TEXT_GENERATION_MODELS = {
81 "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
82 "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
83 "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
84 + "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
85 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
86 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
87 "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),