@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0836be3b03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 00:15 UTC

[Model] Add HyperCLOVAX-SEED-Think-32B vision-language model support (#31471)

Signed-off-by: effortprogrammer <yhjhoward7@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Hojin Yang committed Mar 10, 2026 at 11:59 UTC 0836be3b03c9f4a4da7d2eba0d3e8cbe5511f6bf
8 files changed +760 -10
docs/models/supported_models.md
+1
@@ -701,6 +701,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
701 | `GlmOcrForConditionalGeneration` | GLM-OCR | T + I<sup>E+</sup> | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ |
702 | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
703 | `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
704 +| `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
705 | `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ |
706 | `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + I<sup>E+</sup> | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ |
707 | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | |
tests/entrypoints/openai/test_realtime_validation.py
+1 -1
@@ -118,7 +118,7 @@ async def test_multi_chunk_streaming(
118 # JIT compilation
119 warmup_done = False
120 while not warmup_done:
121 - event = await receive_event(ws, timeout=360.0)
121 + event = await receive_event(ws, timeout=600.0)
122 if event["type"] in ("transcription.done", "error"):
123 warmup_done = True
124
tests/entrypoints/test_chat_utils.py
+32
@@ -1458,6 +1458,38 @@ def test_parse_chat_messages_context_text_format(
1458 assert mm_uuids is None
1459
1460
1461 +def test_parse_chat_messages_openai_format_image_url(
1462 + phi3v_model_config,
1463 + image_url,
1464 +):
1465 + content = [
1466 + {"type": "image_url", "image_url": {"url": image_url}},
1467 + {"type": "text", "text": "What's in the image?"},
1468 + ]
1469 + conversation, mm_data, mm_uuids = parse_chat_messages(
1470 + [
1471 + {
1472 + "role": "user",
1473 + "content": content,
1474 + }
1475 + ],
1476 + phi3v_model_config,
1477 + content_format="openai",
1478 + )
1479 +
1480 + assert conversation == [
1481 + {
1482 + "role": "user",
1483 + "content": [
1484 + {"type": "image"},
1485 + {"type": "text", "text": "What's in the image?"},
1486 + ],
1487 + }
1488 + ]
1489 + _assert_mm_data_is_image_input(mm_data, 1)
1490 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
1491 +
1492 +
1493 def test_parse_chat_messages_rejects_too_many_images_in_one_message(
1494 phi3v_model_config,
1495 image_url,
tests/models/registry.py
+8
@@ -313,6 +313,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
313 "HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
314 "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
315 ),
316 + "HyperCLOVAXForCausalLM": _HfExamplesInfo(
317 + "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
318 + trust_remote_code=True,
319 + ),
320 "InternLMForCausalLM": _HfExamplesInfo(
321 "internlm/internlm-chat-7b", trust_remote_code=True
322 ),
@@ -793,6 +797,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
797 "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
798 trust_remote_code=True,
799 ),
800 + "HCXVisionV2ForCausalLM": _HfExamplesInfo(
801 + "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
802 + trust_remote_code=True,
803 + ),
804 "HunYuanVLForConditionalGeneration": _HfExamplesInfo(
805 "tencent/HunyuanOCR",
806 hf_overrides={"num_experts": 0},
vllm/entrypoints/chat_utils.py
+5 -5
@@ -1428,6 +1428,8 @@ def _parse_chat_message_content_part(
1428 with multimodal placeholders.
1429 """
1430 if isinstance(part, str): # Handle plain text parts
1431 + if wrap_dicts:
1432 + return {"type": "text", "text": part}
1433 return part
1434 # Handle structured dictionary parts
1435 part_type, content = _parse_chat_message_content_mm_part(part)
@@ -1487,11 +1489,9 @@ def _parse_chat_message_content_part(
1489 else:
1490 raise NotImplementedError(f"Unknown part type: {part_type}")
1491
1490 - return (
1491 - {"type": modality}
1492 - if wrap_dicts
1493 - else (MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None)
1494 - )
1492 + if wrap_dicts:
1493 + return {"type": modality}
1494 + return MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None
1495
1496
1497 # No need to validate using Pydantic again
vllm/model_executor/models/hyperclovax_vision.py
+21 -4
@@ -325,7 +325,7 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
325 hf_inputs: BatchFeature,
326 hf_processor_mm_kwargs: Mapping[str, object],
327 ) -> Mapping[str, MultiModalFieldConfig]:
328 - return dict(
328 + fields = dict(
329 pixel_values_images=MultiModalFieldConfig.batched("image"),
330 image_sizes_images=MultiModalFieldConfig.batched("image"),
331 vision_query_lengths_images=MultiModalFieldConfig.batched("image"),
@@ -333,6 +333,8 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
333 vision_query_lengths_videos=MultiModalFieldConfig.batched("video"),
334 )
335
336 + return fields
337 +
338
339 def _build_hcxvision_hf_info(
340 ctx: InputProcessingContext,
@@ -590,12 +592,26 @@ class HCXVisionCAbstractor(nn.Module):
592 dummy_inputs=HCXVisionDummyInputsBuilder,
593 )
594 class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
595 + """
596 + HyperCLOVAX-SEED Vision-Language Model (V1 architecture).
597 +
598 + Supports:
599 + - HyperCLOVAX-SEED-Vision-Instruct-3B
600 +
601 + Uses CLIP/SigLIP as the vision encoder with C-Abstractor projector.
602 + """
603 +
604 packed_modules_mapping = {
605 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
606 "gate_up_proj": ["gate_proj", "up_proj"],
607 }
608
598 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
609 + def __init__(
610 + self,
611 + *,
612 + vllm_config: VllmConfig,
613 + prefix: str = "",
614 + ) -> None:
615 super().__init__()
616
617 # init configs
@@ -647,8 +663,9 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
663 self.vision_config = vision_config
664 self.text_config = text_config
665
650 - # use_sum_loss = bool(kwargs.pop("use_sum_loss", False))
651 - # self.reduction = self._init_reduction_type(use_sum_loss)
666 + self.make_empty_intermediate_tensors = (
667 + self.language_model.make_empty_intermediate_tensors
668 + )
669
670 @classmethod
671 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
vllm/model_executor/models/hyperclovax_vision_v2.py new
+690
@@ -0,0 +1,690 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +HyperCLOVAX V2 (32B Think Model) Implementation.
5 +
6 +This module contains the V2 architecture that uses Qwen2.5 Vision Transformer
7 +instead of CLIP/SigLIP used in V1.
8 +
9 +Supports:
10 +- HyperCLOVAX-SEED-Think-32B: Vision + Text
11 +"""
12 +
13 +from collections.abc import Iterable, Mapping, Sequence
14 +from functools import partial
15 +from typing import Annotated, Literal
16 +
17 +import torch
18 +import torch.nn as nn
19 +from transformers import BatchFeature
20 +
21 +from vllm.config import VllmConfig
22 +from vllm.config.multimodal import BaseDummyOptions
23 +from vllm.forward_context import set_forward_context
24 +from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.inputs import (
26 + MultiModalDataDict,
27 + MultiModalFieldConfig,
28 + MultiModalKwargsItems,
29 +)
30 +from vllm.multimodal.parse import ImageSize, MultiModalDataItems
31 +from vllm.multimodal.processing import (
32 + BaseDummyInputsBuilder,
33 + BaseMultiModalProcessor,
34 + BaseProcessingInfo,
35 + ProcessorInputs,
36 + PromptReplacement,
37 + PromptUpdate,
38 +)
39 +from vllm.sequence import IntermediateTensors
40 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
41 +
42 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
43 +from .qwen2_5_vl import Qwen2_5_VisionTransformer
44 +from .utils import (
45 + AutoWeightsLoader,
46 + WeightsMapper,
47 + init_vllm_registered_model,
48 + maybe_prefix,
49 +)
50 +
51 +# V2 (32B Think model) uses different tokens - retrieved from config at runtime
52 +# These placeholder strings must match the chat template format exactly.
53 +# The chat template produces: <|image_start|><|IMAGE_PAD|><|image_end|>
54 +# Similar to Qwen2-VL's <|vision_start|><|image_pad|><|vision_end|> format.
55 +V2_IMAGE_TOKEN: str = "<|image_start|><|IMAGE_PAD|><|image_end|>"
56 +V2_VIDEO_TOKEN: str = "<|video_start|><|VIDEO_PAD|><|video_end|>"
57 +
58 +
59 +class HCXVisionV2ImagePixelInputs(TensorSchema):
60 + """
61 + V2 Image inputs using Qwen2.5-VL style grid_thw format.
62 +
63 + Dimensions:
64 + - np: Number of patches
65 + - ni: Number of images
66 + - cps: Number of channels * patch_size * patch_size
67 + """
68 +
69 + type: Literal["pixel_values"] = "pixel_values"
70 + pixel_values: Annotated[torch.Tensor, TensorShape("np", "cps")]
71 + image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
72 +
73 +
74 +class HCXVisionV2ImageEmbeddingInputs(TensorSchema):
75 + """
76 + V2 Image embedding inputs.
77 +
78 + Dimensions:
79 + - nf: Number of image features
80 + - hs: Hidden size
81 + - ni: Number of images
82 + """
83 +
84 + type: Literal["image_embeds"] = "image_embeds"
85 + image_embeds: Annotated[torch.Tensor, TensorShape("nf", "hs")]
86 + image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
87 +
88 +
89 +HCXVisionV2ImageInputs = HCXVisionV2ImagePixelInputs | HCXVisionV2ImageEmbeddingInputs
90 +
91 +
92 +class HCXVisionV2VideoPixelInputs(TensorSchema):
93 + """
94 + V2 Video inputs using Qwen2.5-VL style grid_thw format.
95 +
96 + Dimensions:
97 + - np: Number of patches
98 + - nv: Number of videos
99 + - ctps: Number of channels * temporal_patch_size * patch_size * patch_size
100 + """
101 +
102 + type: Literal["pixel_values_videos"] = "pixel_values_videos"
103 + pixel_values_videos: Annotated[torch.Tensor, TensorShape("np", "ctps")]
104 + video_grid_thw: Annotated[torch.Tensor, TensorShape("nv", 3)]
105 +
106 +
107 +class HCXVisionV2VideoEmbeddingInputs(TensorSchema):
108 + """
109 + V2 Video embedding inputs.
110 +
111 + Dimensions:
112 + - nf: Number of video features
113 + - hs: Hidden size
114 + - nv: Number of videos
115 + """
116 +
117 + type: Literal["video_embeds"] = "video_embeds"
118 + video_embeds: Annotated[torch.Tensor, TensorShape("nf", "hs")]
119 + video_grid_thw: Annotated[torch.Tensor, TensorShape("nv", 3)]
120 +
121 +
122 +HCXVisionV2VideoInputs = HCXVisionV2VideoPixelInputs | HCXVisionV2VideoEmbeddingInputs
123 +
124 +
125 +class HCXVisionV2ProcessingInfo(BaseProcessingInfo):
126 + """Processing info for HyperCLOVAX V2 (32B Think model)."""
127 +
128 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
129 + return {"image": None, "video": None}
130 +
131 + def get_num_image_tokens(
132 + self,
133 + *,
134 + image_width: int,
135 + image_height: int,
136 + ) -> int:
137 + hf_config = self.get_hf_config()
138 + vision_config = hf_config.vision_config
139 + patch_size = vision_config.patch_size
140 + spatial_merge_size = vision_config.spatial_merge_size
141 +
142 + grid_h = image_height // patch_size
143 + grid_w = image_width // patch_size
144 +
145 + return (grid_h * grid_w) // (spatial_merge_size**2)
146 +
147 + def get_num_video_tokens(
148 + self,
149 + *,
150 + video_width: int,
151 + video_height: int,
152 + num_frames: int,
153 + ) -> int:
154 + hf_config = self.get_hf_config()
155 + vision_config = hf_config.vision_config
156 + patch_size = vision_config.patch_size
157 + temporal_patch_size = vision_config.temporal_patch_size
158 + spatial_merge_size = vision_config.spatial_merge_size
159 +
160 + grid_t = num_frames // temporal_patch_size
161 + grid_h = video_height // patch_size
162 + grid_w = video_width // patch_size
163 +
164 + return (grid_t * grid_h * grid_w) // (spatial_merge_size**2)
165 +
166 + def get_image_size_with_most_features(self) -> ImageSize:
167 + hf_config = self.get_hf_config()
168 + vision_config = hf_config.vision_config
169 + # Use a reasonable default size
170 + size = getattr(vision_config, "image_size", 448)
171 + return ImageSize(width=size, height=size)
172 +
173 + def get_max_image_tokens(self) -> int:
174 + target_width, target_height = self.get_image_size_with_most_features()
175 + return self.get_num_image_tokens(
176 + image_width=target_width,
177 + image_height=target_height,
178 + )
179 +
180 +
181 +class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2ProcessingInfo]):
182 + """Dummy inputs builder for HyperCLOVAX V2 memory profiling."""
183 +
184 + def get_dummy_text(
185 + self,
186 + mm_counts: Mapping[str, int],
187 + ) -> str:
188 + num_images = mm_counts.get("image", 0)
189 + num_videos = mm_counts.get("video", 0)
190 + return V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos
191 +
192 + def get_dummy_processor_inputs(
193 + self,
194 + seq_len: int,
195 + mm_counts: Mapping[str, int],
196 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
197 + mm_processor_kwargs: Mapping[str, object] | None = None,
198 + ) -> ProcessorInputs:
199 + """Build dummy processor inputs for memory profiling."""
200 + num_images = mm_counts.get("image", 0)
201 + num_videos = mm_counts.get("video", 0)
202 + prompt_text = V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos
203 +
204 + dummy_mm_data = self.get_dummy_mm_data(
205 + seq_len,
206 + mm_counts,
207 + mm_options,
208 + mm_processor_kwargs=mm_processor_kwargs,
209 + )
210 + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
211 +
212 + return ProcessorInputs(
213 + prompt=prompt_text,
214 + mm_data_items=dummy_mm_items,
215 + hf_processor_mm_kwargs=mm_processor_kwargs or {},
216 + tokenization_kwargs={"truncation": False},
217 + )
218 +
219 + def get_dummy_mm_data(
220 + self,
221 + seq_len: int,
222 + mm_counts: Mapping[str, int],
223 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
224 + mm_processor_kwargs: Mapping[str, object] | None = None,
225 + ) -> MultiModalDataDict:
226 + num_images = mm_counts.get("image", 0)
227 + num_videos = mm_counts.get("video", 0)
228 +
229 + target_width, target_height = self.info.get_image_size_with_most_features()
230 + target_num_frames = 16 # Default for video
231 +
232 + image_overrides = mm_options.get("image") if mm_options else None
233 + video_overrides = mm_options.get("video") if mm_options else None
234 +
235 + result: MultiModalDataDict = {
236 + "image": self._get_dummy_images(
237 + width=target_width,
238 + height=target_height,
239 + num_images=num_images,
240 + overrides=image_overrides, # type: ignore
241 + ),
242 + "video": self._get_dummy_videos(
243 + width=target_width,
244 + height=target_height,
245 + num_frames=target_num_frames,
246 + num_videos=num_videos,
247 + overrides=video_overrides, # type: ignore
248 + ),
249 + }
250 +
251 + return result
252 +
253 +
254 +class HCXVisionV2MultiModalProcessor(
255 + BaseMultiModalProcessor[HCXVisionV2ProcessingInfo]
256 +):
257 + """Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
258 +
259 + def _call_hf_processor(
260 + self,
261 + prompt: str,
262 + mm_data: Mapping[str, object],
263 + mm_kwargs: Mapping[str, object],
264 + tok_kwargs: Mapping[str, object],
265 + ) -> BatchFeature:
266 + images = mm_data.get("images")
267 + videos = mm_data.get("videos")
268 +
269 + # Get the HF processor
270 + hf_processor = self.info.get_hf_processor(**mm_kwargs)
271 +
272 + # Build data dict for HF processor (images/videos only)
273 + # NOTE: We pass the prompt as-is without token normalization.
274 + # Token expansion is handled by vLLM via _get_prompt_updates since
275 + # _hf_processor_applies_updates returns False.
276 + data: dict[str, object] = dict(
277 + text=prompt,
278 + images=images,
279 + videos=videos,
280 + )
281 +
282 + processed_outputs = self.info.ctx.call_hf_processor(
283 + hf_processor=hf_processor,
284 + data=data,
285 + kwargs=dict(**mm_kwargs, **tok_kwargs),
286 + )
287 +
288 + return processed_outputs
289 +
290 + def _hf_processor_applies_updates(
291 + self,
292 + prompt_text: str,
293 + mm_items: MultiModalDataItems,
294 + hf_processor_mm_kwargs: Mapping[str, object],
295 + tokenization_kwargs: Mapping[str, object],
296 + ) -> bool:
297 + # Match BaseMultiModalProcessor behavior:
298 + # - raw multimodal inputs: HF processor applies updates
299 + # - embedding inputs: vLLM applies updates
300 + return super()._hf_processor_applies_updates(
301 + prompt_text,
302 + mm_items,
303 + hf_processor_mm_kwargs,
304 + tokenization_kwargs,
305 + )
306 +
307 + def _get_prompt_updates(
308 + self,
309 + mm_items: MultiModalDataItems,
310 + hf_processor_mm_kwargs: Mapping[str, object],
311 + out_mm_kwargs: MultiModalKwargsItems,
312 + ) -> Sequence[PromptUpdate]:
313 + hf_config = self.info.get_hf_config()
314 +
315 + # Use token IDs directly from config.
316 + # This matches what get_dummy_processor_inputs uses, ensuring consistency.
317 + placeholder: dict[str, int] = {
318 + "image": hf_config.image_token_id, # 128060 for <|IMAGE_PAD|>
319 + "video": hf_config.video_token_id, # 128061 for <|VIDEO_PAD|>
320 + }
321 +
322 + merge_size = hf_config.vision_config.spatial_merge_size
323 +
324 + def get_replacement_v2(
325 + item_idx: int,
326 + modality: str,
327 + out_mm_kwargs: MultiModalKwargsItems,
328 + ):
329 + out_item = out_mm_kwargs[modality][item_idx]
330 +
331 + if modality == "image":
332 + grid_thw_elem = out_item.get("image_grid_thw")
333 + if grid_thw_elem is not None:
334 + # Access .data to get the actual tensor from MultiModalFieldElem
335 + grid_thw = grid_thw_elem.data
336 + # Qwen2.5-VL style calculation
337 + h, w = grid_thw[1].item(), grid_thw[2].item()
338 + num_tokens = (h * w) // (merge_size**2)
339 + else:
340 + # Fallback or error
341 + raise ValueError("Missing image_grid_thw for V2 model")
342 + elif modality == "video":
343 + grid_thw_elem = out_item.get("video_grid_thw")
344 + if grid_thw_elem is not None:
345 + # Access .data to get the actual tensor from MultiModalFieldElem
346 + grid_thw = grid_thw_elem.data
347 + t, h, w = grid_thw[0].item(), grid_thw[1].item(), grid_thw[2].item()
348 + num_tokens = (t * h * w) // (merge_size**2)
349 + else:
350 + raise ValueError("Missing video_grid_thw for V2 model")
351 + else:
352 + raise NotImplementedError(modality)
353 +
354 + return [placeholder[modality]] * num_tokens
355 +
356 + return [
357 + PromptReplacement(
358 + modality=modality,
359 + target=[
360 + placeholder[modality],
361 + ],
362 + replacement=partial(
363 + get_replacement_v2,
364 + modality=modality,
365 + out_mm_kwargs=out_mm_kwargs,
366 + ),
367 + )
368 + for modality in ("image", "video")
369 + ]
370 +
371 + def _get_mm_fields_config(
372 + self,
373 + hf_inputs: BatchFeature,
374 + hf_processor_mm_kwargs: Mapping[str, object],
375 + ) -> Mapping[str, MultiModalFieldConfig]:
376 + # HyperCLOVAX V2 uses Qwen2.5-VL style flattened pixel values where
377 + # pixel_values has shape (num_patches, channels*patch_size*patch_size)
378 + # while image_grid_thw has shape (num_images, 3).
379 + # We need to use flat_from_sizes to correctly handle this mismatch.
380 + hf_config = self.info.get_hf_config()
381 + spatial_merge_size = hf_config.vision_config.spatial_merge_size
382 +
383 + image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
384 + image_pixel_grid_sizes = image_grid_thw.prod(-1)
385 + image_embed_grid_sizes = (
386 + image_pixel_grid_sizes // spatial_merge_size // spatial_merge_size
387 + )
388 +
389 + video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
390 + video_pixel_grid_sizes = video_grid_thw.prod(-1)
391 + video_embed_grid_sizes = (
392 + video_pixel_grid_sizes // spatial_merge_size // spatial_merge_size
393 + )
394 +
395 + return dict(
396 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
397 + "image", image_pixel_grid_sizes
398 + ),
399 + image_embeds=MultiModalFieldConfig.flat_from_sizes(
400 + "image", image_embed_grid_sizes
401 + ),
402 + image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
403 + pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
404 + "video", video_pixel_grid_sizes
405 + ),
406 + video_embeds=MultiModalFieldConfig.flat_from_sizes(
407 + "video", video_embed_grid_sizes
408 + ),
409 + video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
410 + )
411 +
412 +
413 +@MULTIMODAL_REGISTRY.register_processor(
414 + HCXVisionV2MultiModalProcessor,
415 + info=HCXVisionV2ProcessingInfo,
416 + dummy_inputs=HCXVisionV2DummyInputsBuilder,
417 +)
418 +class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
419 + """
420 + HyperCLOVAX-SEED Vision-Language Model (V2 architecture).
421 +
422 + Supports:
423 + - HyperCLOVAX-SEED-Think-32B: Vision + Text
424 +
425 + Uses Qwen2.5 Vision Transformer as the vision encoder.
426 + """
427 +
428 + packed_modules_mapping = {
429 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
430 + "gate_up_proj": ["gate_proj", "up_proj"],
431 + "qkv": ["qkv"], # For vision tower
432 + }
433 +
434 + # Weight mapping for loading HuggingFace checkpoints
435 + # NOTE: Order matters! Ignores (None) should come before renames to prevent
436 + # partial matches
437 + hf_to_vllm_mapper = WeightsMapper(
438 + orig_to_new_prefix={
439 + "model.": "", # Remove model. prefix if present
440 + "vision_model.": "visual.", # HF uses vision_model, we use visual
441 + },
442 + orig_to_new_substr={
443 + # Ignore modules not implemented in vLLM
444 + "discrete_vision_model": None, # TextAlignedTokenizer
445 + },
446 + )
447 +
448 + def __init__(
449 + self,
450 + *,
451 + vllm_config: VllmConfig,
452 + prefix: str = "",
453 + ) -> None:
454 + super().__init__()
455 +
456 + config = vllm_config.model_config.hf_config
457 + quant_config = vllm_config.quant_config
458 +
459 + # Text config
460 + text_config = config.text_config
461 + if text_config.model_type in ["gpt2", "hyperclovax", "llama"]:
462 + text_config._attn_implementation = "sdpa"
463 + if text_config.model_type != "hyperclovax":
464 + text_config.logits_scaling = 1.0
465 +
466 + # Vision config
467 + vision_config = config.vision_config
468 +
469 + self.config = config
470 + self.vision_config = vision_config
471 + self.text_config = text_config
472 + self.vllm_config = vllm_config
473 + self.dtype = vllm_config.model_config.dtype
474 +
475 + # Initialize Qwen2.5 Vision Transformer
476 + self.visual = Qwen2_5_VisionTransformer(
477 + vision_config=vision_config,
478 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
479 + quant_config=quant_config,
480 + prefix=maybe_prefix(prefix, "visual"),
481 + )
482 +
483 + # Linear projector (vision_hidden_size -> text_hidden_size)
484 + # For V2 model: mm_projector_type is "linear"
485 + vision_hidden_size = vision_config.hidden_size
486 + text_hidden_size = text_config.hidden_size
487 +
488 + # Check if out_hidden_size is defined (Qwen2.5-VL style)
489 + # The merger in Qwen2.5 VisionTransformer handles projection to out_hidden_size
490 + if hasattr(vision_config, "out_hidden_size"):
491 + out_hidden = vision_config.out_hidden_size
492 + else:
493 + out_hidden = vision_hidden_size
494 +
495 + # Always create Linear projector since HF checkpoint has mm_projector weights
496 + self.mm_projector = nn.Linear(out_hidden, text_hidden_size)
497 +
498 + # Language model
499 + self.lm_head_vocab_size = getattr(
500 + text_config, "padded_vocab_size", text_config.vocab_size
501 + )
502 + self.language_model = init_vllm_registered_model(
503 + vllm_config=vllm_config,
504 + hf_config=text_config,
505 + prefix=maybe_prefix(prefix, "language_model"),
506 + )
507 +
508 + self.make_empty_intermediate_tensors = (
509 + self.language_model.make_empty_intermediate_tensors
510 + )
511 +
512 + @classmethod
513 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
514 + if modality.startswith("image"):
515 + return V2_IMAGE_TOKEN
516 + if modality.startswith("video"):
517 + return V2_VIDEO_TOKEN
518 +
519 + raise ValueError("Only image or video modality is supported")
520 +
521 + def _parse_and_validate_image_input(
522 + self,
523 + **kwargs: object,
524 + ) -> HCXVisionV2ImageInputs | None:
525 + pixel_values = kwargs.pop("pixel_values", None)
526 + image_embeds = kwargs.pop("image_embeds", None)
527 + image_grid_thw = kwargs.pop("image_grid_thw", None)
528 +
529 + if pixel_values is None and image_embeds is None:
530 + return None
531 +
532 + if pixel_values is not None:
533 + return HCXVisionV2ImagePixelInputs(
534 + pixel_values=pixel_values,
535 + image_grid_thw=image_grid_thw,
536 + )
537 +
538 + if image_embeds is not None:
539 + return HCXVisionV2ImageEmbeddingInputs(
540 + image_embeds=image_embeds,
541 + image_grid_thw=image_grid_thw,
542 + )
543 +
544 + return None
545 +
546 + def _parse_and_validate_video_input(
547 + self,
548 + **kwargs: object,
549 + ) -> HCXVisionV2VideoInputs | None:
550 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)
551 + video_embeds = kwargs.pop("video_embeds", None)
552 + video_grid_thw = kwargs.pop("video_grid_thw", None)
553 +
554 + if pixel_values_videos is None and video_embeds is None:
555 + return None
556 +
557 + if pixel_values_videos is not None:
558 + return HCXVisionV2VideoPixelInputs(
559 + pixel_values_videos=pixel_values_videos,
560 + video_grid_thw=video_grid_thw,
561 + )
562 +
563 + if video_embeds is not None:
564 + return HCXVisionV2VideoEmbeddingInputs(
565 + video_embeds=video_embeds,
566 + video_grid_thw=video_grid_thw,
567 + )
568 +
569 + return None
570 +
571 + def _process_image_input(
572 + self,
573 + image_input: HCXVisionV2ImageInputs,
574 + ) -> tuple[torch.Tensor, ...]:
575 + """Process images through Qwen2.5 ViT and projector."""
576 + grid_thw = image_input["image_grid_thw"]
577 + assert grid_thw.ndim == 2
578 + grid_thw_list = grid_thw.tolist()
579 +
580 + if image_input["type"] == "image_embeds":
581 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)
582 + else:
583 + pixel_values = image_input["pixel_values"]
584 + with set_forward_context(None, self.vllm_config):
585 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list)
586 +
587 + # Apply projector
588 + image_embeds = self.mm_projector(image_embeds)
589 +
590 + # Split concatenated embeddings for each image
591 + merge_size = self.visual.spatial_merge_size
592 + sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
593 + return image_embeds.split(sizes)
594 +
595 + def _process_video_input(
596 + self,
597 + video_input: HCXVisionV2VideoInputs,
598 + ) -> tuple[torch.Tensor, ...]:
599 + """Process videos through Qwen2.5 ViT and projector."""
600 + grid_thw = video_input["video_grid_thw"]
601 + assert grid_thw.ndim == 2
602 + grid_thw_list = grid_thw.tolist()
603 +
604 + if video_input["type"] == "video_embeds":
605 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)
606 + else:
607 + pixel_values_videos = video_input["pixel_values_videos"]
608 + with set_forward_context(None, self.vllm_config):
609 + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw_list)
610 +
611 + # Apply projector
612 + video_embeds = self.mm_projector(video_embeds)
613 +
614 + # Split concatenated embeddings for each video
615 + merge_size = self.visual.spatial_merge_size
616 + sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
617 + return video_embeds.split(sizes)
618 +
619 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
620 + modalities = {}
621 +
622 + for input_key in kwargs:
623 + if (
624 + input_key in ("pixel_values", "image_embeds")
625 + and "image" not in modalities
626 + ):
627 + modalities["image"] = self._parse_and_validate_image_input(**kwargs)
628 + if (
629 + input_key in ("pixel_values_videos", "video_embeds")
630 + and "video" not in modalities
631 + ):
632 + modalities["video"] = self._parse_and_validate_video_input(**kwargs)
633 +
634 + return modalities
635 +
636 + def get_language_model(self) -> torch.nn.Module:
637 + return self.language_model
638 +
639 + def embed_multimodal(
640 + self,
641 + **kwargs: object,
642 + ) -> MultiModalEmbeddings:
643 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
644 + if not modalities:
645 + return []
646 +
647 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
648 +
649 + for modality in modalities:
650 + if modality == "image":
651 + image_input = modalities["image"]
652 + if image_input is not None:
653 + image_embeddings = self._process_image_input(image_input)
654 + multimodal_embeddings += tuple(image_embeddings)
655 + if modality == "video":
656 + video_input = modalities["video"]
657 + if video_input is not None:
658 + video_embeddings = self._process_video_input(video_input)
659 + multimodal_embeddings += tuple(video_embeddings)
660 +
661 + return multimodal_embeddings
662 +
663 + def forward(
664 + self,
665 + input_ids: torch.Tensor,
666 + positions: torch.Tensor,
667 + intermediate_tensors: IntermediateTensors | None = None,
668 + inputs_embeds: torch.Tensor | None = None,
669 + **kwargs: object,
670 + ) -> torch.Tensor | IntermediateTensors:
671 + if intermediate_tensors is not None:
672 + inputs_embeds = None
673 +
674 + hidden_states = self.language_model.model(
675 + input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
676 + )
677 + return hidden_states
678 +
679 + def compute_logits(
680 + self,
681 + hidden_states: torch.Tensor,
682 + ) -> torch.Tensor | None:
683 + return self.language_model.compute_logits(hidden_states)
684 +
685 + def load_weights(
686 + self,
687 + weights: Iterable[tuple[str, torch.Tensor]],
688 + ) -> set[str]:
689 + loader = AutoWeightsLoader(self)
690 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/registry.py
+2
@@ -132,6 +132,8 @@ _TEXT_GENERATION_MODELS = {
132 "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
133 "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
134 "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
135 + "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
136 + "HyperCLOVAXForCausalLM": ("llama", "LlamaForCausalLM"),
137 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
138 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
139 "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),