@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 92effb07a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 14:45 UTC

[Model] Add HunyuanOCR support (#29327)

Signed-off-by: manayang <jackmanayang@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: sergeywang <sergeywang@tencent.com> Co-authored-by: manayang <jackmanayang@gmail.com> Co-authored-by: manayang <manayang@tencent.com> Co-authored-by: Roger Wang <hey@rogerw.io>

Isotr0py committed Nov 25, 2025 at 11:28 UTC 92effb07a48e56c531a95b696acd5f699baf16da
18 files changed +2415 -4
docs/models/supported_models.md
+1
@@ -680,6 +680,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
680 | `Glm4vMoeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | ✅︎ | ✅︎ |
681 | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
682 | `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ |
683 +| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + I<sup>E+</sup> | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ |
684 | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | |
685 | `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ |
686 | `InternVLChatModel` | InternVL 3.5, InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3_5-14B`, `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+26
@@ -538,6 +538,31 @@ def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
538 )
539
540
541 +# HunyuanOCR
542 +def run_hunyuan_vl(questions: list[str], modality: str) -> ModelRequestData:
543 + assert modality == "image"
544 +
545 + model_name = "tencent/HunyuanOCR"
546 +
547 + engine_args = EngineArgs(
548 + model=model_name,
549 + max_model_len=8192,
550 + limit_mm_per_prompt={modality: 1},
551 + )
552 +
553 + placeholder = "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>" # noqa: E501
554 + prompts = [
555 + f"<｜hy_begin▁of▁sentence｜>{placeholder}{question}<｜hy_User｜>"
556 + for question in questions
557 + ]
558 +
559 + return ModelRequestData(
560 + engine_args=engine_args,
561 + prompts=prompts,
562 + stop_token_ids=None,
563 + )
564 +
565 +
566 # naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B
567 def run_hyperclovax_seed_vision(
568 questions: list[str], modality: str
@@ -1820,6 +1845,7 @@ model_example_map = {
1845 "glm4_5v": run_glm4_5v,
1846 "glm4_5v_fp8": run_glm4_5v_fp8,
1847 "h2ovl_chat": run_h2ovl,
1848 + "hunyuan_vl": run_hunyuan_vl,
1849 "hyperclovax_seed_vision": run_hyperclovax_seed_vision,
1850 "idefics3": run_idefics3,
1851 "interns1": run_interns1,
tests/models/registry.py
+4
@@ -626,6 +626,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
626 "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
627 trust_remote_code=True,
628 ),
629 + "HunYuanVLForConditionalGeneration": _HfExamplesInfo(
630 + "tencent/HunyuanOCR",
631 + is_available_online=False,
632 + ),
633 "Idefics3ForConditionalGeneration": _HfExamplesInfo(
634 "HuggingFaceM4/Idefics3-8B-Llama3",
635 extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
vllm/config/model.py
+5
@@ -33,6 +33,7 @@ from vllm.transformers_utils.config import (
33 try_get_safetensors_metadata,
34 try_get_tokenizer_config,
35 uses_mrope,
36 + uses_xdrope_dim,
37 )
38 from vllm.transformers_utils.gguf_utils import (
39 maybe_patch_hf_config_from_gguf,
@@ -1615,6 +1616,10 @@ class ModelConfig:
1616 def uses_mrope(self) -> bool:
1617 return uses_mrope(self.hf_config)
1618
1619 + @property
1620 + def uses_xdrope_dim(self) -> int:
1621 + return uses_xdrope_dim(self.hf_config)
1622 +
1623 @property
1624 def is_multimodal_model(self) -> bool:
1625 return self.multimodal_config is not None
vllm/model_executor/layers/rotary_embedding/__init__.py
+13
@@ -17,6 +17,7 @@ from .llama4_vision_rope import Llama4VisionRotaryEmbedding
17 from .mrope import MRotaryEmbedding
18 from .ntk_scaling_rope import NTKScalingRotaryEmbedding
19 from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
20 +from .xdrope import XDRotaryEmbedding
21 from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
22
23 _ROPE_DICT: dict[tuple, RotaryEmbedding] = {}
@@ -184,6 +185,18 @@ def get_rope(
185 raise ValueError(
186 "Dynamic rope scaling must contain either 'alpha' or 'factor' field"
187 )
188 + elif scaling_type == "xdrope":
189 + scaling_alpha = rope_parameters["alpha"]
190 + rotary_emb = XDRotaryEmbedding(
191 + head_size,
192 + rotary_dim,
193 + max_position,
194 + base,
195 + is_neox_style,
196 + scaling_alpha,
197 + dtype,
198 + xdrope_section=rope_parameters["xdrope_section"],
199 + )
200 elif scaling_type == "yarn":
201 scaling_factor = rope_parameters["factor"]
202 original_max_position = rope_parameters["original_max_position_embeddings"]
vllm/model_executor/layers/rotary_embedding/xdrope.py new
+102
@@ -0,0 +1,102 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import numpy as np
5 +import torch
6 +
7 +from .common import apply_rotary_emb_dispatch
8 +from .dynamic_ntk_alpha_rope import DynamicNTKAlphaRotaryEmbedding
9 +
10 +
11 +class XDRotaryEmbedding(DynamicNTKAlphaRotaryEmbedding):
12 + """DynamicNTKAlphaRotaryEmbedding extended with MultiModal(XD) Sections.
13 +
14 + Based on the original DynamicNTKAlphaRotaryEmbedding implementation.
15 + """
16 +
17 + def __init__(
18 + self,
19 + head_size: int,
20 + rotary_dim: int,
21 + max_position_embeddings: int,
22 + base: float,
23 + is_neox_style: bool,
24 + scaling_alpha: float,
25 + dtype: torch.dtype,
26 + xdrope_section: list[int],
27 + ) -> None:
28 + self.xdrope_section = xdrope_section
29 + super().__init__(
30 + head_size,
31 + rotary_dim,
32 + max_position_embeddings,
33 + base,
34 + is_neox_style,
35 + scaling_alpha,
36 + dtype,
37 + )
38 +
39 + def forward(
40 + self,
41 + positions: torch.Tensor,
42 + query: torch.Tensor,
43 + key: torch.Tensor | None = None,
44 + offsets: torch.Tensor | None = None,
45 + ) -> tuple[torch.Tensor, torch.Tensor | None]:
46 + """PyTorch-native implementation equivalent to forward().
47 +
48 + Args:
49 + positions:
50 + [4, num_tokens] (P/W/H/T positions with multimodal inputs)
51 + query: [num_tokens, num_heads * head_size]
52 + key: [num_tokens, num_kv_heads * head_size]
53 + """
54 + assert positions.ndim == 2
55 + assert key is not None
56 +
57 + num_tokens = positions.shape[-1]
58 + cos_sin = self.cos_sin_cache[positions]
59 + cos, sin = cos_sin.chunk(2, dim=-1)
60 + cos = torch.cat(
61 + [m[i] for i, m in enumerate(cos.split(self.xdrope_section, dim=-1))], dim=-1
62 + )
63 + sin = torch.cat(
64 + [m[i] for i, m in enumerate(sin.split(self.xdrope_section, dim=-1))], dim=-1
65 + )
66 +
67 + query_shape = query.shape
68 + query = query.view(num_tokens, -1, self.head_size)
69 + query_rot = query[..., : self.rotary_dim]
70 + query_pass = query[..., self.rotary_dim :]
71 + query_rot = apply_rotary_emb_dispatch(query_rot, cos, sin, self.is_neox_style)
72 + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
73 +
74 + key_shape = key.shape
75 + key = key.view(num_tokens, -1, self.head_size)
76 + key_rot = key[..., : self.rotary_dim]
77 + key_pass = key[..., self.rotary_dim :]
78 + key_rot = apply_rotary_emb_dispatch(key_rot, cos, sin, self.is_neox_style)
79 + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
80 + return query, key
81 +
82 + @staticmethod
83 + def get_next_input_positions(
84 + context_len: int,
85 + seq_len: int,
86 + xd_sections: int = 4,
87 + ) -> list[list[int]]:
88 + return [list(range(context_len, seq_len)) for _ in range(xd_sections)]
89 +
90 + @staticmethod
91 + def get_next_input_positions_tensor(
92 + out: np.ndarray,
93 + out_offset: int,
94 + context_len: int,
95 + num_new_tokens: int,
96 + ):
97 + values = np.arange(
98 + context_len,
99 + context_len + num_new_tokens,
100 + dtype=out.dtype,
101 + )
102 + out[:, out_offset : out_offset + num_new_tokens] = values
vllm/model_executor/models/hunyuan_v1.py
+10 -1
@@ -576,7 +576,16 @@ class HunYuanDecoderLayer(nn.Module):
576 return hidden_states, residual, ori_kv_states
577
578
579 -@support_torch_compile
579 +@support_torch_compile(
580 + dynamic_arg_dims={
581 + "input_ids": 0,
582 + # positions is of shape (xd, seq_len) if xdrope is enabled for hunyuan-vl,
583 + # otherwise (seq_len, ).
584 + "positions": -1,
585 + "intermediate_tensors": 0,
586 + "inputs_embeds": 0,
587 + }
588 +)
589 class HunYuanModel(nn.Module):
590 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
591 super().__init__()
vllm/model_executor/models/hunyuan_vision.py new
+1028
@@ -0,0 +1,1028 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# coding=utf-8
5 +# Copyright 2025 The HunYuan team.
6 +# Copyright 2025 The vLLM team.
7 +# Copyright 2025 EleutherAI and the HuggingFace Inc. team. All rights reserved.
8 +#
9 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
10 +# and OPT implementations in this library. It has been modified from its
11 +# original forms to accommodate minor architectural differences compared
12 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
13 +#
14 +# Licensed under the Apache License, Version 2.0 (the "License");
15 +# you may not use this file except in compliance with the License.
16 +# You may obtain a copy of the License at
17 +#
18 +# http://www.apache.org/licenses/LICENSE-2.0
19 +#
20 +# Unless required by applicable law or agreed to in writing, software
21 +# distributed under the License is distributed on an "AS IS" BASIS,
22 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23 +# See the License for the specific language governing permissions and
24 +# limitations under the License.
25 +"""Inference-only HunYuan-VL model compatible with HuggingFace weights."""
26 +
27 +from collections.abc import Callable, Iterable, Mapping, Sequence
28 +from functools import partial
29 +from typing import Annotated, Any, Literal, TypeAlias
30 +
31 +import torch
32 +import torch.nn as nn
33 +import torch.nn.functional as F
34 +from transformers import BatchFeature
35 +
36 +from vllm.attention.backends.registry import AttentionBackendEnum
37 +from vllm.attention.layer import MultiHeadAttention
38 +from vllm.config import MultiModalConfig, VllmConfig
39 +from vllm.config.multimodal import BaseDummyOptions
40 +from vllm.distributed import parallel_state
41 +from vllm.distributed import utils as dist_utils
42 +from vllm.logger import init_logger
43 +from vllm.model_executor.layers.activation import get_act_fn
44 +from vllm.model_executor.layers.layernorm import RMSNorm
45 +from vllm.model_executor.layers.linear import (
46 + ColumnParallelLinear,
47 + QKVParallelLinear,
48 + RowParallelLinear,
49 +)
50 +from vllm.model_executor.layers.quantization import QuantizationConfig
51 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52 +from vllm.model_executor.models.module_mapping import MultiModelKeys
53 +from vllm.multimodal import MULTIMODAL_REGISTRY
54 +from vllm.multimodal.inputs import (
55 + ImageItem,
56 + ModalityData,
57 + MultiModalDataDict,
58 + MultiModalFeatureSpec,
59 + MultiModalFieldConfig,
60 + MultiModalKwargsItems,
61 +)
62 +from vllm.multimodal.parse import (
63 + DictEmbeddingItems,
64 + ImageSize,
65 + MultiModalDataItems,
66 + MultiModalDataParser,
67 +)
68 +from vllm.multimodal.processing import (
69 + BaseMultiModalProcessor,
70 + BaseProcessingInfo,
71 + PromptReplacement,
72 + PromptUpdate,
73 +)
74 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
75 +from vllm.sequence import IntermediateTensors
76 +from vllm.transformers_utils.configs.hunyuan_vl import (
77 + HunYuanVLConfig,
78 + HunYuanVLVisionConfig,
79 +)
80 +from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor
81 +from vllm.transformers_utils.processors.hunyuan_vl_image import smart_resize
82 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
83 +
84 +from .interfaces import (
85 + MultiModalEmbeddings,
86 + SupportsLoRA,
87 + SupportsMultiModal,
88 + SupportsPP,
89 + SupportsQuant,
90 + SupportsXDRoPE,
91 +)
92 +from .utils import (
93 + AutoWeightsLoader,
94 + WeightsMapper,
95 + init_vllm_registered_model,
96 + maybe_prefix,
97 +)
98 +
99 +logger = init_logger(__name__)
100 +
101 +# === Vision Inputs === #
102 +
103 +
104 +class HunYuanVLImagePixelInputs(TensorSchema):
105 + """
106 + Dimensions:
107 + - np: Number of patches
108 + - ni: Number of images
109 + - cps: Number of channels * patch_size * patch_size
110 + """
111 +
112 + type: Literal["pixel_values"]
113 +
114 + pixel_values: Annotated[
115 + torch.Tensor,
116 + TensorShape("np", "cps"),
117 + ]
118 +
119 + image_grid_thw: Annotated[
120 + torch.Tensor,
121 + TensorShape("ni", 3),
122 + ]
123 +
124 +
125 +class HunYuanVLImageEmbeddingInputs(TensorSchema):
126 + """
127 + Dimensions:
128 + - nf: Number of image features
129 + - hs: Hidden size
130 + - ni: Number of images
131 + """
132 +
133 + type: Literal["image_embeds"]
134 +
135 + image_embeds: Annotated[
136 + torch.Tensor,
137 + TensorShape("nf", "hs"),
138 + ]
139 +
140 + image_grid_thw: Annotated[
141 + torch.Tensor,
142 + TensorShape("ni", 3),
143 + ]
144 +
145 +
146 +HunYuanVLImageInputs: TypeAlias = (
147 + HunYuanVLImagePixelInputs | HunYuanVLImageEmbeddingInputs
148 +)
149 +
150 +# === Vision Encoder === #
151 +
152 +
153 +class HunYuanVisionMLP(nn.Module):
154 + def __init__(
155 + self,
156 + in_features: int,
157 + hidden_features: int,
158 + bias: bool = True,
159 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu,
160 + quant_config: QuantizationConfig | None = None,
161 + prefix: str = "",
162 + use_data_parallel: bool = False,
163 + ):
164 + super().__init__()
165 + self.dense_h_to_4h = ColumnParallelLinear(
166 + in_features,
167 + hidden_features,
168 + bias=bias,
169 + quant_config=quant_config,
170 + prefix=f"{prefix}.dense_h_to_4h",
171 + disable_tp=use_data_parallel,
172 + )
173 + self.dense_4h_to_h = RowParallelLinear(
174 + hidden_features,
175 + in_features,
176 + bias=bias,
177 + quant_config=quant_config,
178 + prefix=f"{prefix}.dense_4h_to_h",
179 + disable_tp=use_data_parallel,
180 + )
181 + self.act_fn = act_fn
182 +
183 + def forward(self, x: torch.Tensor):
184 + x_up, _ = self.dense_h_to_4h(x)
185 + x_down, _ = self.dense_4h_to_h(self.act_fn(x_up))
186 + return x_down
187 +
188 +
189 +class HunYuanVisionAttention(nn.Module):
190 + def __init__(
191 + self,
192 + embed_dim: int,
193 + num_heads: int,
194 + projection_size: int,
195 + quant_config: QuantizationConfig | None = None,
196 + multimodal_config: MultiModalConfig | None = None,
197 + prefix: str = "",
198 + use_data_parallel: bool = False,
199 + ) -> None:
200 + super().__init__()
201 + # Per attention head and per partition values.
202 + self.tp_size = (
203 + 1
204 + if use_data_parallel
205 + else parallel_state.get_tensor_model_parallel_world_size()
206 + )
207 + self.hidden_size_per_attention_head = dist_utils.divide(
208 + projection_size, num_heads
209 + )
210 + self.num_attention_heads_per_partition = dist_utils.divide(
211 + num_heads, self.tp_size
212 + )
213 +
214 + self.qkv = QKVParallelLinear(
215 + hidden_size=embed_dim,
216 + head_size=self.hidden_size_per_attention_head,
217 + total_num_heads=num_heads,
218 + total_num_kv_heads=num_heads,
219 + bias=True,
220 + quant_config=quant_config,
221 + prefix=f"{prefix}.qkv",
222 + disable_tp=use_data_parallel,
223 + )
224 +
225 + self.o_proj = RowParallelLinear(
226 + input_size=projection_size,
227 + output_size=embed_dim,
228 + quant_config=quant_config,
229 + prefix=f"{prefix}.o_proj",
230 + disable_tp=use_data_parallel,
231 + )
232 +
233 + self.scale = self.hidden_size_per_attention_head**-0.5
234 + self.attn = MultiHeadAttention(
235 + self.num_attention_heads_per_partition,
236 + self.hidden_size_per_attention_head,
237 + self.scale,
238 + prefix=f"{prefix}.attn",
239 + multimodal_config=multimodal_config,
240 + )
241 +
242 + def forward(
243 + self,
244 + x: torch.Tensor,
245 + ) -> torch.Tensor:
246 + qkv, _ = self.qkv(x)
247 + q, k, v = qkv.chunk(3, dim=-1)
248 + out = self.attn(q, k, v)
249 + output, _ = self.o_proj(out)
250 + return output
251 +
252 +
253 +class HunYuanVisionBlock(nn.Module):
254 + def __init__(
255 + self,
256 + dim: int,
257 + num_heads: int,
258 + mlp_hidden_dim: int,
259 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu,
260 + norm_layer: Callable[[int], nn.Module] | None = None,
261 + quant_config: QuantizationConfig | None = None,
262 + multimodal_config: MultiModalConfig | None = None,
263 + prefix: str = "",
264 + use_data_parallel: bool = False,
265 + ) -> None:
266 + super().__init__()
267 + if norm_layer is None:
268 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
269 + self.input_layernorm = norm_layer(dim)
270 + self.post_attention_layernorm = norm_layer(dim)
271 + self.self_attn = HunYuanVisionAttention(
272 + embed_dim=dim,
273 + num_heads=num_heads,
274 + projection_size=dim,
275 + quant_config=quant_config,
276 + multimodal_config=multimodal_config,
277 + prefix=f"{prefix}.self_attn",
278 + use_data_parallel=use_data_parallel,
279 + )
280 + self.mlp = HunYuanVisionMLP(
281 + dim,
282 + mlp_hidden_dim,
283 + act_fn=act_fn,
284 + bias=True,
285 + quant_config=quant_config,
286 + prefix=f"{prefix}.mlp",
287 + use_data_parallel=use_data_parallel,
288 + )
289 +
290 + def forward(
291 + self,
292 + x: torch.Tensor,
293 + ) -> torch.Tensor:
294 + x = x + self.self_attn(self.input_layernorm(x))
295 + x = x + self.mlp(self.post_attention_layernorm(x))
296 + return x
297 +
298 +
299 +class HunYuanVisionPatchEmbed(nn.Module):
300 + def __init__(self, config: HunYuanVLVisionConfig):
301 + super().__init__()
302 +
303 + self.config = config
304 + self.embed_dim = config.hidden_size
305 + self.patch_size = config.patch_size
306 + self.num_channels = config.num_channels
307 + self.spatial_merge_size = config.spatial_merge_size
308 + self.interpolate_mode = config.interpolate_mode
309 +
310 + self.patch_embedding = nn.Conv2d(
311 + in_channels=config.num_channels,
312 + out_channels=self.embed_dim,
313 + kernel_size=self.patch_size,
314 + stride=self.patch_size,
315 + bias=True,
316 + )
317 +
318 + self.max_num_patches = (config.max_image_size // self.patch_size) ** 2
319 +
320 + self.num_positions = self.max_num_patches + 1
321 + self.position_edge = int(self.num_positions**0.5)
322 + # first token is cls token, skip it
323 + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
324 +
325 + self.patch_pos_embed = None
326 +
327 + def forward(
328 + self, pixel_values: torch.Tensor, grid_thw: list[list[int]]
329 + ) -> torch.Tensor:
330 + num_patches = pixel_values.size(0)
331 + pixel_values = pixel_values.reshape(
332 + num_patches, self.num_channels, self.patch_size, self.patch_size
333 + )
334 +
335 + patch_embeds = self.patch_embedding(pixel_values)
336 + patch_embeds = patch_embeds.squeeze(-1).squeeze(-1).unsqueeze(0)
337 +
338 + if self.patch_pos_embed is None:
339 + patch_pos_shape = (
340 + 1,
341 + self.position_edge,
342 + self.position_edge,
343 + self.embed_dim,
344 + )
345 + self.patch_pos_embed = (
346 + self.position_embedding.weight[1:, :]
347 + .reshape(patch_pos_shape)
348 + .permute(0, 3, 1, 2)
349 + .float()
350 + )
351 +
352 + patch_pos_embed_list = []
353 + for grid in grid_thw:
354 + _, h0, w0 = grid
355 + # we add a small number to avoid floating point error in the interpolation
356 + # see discussion at https://github.com/facebookresearch/dino/issues/8
357 + h0, w0 = h0 + 0.1, w0 + 0.1
358 + patch_pos_embed = nn.functional.interpolate(
359 + self.patch_pos_embed,
360 + scale_factor=(h0 / self.position_edge, w0 / self.position_edge),
361 + mode=self.interpolate_mode,
362 + align_corners=False,
363 + )
364 +
365 + patch_pos_embed = (
366 + patch_pos_embed.reshape(self.embed_dim, -1)
367 + .transpose(0, 1)
368 + .unsqueeze(0)
369 + .to(patch_embeds.dtype)
370 + )
371 + patch_pos_embed_list.append(patch_pos_embed)
372 +
373 + patch_pos_embed = torch.cat(patch_pos_embed_list, dim=1)
374 + embeddings = patch_embeds + patch_pos_embed
375 +
376 + return embeddings
377 +
378 +
379 +class HunYuanVisionPatchMerger(nn.Module):
380 + def __init__(
381 + self,
382 + in_channels,
383 + out_channels,
384 + spatial_merge_size=2,
385 + rms_norm_eps=1e-5,
386 + prefix="",
387 + ):
388 + super().__init__()
389 + self.spatial_merge_size = spatial_merge_size
390 + embed_std = out_channels**-0.5
391 +
392 + self.proj = nn.Sequential(
393 + nn.Conv2d(
394 + in_channels,
395 + in_channels * 2,
396 + kernel_size=spatial_merge_size,
397 + stride=spatial_merge_size,
398 + ),
399 + nn.GELU(),
400 + nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=1),
401 + )
402 + self.mlp = nn.Linear(in_channels * 4, out_channels)
403 +
404 + self.image_newline = nn.Parameter(torch.randn(in_channels * 4) * embed_std)
405 + self.image_begin = nn.Parameter(torch.randn(out_channels) * embed_std)
406 + self.image_end = nn.Parameter(torch.randn(out_channels) * embed_std)
407 + self.image_sep = nn.Parameter(torch.randn(out_channels) * embed_std)
408 +
409 + self.before_rms = RMSNorm(in_channels, eps=rms_norm_eps)
410 + self.after_rms = RMSNorm(out_channels, eps=rms_norm_eps)
411 +
412 + def forward(self, x, size=(16, 16)):
413 + x = self.before_rms(x)
414 +
415 + h, w = size
416 + dtype = x.dtype
417 + x = x.permute(0, 2, 1).reshape(x.shape[0], -1, h, w)
418 +
419 + x = self.proj(x) # b,c,h,w
420 + b, c, h, w = x.shape
421 + x = torch.cat(
422 + [x, self.image_newline.reshape(1, c, 1, 1).expand(b, c, h, 1).to(dtype)],
423 + dim=-1,
424 + )
425 + x = x.reshape(b, c, -1).permute(0, 2, 1)
426 + x = self.mlp(x)
427 +
428 + begin = self.image_begin.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype)
429 + end = self.image_end.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype)
430 + x = torch.cat([begin, x, end], dim=1)
431 +
432 + return self.after_rms(x)
433 +
434 +
435 +class HunYuanVisionTransformer(nn.Module):
436 + def __init__(
437 + self,
438 + vision_config: HunYuanVLVisionConfig,
439 + quant_config: QuantizationConfig | None = None,
440 + prefix: str = "",
441 + use_data_parallel: bool = False,
442 + multimodal_config: MultiModalConfig | None = None,
443 + attn_backend_override: AttentionBackendEnum | None = None,
444 + ) -> None:
445 + super().__init__()
446 +
447 + num_hidden_layers = vision_config.num_hidden_layers
448 + self.hidden_size = vision_config.hidden_size
449 + self.num_heads = vision_config.num_attention_heads
450 + self.spatial_merge_size = vision_config.spatial_merge_size
451 +
452 + from vllm.compilation.backends import set_model_tag
453 +
454 + with set_model_tag("HunYuanVisionPatchEmbed"):
455 + self.embeddings = HunYuanVisionPatchEmbed(vision_config)
456 +
457 + norm_layer = partial(nn.LayerNorm, eps=vision_config.rms_norm_eps)
458 +
459 + with set_model_tag("HunYuanVisionBlock"):
460 + self.layers = nn.ModuleList(
461 + [
462 + HunYuanVisionBlock(
463 + dim=vision_config.hidden_size,
464 + num_heads=vision_config.num_attention_heads,
465 + mlp_hidden_dim=vision_config.intermediate_size,
466 + act_fn=get_act_fn(vision_config.hidden_act),
467 + norm_layer=norm_layer,
468 + quant_config=quant_config,
469 + multimodal_config=multimodal_config,
470 + prefix=f"{prefix}.layers.{layer_idx}",
471 + use_data_parallel=use_data_parallel,
472 + )
473 + for layer_idx in range(num_hidden_layers)
474 + ]
475 + )
476 +
477 + with set_model_tag("HunYuanVisionPatchMerger"):
478 + self.perceive = HunYuanVisionPatchMerger(
479 + vision_config.hidden_size,
480 + vision_config.out_hidden_size,
481 + spatial_merge_size=vision_config.spatial_merge_size,
482 + rms_norm_eps=vision_config.rms_norm_eps,
483 + prefix=f"{prefix}.perceive",
484 + )
485 +
486 + @property
487 + def dtype(self) -> torch.dtype:
488 + return self.embeddings.patch_embedding.weight.dtype
489 +
490 + @property
491 + def device(self) -> torch.device:
492 + return self.embeddings.patch_embedding.weight.device
493 +
494 + def forward(
495 + self,
496 + x: torch.Tensor,
497 + grid_thw: list[list[int]],
498 + ) -> torch.Tensor:
499 + # patchify
500 + seq_len = x.size(0)
501 + cu_seqlens: list = [0]
502 +
503 + hidden_states = x.to(device=self.device, dtype=self.dtype)
504 + hidden_states = self.embeddings(hidden_states, grid_thw)
505 +
506 + for t, h, w in grid_thw:
507 + t, h, w = int(t), int(h), int(w)
508 + cu_seqlens.append(h * w)
509 +
510 + cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32)
511 + cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)
512 +
513 + cu_seqlens = cu_seqlens.to(device=self.device, non_blocking=True)
514 +
515 + hidden_states = hidden_states.reshape(seq_len, -1)
516 + hidden_states = hidden_states.unsqueeze(0)
517 + for layer_num, layer in enumerate(self.layers):
518 + hidden_states = layer(hidden_states)
519 +
520 + # adapter
521 + split_lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
522 + split_items = hidden_states.split(split_lengths, dim=1)
523 + image_embeds_list = []
524 + for grid, split_item in zip(grid_thw, split_items):
525 + image_embeds_list.append(
526 + self.perceive(split_item.contiguous(), size=grid[1:]).squeeze(0)
527 + )
528 +
529 + return image_embeds_list
530 +
531 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
532 + stacked_params_mapping = [
533 + # (param_name, shard_name, shard_id)
534 + (".qkv", ".q_proj", "q"),
535 + (".qkv", ".k_proj", "k"),
536 + (".qkv", ".v_proj", "v"),
537 + ]
538 + params_dict = dict(self.named_parameters(remove_duplicate=False))
539 + loaded_params: set[str] = set()
540 +
541 + for name, loaded_weight in weights:
542 + for param_name, weight_name, shard_id in stacked_params_mapping:
543 + if weight_name not in name:
544 + continue
545 + name = name.replace(weight_name, param_name)
546 +
547 + param = params_dict[name]
548 + weight_loader = param.weight_loader
549 + weight_loader(param, loaded_weight, shard_id)
550 + break
551 + else:
552 + param = params_dict[name]
553 + weight_loader = getattr(param, "weight_loader", default_weight_loader)
554 + weight_loader(param, loaded_weight)
555 + loaded_params.add(name)
556 + return loaded_params
557 +
558 +
559 +def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
560 + image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
561 + image_grid_sizes = image_grid_thw.prod(-1)
562 + return dict(
563 + pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
564 + image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
565 + image_grid_thw=MultiModalFieldConfig.batched("image"),
566 + )
567 +
568 +
569 +class HunYuanVLMultiModalDataParser(MultiModalDataParser):
570 + def _parse_image_data(
571 + self,
572 + data: dict[str, torch.Tensor] | ModalityData[ImageItem],
573 + ):
574 + if isinstance(data, dict):
575 + return DictEmbeddingItems(
576 + data,
577 + modality="image",
578 + required_fields={"image_embeds", "image_grid_thw"},
579 + fields_factory=_hunyuan_vl_field_config,
580 + )
581 +
582 + return super()._parse_image_data(data)
583 +
584 +
585 +class HunYuanVLProcessingInfo(BaseProcessingInfo):
586 + def get_hf_config(self):
587 + return self.ctx.get_hf_config(HunYuanVLConfig)
588 +
589 + def get_hf_processor(
590 + self,
591 + **kwargs: object,
592 + ) -> HunYuanVLProcessor:
593 + return self.ctx.get_hf_processor(
594 + HunYuanVLProcessor,
595 + use_fast=kwargs.pop("use_fast", True),
596 + **kwargs,
597 + )
598 +
599 + def get_image_processor(
600 + self,
601 + **kwargs: object,
602 + ) -> HunYuanVLProcessor:
603 + return self.get_hf_processor(**kwargs).image_processor
604 +
605 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
606 + return {"image": None}
607 +
608 + def get_mm_max_tokens_per_item(
609 + self,
610 + seq_len: int,
611 + mm_counts: Mapping[str, int],
612 + ) -> Mapping[str, int]:
613 + max_image_tokens = self.get_max_image_tokens()
614 + # TODO: support video
615 + max_video_tokens = 0
616 + return {"image": max_image_tokens, "video": max_video_tokens}
617 +
618 + def _get_vision_info(
619 + self,
620 + *,
621 + image_width: int,
622 + image_height: int,
623 + num_frames: int = 1,
624 + do_resize: bool = True,
625 + image_processor: HunYuanVLProcessor | None,
626 + ) -> tuple[ImageSize, int]:
627 + if image_processor is None:
628 + image_processor = self.get_image_processor()
629 +
630 + hf_config = self.get_hf_config()
631 + vision_config = hf_config.vision_config
632 + patch_size = vision_config.patch_size
633 + spatial_merge_size = vision_config.spatial_merge_size
634 +
635 + if do_resize:
636 + resized_height, resized_width = smart_resize(
637 + height=image_height,
638 + width=image_width,
639 + factor=patch_size * spatial_merge_size,
640 + min_pixels=image_processor.min_pixels,
641 + max_pixels=image_processor.max_pixels,
642 + )
643 + preprocessed_size = ImageSize(width=resized_width, height=resized_height)
644 + else:
645 + preprocessed_size = ImageSize(width=image_width, height=image_height)
646 +
647 + grid_t = 1
648 + grid_h = preprocessed_size.height // patch_size
649 + grid_w = preprocessed_size.width // patch_size
650 +
651 + num_vision_tokens = (
652 + grid_t * grid_h // spatial_merge_size * (grid_w // spatial_merge_size + 1)
653 + + 2
654 + )
655 +
656 + return preprocessed_size, num_vision_tokens
657 +
658 + def get_num_image_tokens(
659 + self,
660 + *,
661 + image_width: int,
662 + image_height: int,
663 + image_processor: HunYuanVLProcessor | None,
664 + ) -> int:
665 + _, num_image_tokens = self._get_vision_info(
666 + image_width=image_width,
667 + image_height=image_height,
668 + image_processor=image_processor,
669 + )
670 + return num_image_tokens
671 +
672 + def get_image_size_with_most_features(self) -> ImageSize:
673 + max_image_size, _ = self._get_vision_info(
674 + image_width=512,
675 + image_height=8192,
676 + image_processor=None,
677 + )
678 + return max_image_size
679 +
680 + def get_max_image_tokens(self) -> int:
681 + target_width, target_height = self.get_image_size_with_most_features()
682 + return self.get_num_image_tokens(
683 + image_width=target_width,
684 + image_height=target_height,
685 + image_processor=None,
686 + )
687 +
688 +
689 +class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo]):
690 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
691 + num_images = mm_counts.get("image", 0)
692 +
693 + hf_processor = self.info.get_hf_processor()
694 + image_token: str = hf_processor.image_token
695 +
696 + return image_token * num_images
697 +
698 + def get_dummy_mm_data(
699 + self,
700 + seq_len: int,
701 + mm_counts: Mapping[str, int],
702 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
703 + ) -> MultiModalDataDict:
704 + num_images = mm_counts.get("image", 1)
705 +
706 + target_width, target_height = self.info.get_image_size_with_most_features()
707 +
708 + return {
709 + "image": self._get_dummy_images(
710 + width=target_width, height=target_height, num_images=num_images
711 + ),
712 + }
713 +
714 +
715 +class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
716 + def _get_data_parser(self) -> MultiModalDataParser:
717 + return HunYuanVLMultiModalDataParser()
718 +
719 + def _call_hf_processor(
720 + self,
721 + prompt: str,
722 + mm_data: Mapping[str, object],
723 + mm_kwargs: Mapping[str, object],
724 + tok_kwargs: Mapping[str, object],
725 + ) -> BatchFeature:
726 + return self.info.ctx.call_hf_processor(
727 + self.info.get_hf_processor(**mm_kwargs),
728 + dict(text=prompt, **mm_data),
729 + dict(**mm_kwargs, **tok_kwargs),
730 + )
731 +
732 + def _get_prompt_updates(
733 + self,
734 + mm_items: MultiModalDataItems,
735 + hf_processor_mm_kwargs: Mapping[str, Any],
736 + out_mm_kwargs: MultiModalKwargsItems,
737 + ) -> Sequence[PromptUpdate]:
738 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
739 + image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
740 +
741 + placeholder = {
742 + "image": hf_processor.image_token_id,
743 + }
744 +
745 + merge_size = image_processor.merge_size
746 +
747 + def get_replacement_hunyuan_vl(item_idx: int, modality: str):
748 + out_item = out_mm_kwargs[modality][item_idx]
749 + grid_thw = out_item[f"{modality}_grid_thw"].data
750 + assert isinstance(grid_thw, torch.Tensor)
751 +
752 + _, grid_h, grid_w = grid_thw
753 + num_tokens = (int(grid_h) // merge_size) * (
754 + int(grid_w) // merge_size + 1
755 + ) + 2
756 + return [placeholder[modality]] * num_tokens
757 +
758 + return [
759 + PromptReplacement(
760 + modality=modality,
761 + target=[placeholder[modality]],
762 + replacement=partial(get_replacement_hunyuan_vl, modality=modality),
763 + )
764 + for modality in ("image",)
765 + ]
766 +
767 + def _get_mm_fields_config(
768 + self,
769 + hf_inputs: BatchFeature,
770 + hf_processor_mm_kwargs: Mapping[str, object],
771 + ) -> Mapping[str, MultiModalFieldConfig]:
772 + return _hunyuan_vl_field_config(hf_inputs)
773 +
774 +
775 +@MULTIMODAL_REGISTRY.register_processor(
776 + HunYuanVLMultiModalProcessor,
777 + info=HunYuanVLProcessingInfo,
778 + dummy_inputs=HunYuanVLDummyInputsBuilder,
779 +)
780 +class HunYuanVLForConditionalGeneration(
781 + nn.Module,
782 + SupportsMultiModal,
783 + SupportsLoRA,
784 + SupportsPP,
785 + SupportsQuant,
786 + SupportsXDRoPE,
787 +):
788 + multimodal_cpu_fields = {"image_grid_thw"}
789 +
790 + # To ensure correct weight loading and mapping.
791 + hf_to_vllm_mapper = WeightsMapper(
792 + orig_to_new_prefix={
793 + # mapping for new names in checkpoint saved after transformers v4.52
794 + "vit.vit.": "visual.",
795 + "vit.": "visual.",
796 + "model.": "language_model.model.",
797 + }
798 + )
799 +
800 + supports_encoder_tp_data = True
801 +
802 + def get_xdrope_input_positions(
803 + self,
804 + input_tokens: list[int],
805 + mm_features: list[MultiModalFeatureSpec],
806 + ) -> torch.Tensor:
807 + kwargs = MultiModalFeatureSpec.gather_kwargs(
808 + mm_features,
809 + {"image_grid_thw"},
810 + )
811 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]
812 +
813 + hf_config = self.config
814 + image_start_token_id = hf_config.image_start_token_id
815 + spatial_merge_size = hf_config.vision_config.spatial_merge_size
816 + xd_num = len(hf_config.rope_scaling["xdrope_section"])
817 +
818 + input_tokens_tensor = torch.tensor(input_tokens)
819 + image_start_indices = torch.argwhere(
820 + input_tokens_tensor == image_start_token_id
821 + ).squeeze(1)
822 +
823 + p_index = torch.arange(len(input_tokens_tensor))
824 + w_index = torch.arange(len(input_tokens_tensor))
825 + h_index = torch.arange(len(input_tokens_tensor))
826 + t_index = torch.arange(len(input_tokens_tensor))
827 + for image_index in range(len(image_start_indices)):
828 + # +1 : first image_token, +2: for xdrope positions
829 + pos = image_start_indices[image_index] + 2
830 + t, h, w = image_grid_thw[image_index]
831 + _, llm_grid_h, llm_grid_w = (
832 + t,
833 + h // spatial_merge_size,
834 + w // spatial_merge_size,
835 + )
836 +
837 + token_num = (llm_grid_w + 1) * llm_grid_h
838 + w_index[pos : pos + token_num].copy_(
839 + torch.arange(0, llm_grid_w + 1)
840 + .reshape(1, -1)
841 + .expand(llm_grid_h, -1)
842 + .reshape(-1)
843 + )
844 + h_index[pos : pos + token_num].copy_(
845 + torch.arange(0, llm_grid_h)
846 + .reshape(-1, 1)
847 + .expand(-1, llm_grid_w + 1)
848 + .reshape(-1)
849 + )
850 + h_index[pos : pos + token_num] = 0
851 +
852 + if xd_num == 4:
853 + llm_positions = torch.stack([p_index, w_index, h_index, t_index])
854 + elif xd_num == 3:
855 + llm_positions = torch.stack([w_index, h_index, t_index])
856 +
857 + return llm_positions
858 +
859 + @classmethod
860 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
861 + if modality.startswith("image"):
862 + return "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>" # noqa: E501
863 +
864 + raise ValueError("Only image modality is supported")
865 +
866 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
867 + super().__init__()
868 + config: HunYuanVLConfig = vllm_config.model_config.hf_config
869 + multimodal_config = vllm_config.model_config.multimodal_config
870 +
871 + self.config = config
872 + self.multimodal_config = multimodal_config
873 +
874 + if multimodal_config.get_limit_per_prompt("image"):
875 + attn_backend_override = (
876 + multimodal_config.mm_encoder_attn_backend
877 + if multimodal_config is not None
878 + else None
879 + )
880 + self.visual = HunYuanVisionTransformer(
881 + config.vision_config,
882 + quant_config=self.quant_config,
883 + prefix=maybe_prefix(prefix, "visual"),
884 + multimodal_config=multimodal_config,
885 + attn_backend_override=attn_backend_override,
886 + )
887 + else:
888 + self.visual = None
889 +
890 + self.language_model = init_vllm_registered_model(
891 + vllm_config=vllm_config,
892 + prefix=maybe_prefix(prefix, "language_model.model"),
893 + architectures=[
894 + "HunYuanDenseV1ForCausalLM",
895 + "HunYuanMoEV1ForCausalLM",
896 + ],
897 + )
898 +
899 + self.make_empty_intermediate_tensors = (
900 + self.language_model.make_empty_intermediate_tensors
901 + )
902 +
903 + def _parse_and_validate_image_input(
904 + self, **kwargs: object
905 + ) -> HunYuanVLImageInputs | None:
906 + pixel_values = kwargs.pop("pixel_values", None)
907 + image_embeds = kwargs.pop("image_embeds", None)
908 + image_grid_thw = kwargs.pop("image_grid_thw", None)
909 +
910 + if pixel_values is None and image_embeds is None:
911 + return None
912 +
913 + # TODO: refine
914 + if isinstance(pixel_values, list):
915 + pixel_values = torch.cat(pixel_values, dim=0)
916 + if len(pixel_values.shape) == 3:
917 + last_dim = pixel_values.shape[-1]
918 + pixel_values = pixel_values.reshape(-1, last_dim)
919 + image_grid_thw = image_grid_thw.reshape(-1, 3)
920 +
921 + if pixel_values is not None:
922 + return HunYuanVLImagePixelInputs(
923 + type="pixel_values",
924 + pixel_values=pixel_values,
925 + image_grid_thw=image_grid_thw,
926 + )
927 +
928 + if image_embeds is not None:
929 + return HunYuanVLImageEmbeddingInputs(
930 + type="image_embeds",
931 + image_embeds=image_embeds,
932 + image_grid_thw=image_grid_thw,
933 + )
934 +
935 + def _process_image_input(
936 + self, image_input: HunYuanVLImageInputs
937 + ) -> tuple[torch.Tensor, ...]:
938 + grid_thw = image_input["image_grid_thw"]
939 + assert grid_thw.ndim == 2
940 + grid_thw_list = grid_thw.tolist()
941 +
942 + if image_input["type"] == "image_embeds":
943 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)
944 + else:
945 + pixel_values = image_input["pixel_values"]
946 +
947 + # TODO: use_data_parallel (split image_embeds in visual)
948 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list)
949 +
950 + return image_embeds
951 +
952 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
953 + mm_input_by_modality = {}
954 +
955 + # Preserve the order of modalities if there are multiple of them
956 + # from the order of kwargs.
957 + for input_key in kwargs:
958 + if (
959 + input_key in ("pixel_values", "image_embeds")
960 + and "image" not in mm_input_by_modality
961 + ):
962 + mm_input_by_modality["image"] = self._parse_and_validate_image_input(
963 + **kwargs
964 + )
965 + return mm_input_by_modality
966 +
967 + def get_language_model(self) -> torch.nn.Module:
968 + return self.language_model
969 +
970 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
971 + mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
972 + if not mm_input_by_modality:
973 + return []
974 +
975 + # The result multimodal_embeddings is tuple of tensors, with each
976 + # tensor correspoending to a multimodal data item (image or video).
977 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
978 +
979 + # NOTE: It is important to iterate over the keys in this dictionary
980 + # to preserve the order of the modalities.
981 + for modality in mm_input_by_modality:
982 + multimodal_input = mm_input_by_modality[modality]
983 + if modality == "image":
984 + image_embeddings = self._process_image_input(multimodal_input)
985 + multimodal_embeddings += tuple(image_embeddings)
986 + return multimodal_embeddings
987 +
988 + def forward(
989 + self,
990 + input_ids: torch.Tensor,
991 + positions: torch.Tensor,
992 + intermediate_tensors: IntermediateTensors | None,
993 + inputs_embeds: torch.Tensor | None,
994 + **kwargs: object,
995 + ) -> torch.Tensor | IntermediateTensors:
996 + if intermediate_tensors is not None:
997 + inputs_embeds = None
998 +
999 + hidden_states = self.language_model(
1000 + input_ids=input_ids,
1001 + positions=positions,
1002 + intermediate_tensors=intermediate_tensors,
1003 + inputs_embeds=inputs_embeds,
1004 + )
1005 + return hidden_states
1006 +
1007 + def compute_logits(
1008 + self,
1009 + hidden_states: torch.Tensor,
1010 + ) -> torch.Tensor | None:
1011 + return self.language_model.compute_logits(hidden_states)
1012 +
1013 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1014 + loader = AutoWeightsLoader(
1015 + self,
1016 + skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
1017 + )
1018 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1019 +
1020 + def get_mm_mapping(self) -> MultiModelKeys:
1021 + """
1022 + Get the module prefix in multimodal models
1023 + """
1024 + return MultiModelKeys.from_string_field(
1025 + language_model="language_model.model",
1026 + connector="visual.perceive",
1027 + tower_model="visual",
1028 + )
vllm/model_executor/models/interfaces.py
+50 -1
@@ -1047,7 +1047,7 @@ class SupportsMRoPE(Protocol):
1047 supports_mrope: ClassVar[Literal[True]] = True
1048 """
1049 A flag that indicates this model supports M-RoPE.
1050 -
1050 +
1051 Note:
1052 There is no need to redefine this flag if this class is in the
1053 MRO of your model class.
@@ -1088,3 +1088,52 @@ def supports_mrope(
1088 model: type[object] | object,
1089 ) -> TypeIs[type[SupportsMRoPE]] | TypeIs[SupportsMRoPE]:
1090 return isinstance(model, SupportsMRoPE)
1091 +
1092 +
1093 +@runtime_checkable
1094 +class SupportsXDRoPE(Protocol):
1095 + """The interface required for all models that support XD-RoPE."""
1096 +
1097 + supports_xdrope: ClassVar[Literal[True]] = True
1098 + """
1099 + A flag that indicates this model supports XD-RoPE.
1100 +
1101 + Note:
1102 + There is no need to redefine this flag if this class is in the
1103 + XDRope of your model class.
1104 + """
1105 +
1106 + def get_xdrope_input_positions(
1107 + self,
1108 + input_tokens: list[int],
1109 + mm_features: list["MultiModalFeatureSpec"],
1110 + ) -> torch.Tensor:
1111 + """
1112 + Get XD-RoPE input positions and delta value for this specific model.
1113 +
1114 + This method should be implemented by each model that supports XD-RoPE
1115 + to provide model-specific logic for computing input positions.
1116 +
1117 + Args:
1118 + input_tokens: List of input token IDs
1119 + mm_features: Information about each multi-modal data item
1120 +
1121 + Returns:
1122 + llm_positions: Tensor of shape `[xdrope_dim, num_tokens]` with
1123 + 4D(P/W/H/T) or 3D(W/H/T) positions.
1124 + """
1125 + ...
1126 +
1127 +
1128 +@overload
1129 +def supports_xdrope(model: type[object]) -> TypeIs[type[SupportsXDRoPE]]: ...
1130 +
1131 +
1132 +@overload
1133 +def supports_xdrope(model: object) -> TypeIs[SupportsXDRoPE]: ...
1134 +
1135 +
1136 +def supports_xdrope(
1137 + model: type[object] | object,
1138 +) -> TypeIs[type[SupportsXDRoPE]] | TypeIs[SupportsXDRoPE]:
1139 + return isinstance(model, SupportsXDRoPE)
vllm/model_executor/models/registry.py
+4
@@ -287,6 +287,10 @@ _MULTIMODAL_MODELS = {
287 "GraniteSpeechForConditionalGeneration",
288 ),
289 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
290 + "HunYuanVLForConditionalGeneration": (
291 + "hunyuan_vision",
292 + "HunYuanVLForConditionalGeneration",
293 + ),
294 "InternVLChatModel": ("internvl", "InternVLChatModel"),
295 "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
296 "OpenCUAForConditionalGeneration": (
vllm/transformers_utils/config.py
+18
@@ -86,6 +86,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
86 deepseek_vl_v2="DeepseekVLV2Config",
87 deepseek_v32="DeepseekV3Config",
88 flex_olmo="FlexOlmoConfig",
89 + hunyuan_vl="HunYuanVLConfig",
90 kimi_linear="KimiLinearConfig",
91 kimi_vl="KimiVLConfig",
92 RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct)
@@ -549,6 +550,23 @@ def thinker_uses_mrope(config: PretrainedConfig) -> bool:
550 return uses_mrope(thinker_text_config)
551
552
553 +def uses_xdrope_dim(config: PretrainedConfig) -> int:
554 + """Detect if the model with this config uses XD-ROPE."""
555 + xdrope_section = getattr(config, "xdrope_section", None)
556 + if xdrope_section is not None and isinstance(xdrope_section, list):
557 + return len(xdrope_section)
558 + rope_scaling = getattr(config, "rope_scaling", None)
559 + if rope_scaling is None:
560 + return 0
561 +
562 + if isinstance(rope_scaling, dict) and "xdrope_section" in rope_scaling:
563 + xdrope_section = rope_scaling["xdrope_section"]
564 + if xdrope_section is not None and isinstance(xdrope_section, list):
565 + return len(xdrope_section)
566 +
567 + return 0
568 +
569 +
570 def is_encoder_decoder(config: PretrainedConfig) -> bool:
571 """Detect if the model with this config is used as an encoder/decoder."""
572
vllm/transformers_utils/configs/__init__.py
+8
@@ -23,6 +23,11 @@ from vllm.transformers_utils.configs.eagle import EAGLEConfig
23 # `FalconConfig` class from the official HuggingFace transformers library.
24 from vllm.transformers_utils.configs.falcon import RWConfig
25 from vllm.transformers_utils.configs.flex_olmo import FlexOlmoConfig
26 +from vllm.transformers_utils.configs.hunyuan_vl import (
27 + HunYuanVLConfig,
28 + HunYuanVLTextConfig,
29 + HunYuanVLVisionConfig,
30 +)
31 from vllm.transformers_utils.configs.jais import JAISConfig
32 from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
33 from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig
@@ -53,6 +58,9 @@ __all__ = [
58 "DotsOCRConfig",
59 "EAGLEConfig",
60 "FlexOlmoConfig",
61 + "HunYuanVLConfig",
62 + "HunYuanVLTextConfig",
63 + "HunYuanVLVisionConfig",
64 "RWConfig",
65 "JAISConfig",
66 "Lfm2MoeConfig",
vllm/transformers_utils/configs/hunyuan_vl.py new
+322
@@ -0,0 +1,322 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/configuration_hunyuan_vl.py
5 +
6 +from transformers import PretrainedConfig
7 +
8 +
9 +class HunYuanVLVisionConfig(PretrainedConfig):
10 + model_type = "hunyuan_vl"
11 + base_config_key = "vision_config"
12 +
13 + def __init__(
14 + self,
15 + hidden_act="gelu",
16 + hidden_size=1152,
17 + intermediate_size=4304,
18 + interpolate_mode="bilinear",
19 + rms_norm_eps=1e-05,
20 + learnable_mlp_pooling_size=0,
21 + num_attention_heads=16,
22 + num_key_value_heads=None,
23 + num_channels=3,
24 + num_hidden_layers=27,
25 + out_hidden_size=4096,
26 + patch_size=16,
27 + remove_prenorm=True,
28 + spatial_merge_size=2,
29 + temporal_patch_size=1,
30 + resize_resolution=2048,
31 + img_max_token_num=4096,
32 + max_image_size=2048,
33 + video_max_image_size=768,
34 + video_min_image_size=256,
35 + min_image_size=512,
36 + anyres_vit_max_image_size=2048,
37 + max_vit_seq_len=16384,
38 + text_hidden_size=3072,
39 + **kwargs,
40 + ):
41 + super().__init__(**kwargs)
42 +
43 + self.hidden_act = hidden_act
44 + self.hidden_size = hidden_size
45 + self.intermediate_size = intermediate_size
46 + self.interpolate_mode = interpolate_mode
47 + self.learnable_mlp_pooling_size = learnable_mlp_pooling_size
48 + self.num_attention_heads = num_attention_heads
49 + if not num_key_value_heads:
50 + self.num_key_value_heads = num_attention_heads
51 + else:
52 + self.num_key_value_heads = num_key_value_heads
53 + self.num_channels = num_channels
54 + self.num_hidden_layers = num_hidden_layers
55 + self.out_hidden_size = out_hidden_size
56 + self.patch_size = patch_size
57 + self.remove_prenorm = remove_prenorm
58 + self.spatial_merge_size = spatial_merge_size
59 + self.temporal_patch_size = temporal_patch_size
60 + self.rms_norm_eps = rms_norm_eps
61 +
62 + self.resize_resolution = resize_resolution
63 + self.img_max_token_num = img_max_token_num
64 + self.max_image_size = max_image_size
65 + self.min_image_size = min_image_size
66 + self.video_max_image_size = video_max_image_size
67 + self.video_min_image_size = video_min_image_size
68 + self.anyres_vit_max_image_size = anyres_vit_max_image_size
69 + self.max_vit_seq_len = max_vit_seq_len
70 + self.text_hidden_size = text_hidden_size
71 +
72 +
73 +class HunYuanVLTextConfig(PretrainedConfig):
74 + r"""
75 + This is the configuration class to store the configuration of a [`HunYuanVLTextConfig`]. It is used to instantiate an
76 + HunYuan model according to the specified arguments, defining the model architecture. Instantiating a configuration
77 + with the defaults will yield a similar configuration to that of the HunYuan-7B.
78 + Hunyuan-7B-Instruct [tencent/Hunyuan-7B-Instruct](https://huggingface.co/tencent/Hunyuan-7B-Instruct).
79 +
80 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
81 + documentation from [`PretrainedConfig`] for more information.
82 +
83 +
84 + Args:
85 + vocab_size (`int`, *optional*, defaults to 290943):
86 + Vocabulary size of the HunYuan model. Defines the number of different tokens that can be represented by the
87 + `inputs_ids` passed when calling [`HunYuanVLTextConfig`]
88 + hidden_size (`int`, *optional*, defaults to 4096):
89 + Dimension of the hidden representations.
90 + intermediate_size (`int`, *optional*, defaults to 11008):
91 + Dimension of the MLP representations or shared MLP representations.
92 + num_hidden_layers (`int`, *optional*, defaults to 32):
93 + Number of hidden layers in the Transformer decoder.
94 + num_attention_heads (`int`, *optional*, defaults to 32):
95 + Number of attention heads for each attention layer in the Transformer decoder.
96 + num_key_value_heads (`int`, *optional*):
97 + This is the number of key_value heads that should be used to implement Grouped Query Attention. If
98 + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
99 + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
100 + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
101 + by meanpooling all the original heads within that group. For more details checkout [this
102 + paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
103 + `num_attention_heads`.
104 + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
105 + The non-linear activation function (function or string) in the decoder.
106 + max_position_embeddings (`int`, *optional*, defaults to 2048):
107 + The maximum sequence length that this model might ever be used with.
108 + initializer_range (`float`, *optional*, defaults to 0.02):
109 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
110 + rms_norm_eps (`float`, *optional*, defaults to 1e-05):
111 + The epsilon used by the rms normalization layers.
112 + use_cache (`bool`, *optional*, defaults to `True`):
113 + Whether or not the model should return the last key/values attentions (not used by all models). Only
114 + relevant if `config.is_decoder=True`.
115 + pad_token_id (`int`, *optional*, defaults to 0):
116 + Padding token id.
117 + bos_token_id (`int`, *optional*, defaults to 1):
118 + Beginning of stream token id.
119 + eos_token_id (`int`, *optional*, defaults to 2):
120 + End of stream token id.
121 + eod_token_id (int, *optional*, defaults to 3):
122 + Token ID representing the end-of-document marker. Used to indicate the termination of a text sequence.
123 + Example: In multi-document processing, this token helps the model distinguish between separate documents.
124 + pretraining_tp (`int`, *optional*, defaults to 1):
125 + Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
126 + document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
127 + necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
128 + issue](https://github.com/pytorch/pytorch/issues/76232).
129 + tie_word_embeddings (`bool`, *optional*, defaults to `False`):
130 + Whether to tie weight embeddings
131 + rope_theta (`float`, *optional*, defaults to 10000.0):
132 + The base period of the RoPE embeddings.
133 + rope_scaling (`Dict`, *optional*):
134 + Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
135 + strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
136 + `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
137 + `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
138 + these scaling strategies behave:
139 + https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
140 + experimental feature, subject to breaking API changes in future versions.
141 + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
142 + Whether to use a bias in the query, key, value and output projection layers during self-attention.
143 + attention_dropout (`float`, *optional*, defaults to 0.0):
144 + The dropout ratio for the attention probabilities.
145 + head_dim (`int`, *optional*, defaults to 128):
146 + The attention head dimension.
147 + """ # noqa: E501
148 +
149 + model_type = "hunyuan_vl_text"
150 + keys_to_ignore_at_inference = ["past_key_values"]
151 +
152 + def __init__(
153 + self,
154 + vocab_size=290943,
155 + hidden_size=4096,
156 + intermediate_size: int = 11008,
157 + num_hidden_layers=32,
158 + num_attention_heads=32,
159 + num_key_value_heads=None,
160 + hidden_act="silu",
161 + max_position_embeddings=2048,
162 + initializer_range=0.02,
163 + rms_norm_eps=1e-5,
164 + use_cache=True,
165 + pad_token_id=0,
166 + bos_token_id=1,
167 + eos_token_id=2,
168 + eod_token_id=3,
169 + pretraining_tp=1,
170 + tie_word_embeddings=False,
171 + rope_theta=10000.0,
172 + rope_scaling=None,
173 + attention_bias=False,
174 + attention_dropout=0.0,
175 + head_dim=None,
176 + **kwargs,
177 + ):
178 + self.vocab_size = vocab_size
179 + self.max_position_embeddings = max_position_embeddings
180 + self.hidden_size = hidden_size
181 + self.intermediate_size = intermediate_size
182 + self.num_hidden_layers = num_hidden_layers
183 + self.num_attention_heads = num_attention_heads
184 + self.head_dim = head_dim
185 + # for backward compatibility
186 + if num_key_value_heads is None:
187 + num_key_value_heads = num_attention_heads
188 +
189 + self.num_key_value_heads = num_key_value_heads
190 + self.hidden_act = hidden_act
191 + self.initializer_range = initializer_range
192 + self.rms_norm_eps = rms_norm_eps
193 + self.pretraining_tp = pretraining_tp
194 + self.use_cache = use_cache
195 + self.rope_theta = rope_theta
196 + self.rope_scaling = rope_scaling
197 + # self._rope_scaling_validation() # TODO: Need validation?
198 + self.attention_bias = attention_bias
199 + self.attention_dropout = attention_dropout
200 +
201 + super().__init__(
202 + pad_token_id=pad_token_id,
203 + bos_token_id=bos_token_id,
204 + eos_token_id=eos_token_id,
205 + tie_word_embeddings=tie_word_embeddings,
206 + **kwargs,
207 + )
208 +
209 + def _rope_scaling_validation(self):
210 + """
211 + Validate the `rope_scaling` configuration.
212 + """
213 + if self.rope_scaling is None:
214 + return
215 +
216 + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
217 + raise ValueError(
218 + "`rope_scaling` must be a dictionary with with two fields, `type` and "
219 + f"`factor` or `type` and `alpha`, got {self.rope_scaling}"
220 + )
221 + rope_scaling_type = self.rope_scaling.get("type", None)
222 + rope_scaling_factor = self.rope_scaling.get("factor", None)
223 + rope_scaling_alpha = self.rope_scaling.get("alpha", None)
224 + if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
225 + raise ValueError(
226 + "`rope_scaling`'s type field must be one of ['linear', 'dynamic'], "
227 + f"got {rope_scaling_type}"
228 + )
229 + if rope_scaling_factor is None and rope_scaling_alpha is None:
230 + raise ValueError(
231 + "`rope_scaling`'s factor or alpha field must be have one, "
232 + "got both of none"
233 + )
234 + if rope_scaling_factor is not None and (
235 + not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0
236 + ):
237 + raise ValueError(
238 + "`rope_scaling`'s factor field must be a float > 1.0, "
239 + f"got {rope_scaling_factor}"
240 + )
241 + if rope_scaling_alpha is not None and (
242 + not isinstance(rope_scaling_alpha, float) or rope_scaling_alpha <= 1.0
243 + ):
244 + raise ValueError(
245 + "`rope_scaling`'s alpha field must be a float > 1.0, "
246 + f"got {rope_scaling_alpha}"
247 + )
248 +
249 +
250 +class HunYuanVLConfig(PretrainedConfig):
251 + model_type = "hunyuan_vl"
252 + sub_configs = {
253 + "vision_config": HunYuanVLVisionConfig,
254 + "text_config": HunYuanVLTextConfig,
255 + }
256 + keys_to_ignore_at_inference = ["past_key_values"]
257 +
258 + def __init__(
259 + self,
260 + text_config=None,
261 + vision_config=None,
262 + im_start_id=120118,
263 + im_end_id=120119,
264 + image_token_id=120120,
265 + im_newline_id=120121,
266 + video_start_id=120122,
267 + video_end_id=120123,
268 + **kwargs,
269 + ):
270 + # We need to init super() here so that it does not reset values
271 + # that are in text config to the BaseClass defaults. The Base
272 + # config has many text related defaults and not all defaults are
273 + # same as for `HunYuanVLTextConfig`.
274 + super().__init__(**kwargs)
275 +
276 + if isinstance(vision_config, dict):
277 + self.vision_config = self.sub_configs["vision_config"](**vision_config)
278 + elif vision_config is None:
279 + self.vision_config = self.sub_configs["vision_config"]()
280 +
281 + if isinstance(text_config, dict):
282 + self.text_config = self.sub_configs["text_config"](**text_config)
283 + elif text_config is None:
284 + # For BC use all kwargs to init `TextConfig`
285 + self.text_config = self.sub_configs["text_config"](**kwargs)
286 +
287 + self.image_token_id = image_token_id
288 + self.im_start_id = im_start_id
289 + self.im_end_id = im_end_id
290 + self.im_newline_id = im_newline_id
291 + self.video_start_id = video_start_id
292 + self.video_end_id = video_end_id
293 +
294 + self.vision_config.text_hidden_size = self.text_config.hidden_size
295 +
296 + # Attention implementation to use. It sets it recursively on sub-configs
297 + # so we call it again in the end.
298 + self._attn_implementation = kwargs.pop("attn_implementation", None)
299 +
300 + def __setattr__(self, key, value):
301 + if (
302 + (text_config := super().__getattribute__("__dict__").get("text_config"))
303 + is not None
304 + and key not in ["dtype", "_attn_implementation_internal"]
305 + and key in text_config.__dict__
306 + ):
307 + setattr(text_config, key, value)
308 + else:
309 + super().__setattr__(key, value)
310 +
311 + def __getattribute__(self, key):
312 + if "text_config" in super().__getattribute__("__dict__") and key not in [
313 + "_name_or_path",
314 + "model_type",
315 + "dtype",
316 + "_attn_implementation_internal",
317 + ]:
318 + text_config = super().__getattribute__("text_config")
319 + if key in text_config.__dict__:
320 + return getattr(text_config, key)
321 +
322 + return super().__getattribute__(key)
vllm/transformers_utils/processors/__init__.py
+9 -1
@@ -9,7 +9,15 @@ reasons:
9 """
10
11 from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor
12 +from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor
13 +from vllm.transformers_utils.processors.hunyuan_vl_image import HunYuanVLImageProcessor
14 from vllm.transformers_utils.processors.ovis import OvisProcessor
15 from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor
16
15 -__all__ = ["DeepseekVLV2Processor", "OvisProcessor", "Ovis2_5Processor"]
17 +__all__ = [
18 + "DeepseekVLV2Processor",
19 + "HunYuanVLProcessor",
20 + "HunYuanVLImageProcessor",
21 + "OvisProcessor",
22 + "Ovis2_5Processor",
23 +]
vllm/transformers_utils/processors/hunyuan_vl.py new
+233
@@ -0,0 +1,233 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/processing_hunyuan_vl.py
5 +
6 +import numpy as np
7 +import torch
8 +from transformers import AutoProcessor
9 +from transformers.feature_extraction_utils import BatchFeature
10 +from transformers.image_utils import ImageInput
11 +from transformers.processing_utils import ProcessorMixin
12 +from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
13 +from transformers.video_utils import VideoInput
14 +
15 +
16 +class HunYuanVLProcessor(ProcessorMixin):
17 + attributes = ["image_processor", "tokenizer"]
18 + valid_kwargs = ["chat_template"]
19 + image_processor_class = "AutoImageProcessor"
20 + tokenizer_class = "AutoTokenizer" # ("AutoTokenizer", None)
21 +
22 + def __init__(
23 + self,
24 + image_processor=None,
25 + tokenizer=None,
26 + video_processor=None,
27 + chat_template=None,
28 + **kwargs,
29 + ):
30 + # TODO Fix the init
31 + self.tokenizer = tokenizer
32 + self.image_token_id = 120120 # self.tokenizer.image_token_id
33 + self.image_token = self.tokenizer.convert_ids_to_tokens(self.image_token_id)
34 + self.im_start_token_id = 120118 # self.tokenizer.im_start_id
35 + self.im_start_token = self.tokenizer.convert_ids_to_tokens(
36 + self.im_start_token_id
37 + )
38 + self.im_end_token_id = 120119 # self.tokenizer.im_end_id
39 + self.im_end_token = self.tokenizer.convert_ids_to_tokens(self.im_end_token_id)
40 + self.placeholder_token = self.tokenizer.convert_ids_to_tokens(
41 + self.tokenizer.vocab_size - 1
42 + )
43 + self.pad_id = 120002 # self.tokenizer.pad_token_id
44 +
45 + super().__init__(
46 + image_processor, tokenizer, video_processor, chat_template=chat_template
47 + )
48 +
49 + def __call__(
50 + self,
51 + images: ImageInput = None,
52 + text: TextInput
53 + | PreTokenizedInput
54 + | list[TextInput]
55 + | list[PreTokenizedInput] = None,
56 + videos: VideoInput = None,
57 + **kwargs,
58 + ) -> BatchFeature:
59 + image_inputs = {}
60 + if images is not None:
61 + image_inputs = self.image_processor(images=images)
62 + image_grid_thw = image_inputs["image_grid_thw"]
63 +
64 + if not isinstance(text, list):
65 + text = [text]
66 +
67 + text = text.copy() # below lines change text in-place
68 +
69 + image_tokens_cumsum = [0]
70 + if images is not None:
71 + index = 0
72 + for i in range(len(text)):
73 + while self.image_token in text[i]:
74 + grid_h, grid_w = image_grid_thw[index][-2:]
75 + patch_h = grid_h // self.image_processor.merge_size
76 + patch_w = grid_w // self.image_processor.merge_size
77 + num_image_tokens = patch_h * (patch_w + 1) + 2
78 + image_tokens_cumsum.append(
79 + image_tokens_cumsum[-1] + num_image_tokens
80 + )
81 + # text[i] = text[i].replace(self.image_token, self.im_start_token + self.placeholder_token * num_image_tokens + self.im_end_token, 1) # noqa: E501
82 + text[i] = text[i].replace(
83 + self.image_token, self.placeholder_token * num_image_tokens, 1
84 + )
85 + index += 1
86 + text[i] = text[i].replace(self.placeholder_token, self.image_token)
87 + # text[i] = self.tokenizer.bos_token + text[i]
88 +
89 + text_inputs = self.tokenizer(text, add_special_tokens=False, **kwargs)
90 + self._check_special_mm_tokens(text, text_inputs, modalities=["image"])
91 +
92 + input_ids = text_inputs["input_ids"]
93 + position_ids = torch.arange(len(input_ids[0]))
94 + position_ids_w = torch.arange(len(input_ids[0]))
95 + position_ids_h = torch.arange(len(input_ids[0]))
96 + position_ids_t = torch.arange(len(input_ids[0]))
97 +
98 + if images is not None:
99 + image_token_pos_indices = torch.where(input_ids[0] == self.image_token_id)[
100 + 0
101 + ]
102 + for i in range(len(image_grid_thw)):
103 + grid_h, grid_w = image_grid_thw[i][-2:]
104 + patch_h = grid_h // self.image_processor.merge_size
105 + patch_w = grid_w // self.image_processor.merge_size
106 + start_pos = image_token_pos_indices[image_tokens_cumsum[i]].item() + 1
107 + replace_num = (patch_w + 1) * patch_h
108 + position_ids_w[start_pos : start_pos + replace_num] = torch.tensor(
109 + list(range(patch_w + 1)) * patch_h, dtype=torch.int64
110 + )
111 + patch_h_list = []
112 + for h in range(patch_h):
113 + patch_h_list += [h] * (patch_w + 1)
114 + position_ids_h[start_pos : start_pos + replace_num] = torch.tensor(
115 + patch_h_list, dtype=torch.int64
116 + )
117 + position_ids_t[start_pos : start_pos + replace_num] = 0
118 +
119 + position_ids = torch.stack(
120 + [position_ids, position_ids_w, position_ids_h, position_ids_t]
121 + ).unsqueeze(0)
122 + text_inputs["position_ids"] = position_ids
123 +
124 + attention_mask = input_ids.ne(self.pad_id)
125 + text_inputs["attention_mask"] = attention_mask
126 + text_inputs["imgs_pos"] = [self.get_imgs_pos(input_ids)]
127 + # image_inputs["imgs"] = [[image_inputs["pixel_values"]]]
128 +
129 + return_tensors = kwargs.pop("return_tensors", None)
130 + return BatchFeature(
131 + data={**text_inputs, **image_inputs},
132 + tensor_type=return_tensors,
133 + )
134 +
135 + def batch_decode(self, *args, **kwargs):
136 + return self.tokenizer.batch_decode(*args, **kwargs)
137 +
138 + def decode(self, *args, **kwargs):
139 + return self.tokenizer.decode(*args, **kwargs)
140 +
141 + def post_process_image_text_to_text(
142 + self,
143 + generated_outputs,
144 + skip_special_tokens=True,
145 + clean_up_tokenization_spaces=False,
146 + **kwargs,
147 + ):
148 + assert 0
149 +
150 + def apply_chat_template(self, *args, **kwargs):
151 + token_ids = self.tokenizer.apply_chat_template(*args, **kwargs)
152 + return token_ids
153 +
154 + def get_imgs_pos(self, doc_ids):
155 + doc_ids = np.array(doc_ids, dtype=np.int64)
156 + img_begin_index = np.where(doc_ids == self.im_start_token_id)[0]
157 + img_end_index = np.where(doc_ids == self.im_end_token_id)[0]
158 + imgs_pos = np.concatenate(
159 + (
160 + np.reshape(img_begin_index + 1, (-1, 1)),
161 + np.reshape(img_end_index, (-1, 1)),
162 + ),
163 + axis=-1,
164 + ).tolist()
165 + return imgs_pos
166 +
167 + @property
168 + def model_input_names(self):
169 + tokenizer_input_names = self.tokenizer.model_input_names
170 + image_processor_input_names = self.image_processor.model_input_names
171 + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
172 +
173 +
174 +def split_image_into_patch_blocks(
175 + pixel_values: torch.Tensor, # shape: [batch_size, 3, H, W]
176 + patch_size: int = 16, # e.g. 16
177 + adaptor_patch_div: int = 4, # e.g. 4 --> each patch_size is cut into 4x4 small regions, i.e. patch_size // 4 # noqa: E501
178 +) -> torch.Tensor:
179 + """
180 + Split the input image tensor (supporting batch) into large patches of size `patch_size`,
181 + and then further divide each large patch into smaller regions of size
182 + (patch_size // adaptor_patch_div) x (patch_size // adaptor_patch_div).
183 + Each small region is extracted as a tensor of shape [3, patch_size, patch_size].
184 + The final output contains all such small region tensors.
185 +
186 + Args:
187 + pixel_values: Input image tensor of shape [batch_size, 3, H, W].
188 + patch_size: Size of the large patch, e.g., 16.
189 + adaptor_patch_div: Each large patch is divided into
190 + (patch_size // adaptor_patch_div) x (patch_size // adaptor_patch_div)
191 + smaller regions.
192 +
193 + Returns:
194 + patches: A tensor of shape [N, 3, patch_size, patch_size],
195 + where N = batch_size * (H // patch_size) * (W // patch_size) * (patch_size // adaptor_patch_div)^2.
196 + Each element in the batch corresponds to one small image region.
197 + """ # noqa: E501
198 + batch_size, channels, height, width = pixel_values.shape
199 + assert channels == 3, "Pixel values must have 3 channels in dim=1"
200 + assert height % patch_size == 0 and width % patch_size == 0, (
201 + "H and W must be divisible by patch_size"
202 + )
203 +
204 + patch_height_num = height // patch_size
205 + patch_width_num = width // patch_size
206 +
207 + # Reshape to [B, 3, ph, ps, pw, ps]
208 + img = pixel_values.reshape(
209 + batch_size, 3, patch_height_num, patch_size, patch_width_num, patch_size
210 + )
211 +
212 + # Further split each psxps patch into (ps//aps)x(ps//aps) small regions
213 + img = img.reshape(
214 + batch_size,
215 + 3,
216 + patch_height_num,
217 + patch_size // adaptor_patch_div, # ps // aps
218 + adaptor_patch_div,
219 + patch_width_num,
220 + patch_size // adaptor_patch_div, # ps // aps
221 + adaptor_patch_div,
222 + )
223 +
224 + # Permute to group the small regions: [B, ph, pw, ps//aps, ps//aps, 3, aps, aps]
225 + img = img.permute(0, 2, 5, 3, 6, 1, 4, 7)
226 +
227 + # Reshape into [B * ph * pw * (ps//aps)^2, 3, patch_size, patch_size]
228 + patches = img.reshape(-1, 3, patch_size, patch_size)
229 +
230 + return patches
231 +
232 +
233 +AutoProcessor.register("HunYuanVLProcessor", HunYuanVLProcessor)
vllm/transformers_utils/processors/hunyuan_vl_image.py new
+477
@@ -0,0 +1,477 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/image_processing_hunyuan_vl.py
5 +"""Image processor class for HunYuanVL."""
6 +
7 +# isort conflicts with ruff for transformers imports
8 +# isort: skip_file
9 +import math
10 +
11 +import numpy as np
12 +import torchvision.transforms as transforms
13 +from transformers import AutoImageProcessor
14 +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
15 +from transformers.image_transforms import (
16 + convert_to_rgb,
17 +)
18 +from transformers.image_utils import (
19 + OPENAI_CLIP_MEAN,
20 + OPENAI_CLIP_STD,
21 + ChannelDimension,
22 + ImageInput,
23 + PILImageResampling,
24 + make_flat_list_of_images,
25 + make_list_of_images,
26 + valid_images,
27 + validate_preprocess_arguments,
28 +)
29 +from transformers.utils import TensorType, logging
30 +from transformers.video_utils import VideoInput, make_batched_videos
31 +
32 +logger = logging.get_logger(__name__)
33 +
34 +
35 +def smart_resize(
36 + height: int,
37 + width: int,
38 + factor: int = 16,
39 + min_pixels: int = 512 * 512,
40 + max_pixels: int = 2048 * 2048,
41 +):
42 + """Rescales the image so that the following conditions are met:
43 +
44 + 1. Both dimensions (height and width) are divisible by 'factor'.
45 +
46 + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
47 +
48 + 3. The aspect ratio of the image is maintained as closely as possible.
49 +
50 + """
51 + if max(height, width) / min(height, width) > 200:
52 + raise ValueError(
53 + "absolute aspect ratio must be smaller than 200, got "
54 + f"{max(height, width) / min(height, width)}"
55 + )
56 + h_bar = round(height / factor) * factor
57 + w_bar = round(width / factor) * factor
58 + if h_bar * w_bar > max_pixels:
59 + beta = math.sqrt((height * width) / max_pixels)
60 + h_bar = max(factor, math.floor(height / beta / factor) * factor)
61 + w_bar = max(factor, math.floor(width / beta / factor) * factor)
62 + elif h_bar * w_bar < min_pixels:
63 + beta = math.sqrt(min_pixels / (height * width))
64 + h_bar = math.ceil(height * beta / factor) * factor
65 + w_bar = math.ceil(width * beta / factor) * factor
66 + return h_bar, w_bar
67 +
68 +
69 +class HunYuanVLImageProcessor(BaseImageProcessor):
70 + model_input_names = [
71 + "pixel_values",
72 + "image_grid_thw",
73 + "pixel_values_videos",
74 + "video_grid_thw",
75 + ]
76 +
77 + def __init__(
78 + self,
79 + do_resize: bool = True,
80 + size: dict[str, int] | None = None,
81 + resample: PILImageResampling = PILImageResampling.BICUBIC,
82 + do_rescale: bool = True,
83 + rescale_factor: int | float = 1 / 255,
84 + do_normalize: bool = True,
85 + image_mean: float | list[float] | None = None,
86 + image_std: float | list[float] | None = None,
87 + do_convert_rgb: bool = True,
88 + min_pixels: int | None = None,
89 + max_pixels: int | None = None,
90 + patch_size: int = 16,
91 + temporal_patch_size: int = 2,
92 + merge_size: int = 2,
93 + **kwargs,
94 + ) -> None:
95 + super().__init__(**kwargs)
96 + if size is not None and (
97 + "shortest_edge" not in size or "longest_edge" not in size
98 + ):
99 + raise ValueError(
100 + "size must contain 'shortest_edge' and 'longest_edge' keys."
101 + )
102 + else:
103 + size = {"shortest_edge": 512 * 512, "longest_edge": 2048 * 2048}
104 + # backward compatibility: override size with min_pixels and max_pixels
105 + # if they are provided.
106 + if min_pixels is not None:
107 + size["shortest_edge"] = min_pixels
108 + if max_pixels is not None:
109 + size["longest_edge"] = max_pixels
110 + self.min_pixels = size["shortest_edge"]
111 + self.max_pixels = size["longest_edge"]
112 + self.size = size
113 +
114 + self.do_resize = do_resize
115 + self.resample = resample
116 + self.do_rescale = do_rescale
117 + self.rescale_factor = rescale_factor
118 + self.do_normalize = do_normalize
119 + self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
120 + self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
121 +
122 + self.patch_size = patch_size
123 + self.temporal_patch_size = temporal_patch_size
124 + self.merge_size = merge_size
125 + self.do_convert_rgb = do_convert_rgb
126 +
127 + # hard-code
128 +
129 + def _preprocess(
130 + self,
131 + images: ImageInput | VideoInput,
132 + do_resize: bool | None = None,
133 + size: dict[str, int] | None = None,
134 + resample: PILImageResampling = None,
135 + do_rescale: bool | None = None,
136 + rescale_factor: float | None = None,
137 + do_normalize: bool | None = None,
138 + image_mean: float | list[float] | None = None,
139 + image_std: float | list[float] | None = None,
140 + patch_size: int = 16,
141 + temporal_patch_size: int = 2,
142 + merge_size: int = 2,
143 + do_convert_rgb: bool | None = None,
144 + data_format: ChannelDimension | None = ChannelDimension.FIRST,
145 + input_data_format: str | ChannelDimension | None = None,
146 + ):
147 + """
148 + Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
149 +
150 + Args:
151 + images (`ImageInput`):
152 + Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
153 + do_resize (`bool`, *optional*, defaults to `self.do_resize`):
154 + Whether to resize the image.
155 + size (`dict[str, int]`, *optional*, defaults to `self.size`):
156 + Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present.
157 + resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
158 + Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
159 + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
160 + Whether to rescale the image.
161 + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
162 + Scale factor to use if rescaling the image.
163 + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
164 + Whether to normalize the image.
165 + image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):
166 + Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
167 + image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):
168 + Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
169 + patch_size (`int`, *optional*, defaults to `self.patch_size`):
170 + The spatial patch size of the vision encoder.
171 + temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`):
172 + The temporal patch size of the vision encoder.
173 + merge_size (`int`, *optional*, defaults to `self.merge_size`):
174 + The merge size of the vision encoder to llm encoder.
175 + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
176 + Whether to convert the image to RGB.
177 + data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
178 + The channel dimension format for the output image. Can be one of:
179 + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
180 + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
181 + - Unset: Use the channel dimension format of the input image.
182 + input_data_format (`ChannelDimension` or `str`, *optional*):
183 + The channel dimension format for the input image. Can be one of:
184 + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
185 + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
186 + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
187 + """ # noqa: E501
188 + images = make_list_of_images(images)
189 +
190 + if do_convert_rgb:
191 + images = [convert_to_rgb(image) for image in images]
192 +
193 + width, height = images[0].width, images[0].height
194 + resized_width, resized_height = width, height
195 + processed_images = []
196 + for image in images:
197 + if do_resize:
198 + resized_width, resized_height = smart_resize(
199 + width,
200 + height,
201 + factor=patch_size * merge_size,
202 + min_pixels=self.min_pixels,
203 + max_pixels=self.max_pixels,
204 + )
205 + image = image.resize((resized_width, resized_height))
206 +
207 + if do_normalize:
208 + image = transforms.Compose(
209 + [
210 + transforms.ToTensor(),
211 + transforms.Normalize(self.image_mean, self.image_std),
212 + ]
213 + )(image)
214 + processed_images.append(image)
215 +
216 + patches = np.array(processed_images)
217 + channel = patches.shape[1]
218 + grid_t = patches.shape[0] // temporal_patch_size
219 + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
220 + patches = patches.reshape(
221 + 1,
222 + channel,
223 + grid_h // merge_size,
224 + merge_size,
225 + patch_size,
226 + grid_w // merge_size,
227 + merge_size,
228 + patch_size,
229 + )
230 + patches = patches.transpose(0, 2, 3, 5, 6, 1, 4, 7)
231 + flatten_patches = patches.reshape(
232 + 1 * grid_h * grid_w, channel * patch_size * patch_size
233 + )
234 +
235 + return flatten_patches, (grid_t, grid_h, grid_w)
236 +
237 + def preprocess(
238 + self,
239 + images: ImageInput,
240 + videos: VideoInput = None,
241 + do_resize: bool | None = None,
242 + size: dict[str, int] | None = None,
243 + min_pixels: int | None = None,
244 + max_pixels: int | None = None,
245 + resample: PILImageResampling = None,
246 + do_rescale: bool | None = None,
247 + rescale_factor: float | None = None,
248 + do_normalize: bool | None = None,
249 + image_mean: float | list[float] | None = None,
250 + image_std: float | list[float] | None = None,
251 + patch_size: int | None = None,
252 + temporal_patch_size: int | None = None,
253 + merge_size: int | None = None,
254 + do_convert_rgb: bool | None = None,
255 + return_tensors: str | TensorType | None = None,
256 + data_format: ChannelDimension | None = ChannelDimension.FIRST,
257 + input_data_format: str | ChannelDimension | None = None,
258 + ):
259 + """
260 + Args:
261 + images (`ImageInput`):
262 + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
263 + passing in images with pixel values between 0 and 1, set `do_rescale=False`.
264 + videos (`VideoInput`):
265 + Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
266 + passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
267 + do_resize (`bool`, *optional*, defaults to `self.do_resize`):
268 + Whether to resize the image.
269 + size (`dict[str, int]`, *optional*, defaults to `self.size`):
270 + Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
271 + the longest edge resized to keep the input aspect ratio.
272 + resample (`int`, *optional*, defaults to `self.resample`):
273 + Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
274 + has an effect if `do_resize` is set to `True`.
275 + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
276 + Whether to rescale the image.
277 + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
278 + Rescale factor to rescale the image by if `do_rescale` is set to `True`.
279 + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
280 + Whether to normalize the image.
281 + image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):
282 + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
283 + image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):
284 + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
285 + `True`.
286 + min_pixels (`int`, *optional*, defaults to `self.min_pixels`):
287 + The min pixels of the image to resize the image.
288 + max_pixels (`int`, *optional*, defaults to `self.max_pixels`):
289 + The max pixels of the image to resize the image.
290 + patch_size (`int`, *optional*, defaults to `self.patch_size`):
291 + The spatial patch size of the vision encoder.
292 + temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`):
293 + The temporal patch size of the vision encoder.
294 + merge_size (`int`, *optional*, defaults to `self.merge_size`):
295 + The merge size of the vision encoder to llm encoder.
296 + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
297 + Whether to convert the image to RGB.
298 + return_tensors (`str` or `TensorType`, *optional*):
299 + The type of tensors to return. Can be one of:
300 + - Unset: Return a list of `np.ndarray`.
301 + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
302 + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
303 + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
304 + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
305 + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
306 + The channel dimension format for the output image. Can be one of:
307 + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
308 + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
309 + - Unset: Use the channel dimension format of the input image.
310 + input_data_format (`ChannelDimension` or `str`, *optional*):
311 + The channel dimension format for the input image. If unset, the channel dimension format is inferred
312 + from the input image. Can be one of:
313 + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
314 + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
315 + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
316 +
317 + """ # noqa: E501
318 + min_pixels = min_pixels if min_pixels is not None else self.min_pixels
319 + max_pixels = max_pixels if max_pixels is not None else self.max_pixels
320 +
321 + if size is not None:
322 + if "shortest_edge" not in size or "longest_edge" not in size:
323 + raise ValueError(
324 + "size must contain 'shortest_edge' and 'longest_edge' keys."
325 + )
326 + min_pixels = size["shortest_edge"]
327 + elif min_pixels is not None and max_pixels is not None:
328 + # backward compatibility: override size with min_pixels and max_pixels
329 + # if they are provided.
330 + size = {"shortest_edge": min_pixels, "longest_edge": max_pixels}
331 + else:
332 + size = {**self.size}
333 +
334 + do_resize = do_resize if do_resize is not None else self.do_resize
335 +
336 + resample = resample if resample is not None else self.resample
337 + do_rescale = do_rescale if do_rescale is not None else self.do_rescale
338 + rescale_factor = (
339 + rescale_factor if rescale_factor is not None else self.rescale_factor
340 + )
341 + do_normalize = do_normalize if do_normalize is not None else self.do_normalize
342 + image_mean = image_mean if image_mean is not None else self.image_mean
343 + image_std = image_std if image_std is not None else self.image_std
344 + patch_size = patch_size if patch_size is not None else self.patch_size
345 + temporal_patch_size = (
346 + temporal_patch_size
347 + if temporal_patch_size is not None
348 + else self.temporal_patch_size
349 + )
350 + merge_size = merge_size if merge_size is not None else self.merge_size
351 + do_convert_rgb = (
352 + do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
353 + )
354 +
355 + if images is not None:
356 + images = make_flat_list_of_images(images)
357 +
358 + if images is not None and not valid_images(images):
359 + raise ValueError(
360 + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
361 + "torch.Tensor, tf.Tensor or jax.ndarray."
362 + )
363 +
364 + validate_preprocess_arguments(
365 + rescale_factor=rescale_factor,
366 + do_normalize=do_normalize,
367 + image_mean=image_mean,
368 + image_std=image_std,
369 + do_resize=do_resize,
370 + size=size,
371 + resample=resample,
372 + )
373 +
374 + data = {}
375 + if images is not None:
376 + pixel_values, vision_grid_thws = [], []
377 + for image in images:
378 + patches, image_grid_thw = self._preprocess(
379 + image,
380 + do_resize=do_resize,
381 + size=size,
382 + resample=resample,
383 + do_rescale=do_rescale,
384 + rescale_factor=rescale_factor,
385 + do_normalize=do_normalize,
386 + image_mean=image_mean,
387 + image_std=image_std,
388 + patch_size=patch_size,
389 + temporal_patch_size=temporal_patch_size,
390 + merge_size=merge_size,
391 + data_format=data_format,
392 + do_convert_rgb=do_convert_rgb,
393 + input_data_format=input_data_format,
394 + )
395 + pixel_values.extend(patches)
396 + vision_grid_thws.append(image_grid_thw)
397 + pixel_values = np.array(pixel_values)
398 + vision_grid_thws = np.array(vision_grid_thws)
399 + data.update(
400 + {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}
401 + )
402 +
403 + # kept for BC only and should be removed after v5.0
404 + if videos is not None:
405 + logger.warning(
406 + "`HunYuanVLV1ImageProcessor` works only with image inputs "
407 + "and doesn't process videos anymore. "
408 + "This is a deprecated behavior and will be removed in v5.0. "
409 + "Your videos should be forwarded to `HunYuanVLV1VideoProcessor`. "
410 + )
411 + videos = make_batched_videos(videos)
412 + pixel_values_videos, vision_grid_thws_videos = [], []
413 + for images in videos:
414 + patches, video_grid_thw = self._preprocess(
415 + images,
416 + do_resize=do_resize,
417 + size=size,
418 + resample=resample,
419 + do_rescale=do_rescale,
420 + rescale_factor=rescale_factor,
421 + do_normalize=do_normalize,
422 + image_mean=image_mean,
423 + image_std=image_std,
424 + patch_size=patch_size,
425 + temporal_patch_size=temporal_patch_size,
426 + merge_size=merge_size,
427 + data_format=data_format,
428 + do_convert_rgb=do_convert_rgb,
429 + input_data_format=input_data_format,
430 + )
431 + pixel_values_videos.extend(patches)
432 + vision_grid_thws_videos.append(video_grid_thw)
433 + data.update(
434 + {
435 + "pixel_values_videos": np.array(pixel_values_videos),
436 + "video_grid_thw": np.array(vision_grid_thws_videos),
437 + }
438 + )
439 +
440 + return BatchFeature(data=data, tensor_type=return_tensors)
441 +
442 + def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
443 + """
444 + A utility that returns number of image patches for a given image size.
445 +
446 + Args:
447 + height (`int`):
448 + Height of the input image.
449 + width (`int`):
450 + Width of the input image.
451 + images_kwargs (`dict`, *optional*):
452 + Any kwargs to override defaults of the image processor.
453 + Returns:
454 + `int`: Number of image patches per image.
455 + """
456 + min_pixels = (
457 + images_kwargs["min_pixels"]
458 + if "min_pixels" in images_kwargs
459 + else self.size["shortest_edge"]
460 + )
461 + max_pixels = (
462 + images_kwargs["max_pixels"]
463 + if "max_pixels" in images_kwargs
464 + else self.size["longest_edge"]
465 + )
466 + patch_size = images_kwargs.get("patch_size", self.patch_size)
467 + merge_size = images_kwargs.get("merge_size", self.merge_size)
468 +
469 + factor = patch_size * merge_size
470 + resized_height, resized_width = smart_resize(
471 + height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels
472 + )
473 + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
474 + return grid_h * (grid_w + 1) + 2
475 +
476 +
477 +AutoImageProcessor.register("HunYuanVLImageProcessor", HunYuanVLImageProcessor)
vllm/v1/worker/gpu_input_batch.py
+2
@@ -43,6 +43,8 @@ class CachedRequestState:
43 mrope_positions: torch.Tensor | None = None
44 mrope_position_delta: int | None = None
45
46 + xdrope_positions: torch.Tensor | None = None
47 +
48 lora_request: LoRARequest | None = None
49 prompt_embeds: torch.Tensor | None = None
50
vllm/v1/worker/gpu_model_runner.py
+103 -1
@@ -50,16 +50,21 @@ from vllm.distributed.parallel_state import (
50 from vllm.forward_context import BatchDescriptor, set_forward_context
51 from vllm.logger import init_logger
52 from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
53 -from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
53 +from vllm.model_executor.layers.rotary_embedding import (
54 + MRotaryEmbedding,
55 + XDRotaryEmbedding,
56 +)
57 from vllm.model_executor.model_loader import TensorizerLoader, get_model_loader
58 from vllm.model_executor.models.interfaces import (
59 SupportsMRoPE,
60 SupportsMultiModal,
61 + SupportsXDRoPE,
62 is_mixture_of_experts,
63 supports_eagle3,
64 supports_mrope,
65 supports_multimodal_pruning,
66 supports_transcription,
67 + supports_xdrope,
68 )
69 from vllm.model_executor.models.interfaces_base import (
70 VllmModelForPooling,
@@ -324,6 +329,7 @@ class GPUModelRunner(
329 # Multi-modal data support
330 self.mm_registry = MULTIMODAL_REGISTRY
331 self.uses_mrope = model_config.uses_mrope
332 + self.uses_xdrope_dim = model_config.uses_xdrope_dim
333 self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
334 model_config
335 )
@@ -512,6 +518,13 @@ class GPUModelRunner(
518 (3, self.max_num_tokens + 1), dtype=torch.int64
519 )
520
521 + # Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
522 + if self.uses_xdrope_dim > 0:
523 + # Similar to mrope but use assigned dimension number for RoPE, 4 as default.
524 + self.xdrope_positions = self._make_buffer(
525 + (self.uses_xdrope_dim, self.max_num_tokens + 1), dtype=torch.int64
526 + )
527 +
528 # None in the first PP rank. The rest are set after load_model.
529 self.intermediate_tensors: IntermediateTensors | None = None
530
@@ -593,10 +606,14 @@ class GPUModelRunner(
606 if isinstance(num_tokens, int):
607 if self.uses_mrope:
608 return self.mrope_positions.gpu[:, :num_tokens]
609 + if self.uses_xdrope_dim > 0:
610 + return self.xdrope_positions.gpu[:, :num_tokens]
611 return self.positions.gpu[:num_tokens]
612 else:
613 if self.uses_mrope:
614 return self.mrope_positions.gpu[:, num_tokens]
615 + if self.uses_xdrope_dim > 0:
616 + return self.xdrope_positions.gpu[:, num_tokens]
617 return self.positions.gpu[num_tokens]
618
619 def _make_buffer(
@@ -772,6 +789,10 @@ class GPUModelRunner(
789 if self.uses_mrope:
790 self._init_mrope_positions(req_state)
791
792 + # Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
793 + if self.uses_xdrope_dim > 0:
794 + self._init_xdrope_positions(req_state)
795 +
796 reqs_to_add.append(req_state)
797
798 # Update the states of the running/resumed requests.
@@ -987,6 +1008,19 @@ class GPUModelRunner(
1008 )
1009 )
1010
1011 + def _init_xdrope_positions(self, req_state: CachedRequestState):
1012 + model = self.get_model()
1013 + xdrope_model = cast(SupportsXDRoPE, model)
1014 + assert req_state.prompt_token_ids is not None, (
1015 + "XD-RoPE requires prompt_token_ids to be available."
1016 + )
1017 + assert supports_xdrope(model), "XD-RoPE support is not implemented."
1018 +
1019 + req_state.xdrope_positions = xdrope_model.get_xdrope_input_positions(
1020 + req_state.prompt_token_ids,
1021 + req_state.mm_features,
1022 + )
1023 +
1024 def _extract_mm_kwargs(
1025 self,
1026 scheduler_output: "SchedulerOutput",
@@ -1231,6 +1265,11 @@ class GPUModelRunner(
1265 if self.uses_mrope:
1266 self._calc_mrope_positions(scheduler_output)
1267
1268 + # Calculate XD-RoPE positions.
1269 + # Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
1270 + if self.uses_xdrope_dim > 0:
1271 + self._calc_xdrope_positions(scheduler_output)
1272 +
1273 # Get token indices.
1274 # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
1275 # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
@@ -1364,6 +1403,12 @@ class GPUModelRunner(
1403 self.mrope_positions.cpu[:, :total_num_scheduled_tokens],
1404 non_blocking=True,
1405 )
1406 + elif self.uses_xdrope_dim > 0:
1407 + # Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
1408 + self.xdrope_positions.gpu[:, :total_num_scheduled_tokens].copy_(
1409 + self.xdrope_positions.cpu[:, :total_num_scheduled_tokens],
1410 + non_blocking=True,
1411 + )
1412 else:
1413 # Common case (1D positions)
1414 self.positions.copy_to_gpu(total_num_scheduled_tokens)
@@ -1793,6 +1838,53 @@ class GPUModelRunner(
1838
1839 mrope_pos_ptr += completion_part_len
1840
1841 + def _calc_xdrope_positions(self, scheduler_output: "SchedulerOutput"):
1842 + xdrope_pos_ptr = 0
1843 + for index, req_id in enumerate(self.input_batch.req_ids):
1844 + req = self.requests[req_id]
1845 + assert req.xdrope_positions is not None
1846 +
1847 + num_computed_tokens = self.input_batch.num_computed_tokens_cpu[index]
1848 + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
1849 + num_prompt_tokens = length_from_prompt_token_ids_or_embeds(
1850 + req.prompt_token_ids, req.prompt_embeds
1851 + )
1852 +
1853 + if num_computed_tokens + num_scheduled_tokens > num_prompt_tokens:
1854 + prompt_part_len = max(0, num_prompt_tokens - num_computed_tokens)
1855 + completion_part_len = max(0, num_scheduled_tokens - prompt_part_len)
1856 + else:
1857 + prompt_part_len = num_scheduled_tokens
1858 + completion_part_len = 0
1859 +
1860 + assert num_scheduled_tokens == prompt_part_len + completion_part_len
1861 +
1862 + if prompt_part_len > 0:
1863 + # prompt's xdrope_positions are pre-computed
1864 + dst_start = xdrope_pos_ptr
1865 + dst_end = xdrope_pos_ptr + prompt_part_len
1866 + src_start = num_computed_tokens
1867 + src_end = num_computed_tokens + prompt_part_len
1868 +
1869 + self.xdrope_positions.cpu[:, dst_start:dst_end] = req.xdrope_positions[
1870 + :, src_start:src_end
1871 + ]
1872 + xdrope_pos_ptr += prompt_part_len
1873 +
1874 + if completion_part_len > 0:
1875 + # compute completion's xdrope_positions on-the-fly
1876 + dst_start = xdrope_pos_ptr
1877 + dst_end = xdrope_pos_ptr + completion_part_len
1878 +
1879 + XDRotaryEmbedding.get_next_input_positions_tensor(
1880 + out=self.xdrope_positions.np,
1881 + out_offset=dst_start,
1882 + context_len=num_computed_tokens + prompt_part_len,
1883 + num_new_tokens=completion_part_len,
1884 + )
1885 +
1886 + xdrope_pos_ptr += completion_part_len
1887 +
1888 def _calc_spec_decode_metadata(
1889 self,
1890 num_draft_tokens: np.ndarray,
@@ -2037,6 +2129,7 @@ class GPUModelRunner(
2129
2130 req_start_idx = 0
2131 should_sync_mrope_positions = False
2132 + should_sync_xdrope_positions = False
2133
2134 for req_id in self.input_batch.req_ids:
2135 mm_embeds_req: list[torch.Tensor] = []
@@ -2110,6 +2203,10 @@ class GPUModelRunner(
2203 self._calc_mrope_positions(scheduler_output)
2204 self.mrope_positions.copy_to_gpu(total_num_scheduled_tokens)
2205
2206 + if should_sync_xdrope_positions:
2207 + self._calc_xdrope_positions(scheduler_output)
2208 + self.xdrope_positions.copy_to_gpu(total_num_scheduled_tokens)
2209 +
2210 return mm_embeds, is_mm_embed
2211
2212 def get_model(self) -> nn.Module:
@@ -2384,8 +2481,11 @@ class GPUModelRunner(
2481 input_ids = self.input_ids.gpu[:num_input_tokens]
2482 inputs_embeds = None
2483 model_kwargs = self._init_model_kwargs(num_input_tokens)
2484 +
2485 if self.uses_mrope:
2486 positions = self.mrope_positions.gpu[:, :num_input_tokens]
2487 + elif self.uses_xdrope_dim > 0:
2488 + positions = self.xdrope_positions.gpu[:, :num_input_tokens]
2489 else:
2490 positions = self.positions.gpu[:num_input_tokens]
2491
@@ -3824,6 +3924,8 @@ class GPUModelRunner(
3924
3925 if self.uses_mrope:
3926 positions = self.mrope_positions.gpu[:, :num_tokens_after_padding]
3927 + elif self.uses_xdrope_dim > 0:
3928 + positions = self.xdrope_positions.gpu[:, :num_tokens_after_padding]
3929 else:
3930 positions = self.positions.gpu[:num_tokens_after_padding]
3931