[Model][HunyuanVL] Use native transformers processor and adapt to transformers 5.13 (#47872)
Co-authored-by: manayang <manayang@tencent.com>
manayang committed
Jul 8, 2026 at 15:58 UTC
51e5372f3d8b2eb8cf96e20120af4de78b924fba
4 files changed
+21
-712
tests/models/registry.py
+1
-1
@@ -936,7 +936,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
936
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
937
"tencent/HunyuanOCR",
938
hf_overrides={"num_experts": 0},
939
- is_available_online=False,
939
+ min_transformers_version="5.13",
940
),
941
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
942
"HuggingFaceM4/Idefics3-8B-Llama3",
vllm/model_executor/models/hunyuan_vision.py
+20
-8
@@ -31,7 +31,11 @@ from typing import Annotated, Any, Literal, TypeAlias
31
import torch
32
import torch.nn as nn
33
import torch.nn.functional as F
34
-from transformers import BatchFeature
34
+from transformers import BatchFeature, HunYuanVLProcessor
35
+from transformers.models.hunyuan_vl.image_processing_hunyuan_vl import (
36
+ HunYuanVLImageProcessor,
37
+ smart_resize,
38
+)
39
40
from vllm.config import VllmConfig
41
from vllm.config.multimodal import BaseDummyOptions
@@ -75,11 +79,6 @@ from vllm.transformers_utils.configs.hunyuan_vl import (
79
HunYuanVLConfig,
80
HunYuanVLVisionConfig,
81
)
78
-from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor
79
-from vllm.transformers_utils.processors.hunyuan_vl_image import (
80
- HunYuanVLImageProcessor,
81
- smart_resize,
82
-)
82
from vllm.utils.tensor_schema import TensorSchema, TensorShape
83
84
from .interfaces import (
@@ -574,9 +573,12 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
573
self,
574
**kwargs: object,
575
) -> HunYuanVLProcessor:
576
+ # transformers>=5.13 replaced `use_fast` with `backend`; pin the
577
+ # PIL backend to match the released HunyuanOCR checkpoint packing.
578
+ kwargs.pop("use_fast", None)
579
+ kwargs.setdefault("backend", "pil")
580
return self.ctx.get_hf_processor(
581
HunYuanVLProcessor,
579
- use_fast=kwargs.pop("use_fast", True),
582
**kwargs,
583
)
584
@@ -724,8 +726,18 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
726
mm_kwargs: Mapping[str, object],
727
tok_kwargs: Mapping[str, object],
728
) -> BatchFeature:
729
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
730
+ # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
731
+ if mm_data.get("images") is not None and prompt:
732
+ img_tok = hf_processor.image_token
733
+ wrapped = (
734
+ f"{hf_processor.image_start_token}{img_tok}"
735
+ f"{hf_processor.image_end_token}"
736
+ )
737
+ if img_tok in prompt and wrapped not in prompt:
738
+ prompt = prompt.replace(img_tok, wrapped)
739
return self.info.ctx.call_hf_processor(
728
- self.info.get_hf_processor(**mm_kwargs),
740
+ hf_processor,
741
dict(text=prompt, **mm_data),
742
dict(**mm_kwargs, **tok_kwargs),
743
)
vllm/transformers_utils/processors/hunyuan_vl.py
deleted
-226
@@ -1,226 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/processing_hunyuan_vl.py
5
-
6
-import numpy as np
7
-import torch
8
-from transformers.feature_extraction_utils import BatchFeature
9
-from transformers.image_utils import ImageInput
10
-from transformers.processing_utils import ProcessorMixin
11
-from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
12
-from transformers.video_utils import VideoInput
13
-
14
-
15
-class HunYuanVLProcessor(ProcessorMixin):
16
- attributes = ["image_processor", "tokenizer"]
17
- valid_kwargs = ["chat_template"]
18
- image_processor_class = "AutoImageProcessor"
19
- tokenizer_class = "AutoTokenizer" # ("AutoTokenizer", None)
20
-
21
- def __init__(
22
- self,
23
- image_processor=None,
24
- tokenizer=None,
25
- chat_template=None,
26
- **kwargs,
27
- ):
28
- # TODO Fix the init
29
- self.tokenizer = tokenizer
30
- self.image_token_id = 120120 # self.tokenizer.image_token_id
31
- self.image_token = self.tokenizer.convert_ids_to_tokens(self.image_token_id)
32
- self.im_start_token_id = 120118 # self.tokenizer.im_start_id
33
- self.im_start_token = self.tokenizer.convert_ids_to_tokens(
34
- self.im_start_token_id
35
- )
36
- self.im_end_token_id = 120119 # self.tokenizer.im_end_id
37
- self.im_end_token = self.tokenizer.convert_ids_to_tokens(self.im_end_token_id)
38
- self.placeholder_token = self.tokenizer.convert_ids_to_tokens(
39
- self.tokenizer.vocab_size - 1
40
- )
41
- self.pad_id = 120002 # self.tokenizer.pad_token_id
42
-
43
- super().__init__(image_processor, tokenizer, chat_template=chat_template)
44
-
45
- def __call__(
46
- self,
47
- images: ImageInput = None,
48
- text: TextInput
49
- | PreTokenizedInput
50
- | list[TextInput]
51
- | list[PreTokenizedInput] = None,
52
- videos: VideoInput = None,
53
- **kwargs,
54
- ) -> BatchFeature:
55
- image_inputs = {}
56
- if images is not None:
57
- image_inputs = self.image_processor(images=images)
58
- image_grid_thw = image_inputs["image_grid_thw"]
59
-
60
- if not isinstance(text, list):
61
- text = [text]
62
-
63
- text = text.copy() # below lines change text in-place
64
-
65
- image_tokens_cumsum = [0]
66
- if images is not None:
67
- index = 0
68
- for i in range(len(text)):
69
- while self.image_token in text[i]:
70
- grid_h, grid_w = image_grid_thw[index][-2:]
71
- patch_h = grid_h // self.image_processor.merge_size
72
- patch_w = grid_w // self.image_processor.merge_size
73
- num_image_tokens = patch_h * (patch_w + 1) + 2
74
- image_tokens_cumsum.append(
75
- image_tokens_cumsum[-1] + num_image_tokens
76
- )
77
- # text[i] = text[i].replace(self.image_token, self.im_start_token + self.placeholder_token * num_image_tokens + self.im_end_token, 1) # noqa: E501
78
- text[i] = text[i].replace(
79
- self.image_token, self.placeholder_token * num_image_tokens, 1
80
- )
81
- index += 1
82
- text[i] = text[i].replace(self.placeholder_token, self.image_token)
83
- # text[i] = self.tokenizer.bos_token + text[i]
84
-
85
- text_inputs = self.tokenizer(text, add_special_tokens=False, **kwargs)
86
- self._check_special_mm_tokens(text, text_inputs, modalities=["image"])
87
-
88
- input_ids = text_inputs["input_ids"]
89
- position_ids = torch.arange(len(input_ids[0]))
90
- position_ids_w = torch.arange(len(input_ids[0]))
91
- position_ids_h = torch.arange(len(input_ids[0]))
92
- position_ids_t = torch.arange(len(input_ids[0]))
93
-
94
- if images is not None:
95
- image_token_pos_indices = torch.where(input_ids[0] == self.image_token_id)[
96
- 0
97
- ]
98
- for i in range(len(image_grid_thw)):
99
- grid_h, grid_w = image_grid_thw[i][-2:]
100
- patch_h = grid_h // self.image_processor.merge_size
101
- patch_w = grid_w // self.image_processor.merge_size
102
- start_pos = image_token_pos_indices[image_tokens_cumsum[i]].item() + 1
103
- replace_num = (patch_w + 1) * patch_h
104
- position_ids_w[start_pos : start_pos + replace_num] = torch.tensor(
105
- list(range(patch_w + 1)) * patch_h, dtype=torch.int64
106
- )
107
- patch_h_list = []
108
- for h in range(patch_h):
109
- patch_h_list += [h] * (patch_w + 1)
110
- position_ids_h[start_pos : start_pos + replace_num] = torch.tensor(
111
- patch_h_list, dtype=torch.int64
112
- )
113
- position_ids_t[start_pos : start_pos + replace_num] = 0
114
-
115
- position_ids = torch.stack(
116
- [position_ids, position_ids_w, position_ids_h, position_ids_t]
117
- ).unsqueeze(0)
118
- text_inputs["position_ids"] = position_ids
119
-
120
- attention_mask = input_ids.ne(self.pad_id)
121
- text_inputs["attention_mask"] = attention_mask
122
- text_inputs["imgs_pos"] = [self.get_imgs_pos(e) for e in input_ids]
123
- # image_inputs["imgs"] = [[image_inputs["pixel_values"]]]
124
-
125
- return_tensors = kwargs.pop("return_tensors", None)
126
- return BatchFeature(
127
- data={**text_inputs, **image_inputs},
128
- tensor_type=return_tensors,
129
- )
130
-
131
- def batch_decode(self, *args, **kwargs):
132
- return self.tokenizer.batch_decode(*args, **kwargs)
133
-
134
- def decode(self, *args, **kwargs):
135
- return self.tokenizer.decode(*args, **kwargs)
136
-
137
- def post_process_image_text_to_text(
138
- self,
139
- generated_outputs,
140
- skip_special_tokens=True,
141
- clean_up_tokenization_spaces=False,
142
- **kwargs,
143
- ):
144
- assert 0
145
-
146
- def apply_chat_template(self, *args, **kwargs):
147
- kwargs["return_dict"] = False
148
- return self.tokenizer.apply_chat_template(*args, **kwargs)
149
-
150
- def get_imgs_pos(self, doc_ids):
151
- doc_ids = np.array(doc_ids, dtype=np.int64)
152
- img_begin_index = np.where(doc_ids == self.im_start_token_id)[0]
153
- img_end_index = np.where(doc_ids == self.im_end_token_id)[0]
154
- imgs_pos = np.concatenate(
155
- (
156
- np.reshape(img_begin_index + 1, (-1, 1)),
157
- np.reshape(img_end_index, (-1, 1)),
158
- ),
159
- axis=-1,
160
- ).tolist()
161
- return imgs_pos
162
-
163
- @property
164
- def model_input_names(self):
165
- tokenizer_input_names = self.tokenizer.model_input_names
166
- image_processor_input_names = self.image_processor.model_input_names
167
- return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
168
-
169
-
170
-def split_image_into_patch_blocks(
171
- pixel_values: torch.Tensor, # shape: [batch_size, 3, H, W]
172
- patch_size: int = 16, # e.g. 16
173
- adaptor_patch_div: int = 4, # e.g. 4 --> each patch_size is cut into 4x4 small regions, i.e. patch_size // 4 # noqa: E501
174
-) -> torch.Tensor:
175
- """
176
- Split the input image tensor (supporting batch) into large patches of size `patch_size`,
177
- and then further divide each large patch into smaller regions of size
178
- (patch_size // adaptor_patch_div) x (patch_size // adaptor_patch_div).
179
- Each small region is extracted as a tensor of shape [3, patch_size, patch_size].
180
- The final output contains all such small region tensors.
181
-
182
- Args:
183
- pixel_values: Input image tensor of shape [batch_size, 3, H, W].
184
- patch_size: Size of the large patch, e.g., 16.
185
- adaptor_patch_div: Each large patch is divided into
186
- (patch_size // adaptor_patch_div) x (patch_size // adaptor_patch_div)
187
- smaller regions.
188
-
189
- Returns:
190
- patches: A tensor of shape [N, 3, patch_size, patch_size],
191
- where N = batch_size * (H // patch_size) * (W // patch_size) * (patch_size // adaptor_patch_div)^2.
192
- Each element in the batch corresponds to one small image region.
193
- """ # noqa: E501
194
- batch_size, channels, height, width = pixel_values.shape
195
- assert channels == 3, "Pixel values must have 3 channels in dim=1"
196
- assert height % patch_size == 0 and width % patch_size == 0, (
197
- "H and W must be divisible by patch_size"
198
- )
199
-
200
- patch_height_num = height // patch_size
201
- patch_width_num = width // patch_size
202
-
203
- # Reshape to [B, 3, ph, ps, pw, ps]
204
- img = pixel_values.reshape(
205
- batch_size, 3, patch_height_num, patch_size, patch_width_num, patch_size
206
- )
207
-
208
- # Further split each psxps patch into (ps//aps)x(ps//aps) small regions
209
- img = img.reshape(
210
- batch_size,
211
- 3,
212
- patch_height_num,
213
- patch_size // adaptor_patch_div, # ps // aps
214
- adaptor_patch_div,
215
- patch_width_num,
216
- patch_size // adaptor_patch_div, # ps // aps
217
- adaptor_patch_div,
218
- )
219
-
220
- # Permute to group the small regions: [B, ph, pw, ps//aps, ps//aps, 3, aps, aps]
221
- img = img.permute(0, 2, 5, 3, 6, 1, 4, 7)
222
-
223
- # Reshape into [B * ph * pw * (ps//aps)^2, 3, patch_size, patch_size]
224
- patches = img.reshape(-1, 3, patch_size, patch_size)
225
-
226
- return patches
vllm/transformers_utils/processors/hunyuan_vl_image.py
deleted
-477
@@ -1,477 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/image_processing_hunyuan_vl.py
5
-"""Image processor class for HunYuanVL."""
6
-
7
-# isort conflicts with ruff for transformers imports
8
-# isort: skip_file
9
-import math
10
-
11
-import numpy as np
12
-import torchvision.transforms as transforms
13
-from transformers import AutoImageProcessor
14
-from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
15
-from transformers.image_transforms import (
16
- convert_to_rgb,
17
-)
18
-from transformers.image_utils import (
19
- OPENAI_CLIP_MEAN,
20
- OPENAI_CLIP_STD,
21
- ChannelDimension,
22
- ImageInput,
23
- PILImageResampling,
24
- make_flat_list_of_images,
25
- make_list_of_images,
26
- valid_images,
27
- validate_preprocess_arguments,
28
-)
29
-from transformers.utils import TensorType, logging
30
-from transformers.video_utils import VideoInput, make_batched_videos
31
-
32
-logger = logging.get_logger(__name__)
33
-
34
-
35
-def smart_resize(
36
- height: int,
37
- width: int,
38
- factor: int = 16,
39
- min_pixels: int = 512 * 512,
40
- max_pixels: int = 2048 * 2048,
41
-):
42
- """Rescales the image so that the following conditions are met:
43
-
44
- 1. Both dimensions (height and width) are divisible by 'factor'.
45
-
46
- 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
47
-
48
- 3. The aspect ratio of the image is maintained as closely as possible.
49
-
50
- """
51
- if max(height, width) / min(height, width) > 200:
52
- raise ValueError(
53
- "absolute aspect ratio must be smaller than 200, got "
54
- f"{max(height, width) / min(height, width)}"
55
- )
56
- h_bar = round(height / factor) * factor
57
- w_bar = round(width / factor) * factor
58
- if h_bar * w_bar > max_pixels:
59
- beta = math.sqrt((height * width) / max_pixels)
60
- h_bar = max(factor, math.floor(height / beta / factor) * factor)
61
- w_bar = max(factor, math.floor(width / beta / factor) * factor)
62
- elif h_bar * w_bar < min_pixels:
63
- beta = math.sqrt(min_pixels / (height * width))
64
- h_bar = math.ceil(height * beta / factor) * factor
65
- w_bar = math.ceil(width * beta / factor) * factor
66
- return h_bar, w_bar
67
-
68
-
69
-class HunYuanVLImageProcessor(BaseImageProcessor):
70
- model_input_names = [
71
- "pixel_values",
72
- "image_grid_thw",
73
- "pixel_values_videos",
74
- "video_grid_thw",
75
- ]
76
-
77
- def __init__(
78
- self,
79
- do_resize: bool = True,
80
- size: dict[str, int] | None = None,
81
- resample: PILImageResampling = PILImageResampling.BICUBIC,
82
- do_rescale: bool = True,
83
- rescale_factor: int | float = 1 / 255,
84
- do_normalize: bool = True,
85
- image_mean: float | list[float] | None = None,
86
- image_std: float | list[float] | None = None,
87
- do_convert_rgb: bool = True,
88
- min_pixels: int | None = None,
89
- max_pixels: int | None = None,
90
- patch_size: int = 16,
91
- temporal_patch_size: int = 2,
92
- merge_size: int = 2,
93
- **kwargs,
94
- ) -> None:
95
- super().__init__(**kwargs)
96
- if size is not None and (
97
- "shortest_edge" not in size or "longest_edge" not in size
98
- ):
99
- raise ValueError(
100
- "size must contain 'shortest_edge' and 'longest_edge' keys."
101
- )
102
- else:
103
- size = {"shortest_edge": 512 * 512, "longest_edge": 2048 * 2048}
104
- # backward compatibility: override size with min_pixels and max_pixels
105
- # if they are provided.
106
- if min_pixels is not None:
107
- size["shortest_edge"] = min_pixels
108
- if max_pixels is not None:
109
- size["longest_edge"] = max_pixels
110
- self.min_pixels = size["shortest_edge"]
111
- self.max_pixels = size["longest_edge"]
112
- self.size = size
113
-
114
- self.do_resize = do_resize
115
- self.resample = resample
116
- self.do_rescale = do_rescale
117
- self.rescale_factor = rescale_factor
118
- self.do_normalize = do_normalize
119
- self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
120
- self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
121
-
122
- self.patch_size = patch_size
123
- self.temporal_patch_size = temporal_patch_size
124
- self.merge_size = merge_size
125
- self.do_convert_rgb = do_convert_rgb
126
-
127
- # hard-code
128
-
129
- def _preprocess(
130
- self,
131
- images: ImageInput | VideoInput,
132
- do_resize: bool | None = None,
133
- size: dict[str, int] | None = None,
134
- resample: PILImageResampling = None,
135
- do_rescale: bool | None = None,
136
- rescale_factor: float | None = None,
137
- do_normalize: bool | None = None,
138
- image_mean: float | list[float] | None = None,
139
- image_std: float | list[float] | None = None,
140
- patch_size: int = 16,
141
- temporal_patch_size: int = 2,
142
- merge_size: int = 2,
143
- do_convert_rgb: bool | None = None,
144
- data_format: ChannelDimension | None = ChannelDimension.FIRST,
145
- input_data_format: str | ChannelDimension | None = None,
146
- ):
147
- """
148
- Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
149
-
150
- Args:
151
- images (`ImageInput`):
152
- Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
153
- do_resize (`bool`, *optional*, defaults to `self.do_resize`):
154
- Whether to resize the image.
155
- size (`dict[str, int]`, *optional*, defaults to `self.size`):
156
- Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present.
157
- resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
158
- Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
159
- do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
160
- Whether to rescale the image.
161
- rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
162
- Scale factor to use if rescaling the image.
163
- do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
164
- Whether to normalize the image.
165
- image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):
166
- Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
167
- image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):
168
- Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
169
- patch_size (`int`, *optional*, defaults to `self.patch_size`):
170
- The spatial patch size of the vision encoder.
171
- temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`):
172
- The temporal patch size of the vision encoder.
173
- merge_size (`int`, *optional*, defaults to `self.merge_size`):
174
- The merge size of the vision encoder to llm encoder.
175
- do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
176
- Whether to convert the image to RGB.
177
- data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
178
- The channel dimension format for the output image. Can be one of:
179
- - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
180
- - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
181
- - Unset: Use the channel dimension format of the input image.
182
- input_data_format (`ChannelDimension` or `str`, *optional*):
183
- The channel dimension format for the input image. Can be one of:
184
- - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
185
- - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
186
- - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
187
- """ # noqa: E501
188
- images = make_list_of_images(images)
189
-
190
- if do_convert_rgb:
191
- images = [convert_to_rgb(image) for image in images]
192
-
193
- width, height = images[0].width, images[0].height
194
- resized_width, resized_height = width, height
195
- processed_images = []
196
- for image in images:
197
- if do_resize:
198
- resized_height, resized_width = smart_resize(
199
- height=height,
200
- width=width,
201
- factor=patch_size * merge_size,
202
- min_pixels=self.min_pixels,
203
- max_pixels=self.max_pixels,
204
- )
205
- image = image.resize((resized_width, resized_height))
206
-
207
- if do_normalize:
208
- image = transforms.Compose(
209
- [
210
- transforms.ToTensor(),
211
- transforms.Normalize(self.image_mean, self.image_std),
212
- ]
213
- )(image)
214
- processed_images.append(image)
215
-
216
- patches = np.array(processed_images)
217
- channel = patches.shape[1]
218
- grid_t = patches.shape[0] // temporal_patch_size
219
- grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
220
- patches = patches.reshape(
221
- 1,
222
- channel,
223
- grid_h // merge_size,
224
- merge_size,
225
- patch_size,
226
- grid_w // merge_size,
227
- merge_size,
228
- patch_size,
229
- )
230
- patches = patches.transpose(0, 2, 3, 5, 6, 1, 4, 7)
231
- flatten_patches = patches.reshape(
232
- 1 * grid_h * grid_w, channel * patch_size * patch_size
233
- )
234
-
235
- return flatten_patches, (grid_t, grid_h, grid_w)
236
-
237
- def preprocess(
238
- self,
239
- images: ImageInput,
240
- videos: VideoInput = None,
241
- do_resize: bool | None = None,
242
- size: dict[str, int] | None = None,
243
- min_pixels: int | None = None,
244
- max_pixels: int | None = None,
245
- resample: PILImageResampling = None,
246
- do_rescale: bool | None = None,
247
- rescale_factor: float | None = None,
248
- do_normalize: bool | None = None,
249
- image_mean: float | list[float] | None = None,
250
- image_std: float | list[float] | None = None,
251
- patch_size: int | None = None,
252
- temporal_patch_size: int | None = None,
253
- merge_size: int | None = None,
254
- do_convert_rgb: bool | None = None,
255
- return_tensors: str | TensorType | None = None,
256
- data_format: ChannelDimension | None = ChannelDimension.FIRST,
257
- input_data_format: str | ChannelDimension | None = None,
258
- ):
259
- """
260
- Args:
261
- images (`ImageInput`):
262
- Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
263
- passing in images with pixel values between 0 and 1, set `do_rescale=False`.
264
- videos (`VideoInput`):
265
- Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
266
- passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
267
- do_resize (`bool`, *optional*, defaults to `self.do_resize`):
268
- Whether to resize the image.
269
- size (`dict[str, int]`, *optional*, defaults to `self.size`):
270
- Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
271
- the longest edge resized to keep the input aspect ratio.
272
- resample (`int`, *optional*, defaults to `self.resample`):
273
- Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
274
- has an effect if `do_resize` is set to `True`.
275
- do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
276
- Whether to rescale the image.
277
- rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
278
- Rescale factor to rescale the image by if `do_rescale` is set to `True`.
279
- do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
280
- Whether to normalize the image.
281
- image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):
282
- Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
283
- image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):
284
- Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
285
- `True`.
286
- min_pixels (`int`, *optional*, defaults to `self.min_pixels`):
287
- The min pixels of the image to resize the image.
288
- max_pixels (`int`, *optional*, defaults to `self.max_pixels`):
289
- The max pixels of the image to resize the image.
290
- patch_size (`int`, *optional*, defaults to `self.patch_size`):
291
- The spatial patch size of the vision encoder.
292
- temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`):
293
- The temporal patch size of the vision encoder.
294
- merge_size (`int`, *optional*, defaults to `self.merge_size`):
295
- The merge size of the vision encoder to llm encoder.
296
- do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
297
- Whether to convert the image to RGB.
298
- return_tensors (`str` or `TensorType`, *optional*):
299
- The type of tensors to return. Can be one of:
300
- - Unset: Return a list of `np.ndarray`.
301
- - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
302
- - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
303
- - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
304
- - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
305
- data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
306
- The channel dimension format for the output image. Can be one of:
307
- - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
308
- - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
309
- - Unset: Use the channel dimension format of the input image.
310
- input_data_format (`ChannelDimension` or `str`, *optional*):
311
- The channel dimension format for the input image. If unset, the channel dimension format is inferred
312
- from the input image. Can be one of:
313
- - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
314
- - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
315
- - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
316
-
317
- """ # noqa: E501
318
- min_pixels = min_pixels if min_pixels is not None else self.min_pixels
319
- max_pixels = max_pixels if max_pixels is not None else self.max_pixels
320
-
321
- if size is not None:
322
- if "shortest_edge" not in size or "longest_edge" not in size:
323
- raise ValueError(
324
- "size must contain 'shortest_edge' and 'longest_edge' keys."
325
- )
326
- min_pixels = size["shortest_edge"]
327
- elif min_pixels is not None and max_pixels is not None:
328
- # backward compatibility: override size with min_pixels and max_pixels
329
- # if they are provided.
330
- size = {"shortest_edge": min_pixels, "longest_edge": max_pixels}
331
- else:
332
- size = {**self.size}
333
-
334
- do_resize = do_resize if do_resize is not None else self.do_resize
335
-
336
- resample = resample if resample is not None else self.resample
337
- do_rescale = do_rescale if do_rescale is not None else self.do_rescale
338
- rescale_factor = (
339
- rescale_factor if rescale_factor is not None else self.rescale_factor
340
- )
341
- do_normalize = do_normalize if do_normalize is not None else self.do_normalize
342
- image_mean = image_mean if image_mean is not None else self.image_mean
343
- image_std = image_std if image_std is not None else self.image_std
344
- patch_size = patch_size if patch_size is not None else self.patch_size
345
- temporal_patch_size = (
346
- temporal_patch_size
347
- if temporal_patch_size is not None
348
- else self.temporal_patch_size
349
- )
350
- merge_size = merge_size if merge_size is not None else self.merge_size
351
- do_convert_rgb = (
352
- do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
353
- )
354
-
355
- if images is not None:
356
- images = make_flat_list_of_images(images)
357
-
358
- if images is not None and not valid_images(images):
359
- raise ValueError(
360
- "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
361
- "torch.Tensor, tf.Tensor or jax.ndarray."
362
- )
363
-
364
- validate_preprocess_arguments(
365
- rescale_factor=rescale_factor,
366
- do_normalize=do_normalize,
367
- image_mean=image_mean,
368
- image_std=image_std,
369
- do_resize=do_resize,
370
- size=size,
371
- resample=resample,
372
- )
373
-
374
- data = {}
375
- if images is not None:
376
- pixel_values, vision_grid_thws = [], []
377
- for image in images:
378
- patches, image_grid_thw = self._preprocess(
379
- image,
380
- do_resize=do_resize,
381
- size=size,
382
- resample=resample,
383
- do_rescale=do_rescale,
384
- rescale_factor=rescale_factor,
385
- do_normalize=do_normalize,
386
- image_mean=image_mean,
387
- image_std=image_std,
388
- patch_size=patch_size,
389
- temporal_patch_size=temporal_patch_size,
390
- merge_size=merge_size,
391
- data_format=data_format,
392
- do_convert_rgb=do_convert_rgb,
393
- input_data_format=input_data_format,
394
- )
395
- pixel_values.extend(patches)
396
- vision_grid_thws.append(image_grid_thw)
397
- pixel_values = np.array(pixel_values)
398
- vision_grid_thws = np.array(vision_grid_thws)
399
- data.update(
400
- {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}
401
- )
402
-
403
- # kept for BC only and should be removed after v5.0
404
- if videos is not None:
405
- logger.warning(
406
- "`HunYuanVLV1ImageProcessor` works only with image inputs "
407
- "and doesn't process videos anymore. "
408
- "This is a deprecated behavior and will be removed in v5.0. "
409
- "Your videos should be forwarded to `HunYuanVLV1VideoProcessor`. "
410
- )
411
- videos = make_batched_videos(videos)
412
- pixel_values_videos, vision_grid_thws_videos = [], []
413
- for images in videos:
414
- patches, video_grid_thw = self._preprocess(
415
- images,
416
- do_resize=do_resize,
417
- size=size,
418
- resample=resample,
419
- do_rescale=do_rescale,
420
- rescale_factor=rescale_factor,
421
- do_normalize=do_normalize,
422
- image_mean=image_mean,
423
- image_std=image_std,
424
- patch_size=patch_size,
425
- temporal_patch_size=temporal_patch_size,
426
- merge_size=merge_size,
427
- data_format=data_format,
428
- do_convert_rgb=do_convert_rgb,
429
- input_data_format=input_data_format,
430
- )
431
- pixel_values_videos.extend(patches)
432
- vision_grid_thws_videos.append(video_grid_thw)
433
- data.update(
434
- {
435
- "pixel_values_videos": np.array(pixel_values_videos),
436
- "video_grid_thw": np.array(vision_grid_thws_videos),
437
- }
438
- )
439
-
440
- return BatchFeature(data=data, tensor_type=return_tensors)
441
-
442
- def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
443
- """
444
- A utility that returns number of image patches for a given image size.
445
-
446
- Args:
447
- height (`int`):
448
- Height of the input image.
449
- width (`int`):
450
- Width of the input image.
451
- images_kwargs (`dict`, *optional*):
452
- Any kwargs to override defaults of the image processor.
453
- Returns:
454
- `int`: Number of image patches per image.
455
- """
456
- min_pixels = (
457
- images_kwargs["min_pixels"]
458
- if "min_pixels" in images_kwargs
459
- else self.size["shortest_edge"]
460
- )
461
- max_pixels = (
462
- images_kwargs["max_pixels"]
463
- if "max_pixels" in images_kwargs
464
- else self.size["longest_edge"]
465
- )
466
- patch_size = images_kwargs.get("patch_size", self.patch_size)
467
- merge_size = images_kwargs.get("merge_size", self.merge_size)
468
-
469
- factor = patch_size * merge_size
470
- resized_height, resized_width = smart_resize(
471
- height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels
472
- )
473
- grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
474
- return grid_h * (grid_w + 1) + 2
475
-
476
-
477
-AutoImageProcessor.register("HunYuanVLImageProcessor", HunYuanVLImageProcessor)