A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 14:30 UTC
[1/2] Move InternVL-based processors (#37260)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 17, 2026 at 21:50 UTC
f34032433573cda9bc495cf02e783c8b0d99d20d
20 files changed
+3252
-3099
tests/models/multimodal/processing/test_h2ovl.py
+1
-1
@@ -23,7 +23,7 @@ def _get_expected_num_patches(
23
min_num: int,
24
max_num: int,
25
):
26
- from vllm.model_executor.models.h2ovl import (
26
+ from vllm.transformers_utils.processors.h2ovl import (
27
calculate_h2ovl_targets,
28
get_h2ovl_target_ratios,
29
)
tests/models/multimodal/processing/test_internvl.py
+1
-1
@@ -23,7 +23,7 @@ def _get_expected_num_patches(
23
min_num: int,
24
max_num: int,
25
):
26
- from vllm.model_executor.models.internvl import (
26
+ from vllm.transformers_utils.processors.internvl import (
27
calculate_internvl_targets,
28
get_internvl_target_ratios,
29
)
tests/models/multimodal/processing/test_nemotron_vl.py
+1
-1
@@ -23,7 +23,7 @@ def _get_expected_num_patches(
23
min_num: int,
24
max_num: int,
25
):
26
- from vllm.model_executor.models.nemotron_vl import (
26
+ from vllm.transformers_utils.processors.nemotron_vl import (
27
calculate_nemotron_vl_targets,
28
get_nemotron_vl_target_ratios,
29
)
vllm/model_executor/models/eagle2_5_vl.py
+1
-81
@@ -15,9 +15,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.models.module_mapping import MultiModelKeys
16
from vllm.model_executor.models.siglip import SiglipVisionModel
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
-from vllm.multimodal.processing import PromptUpdateDetails
18
from vllm.sequence import IntermediateTensors
20
-from vllm.tokenizers import TokenizerLike
19
+from vllm.transformers_utils.processors.eagle2_5_vl import Eagle2_5_VLProcessor
20
from vllm.utils.tensor_schema import TensorSchema, TensorShape
21
22
from .interfaces import (
@@ -27,13 +26,9 @@ from .interfaces import (
26
SupportsPP,
27
)
28
from .internvl import (
30
- IMG_CONTEXT,
31
- IMG_END,
32
- IMG_START,
29
BaseInternVLDummyInputsBuilder,
30
BaseInternVLMultiModalProcessor,
31
BaseInternVLProcessingInfo,
36
- BaseInternVLProcessor,
32
)
33
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
34
@@ -70,81 +65,6 @@ Eagle2_5_VLImageInputs: TypeAlias = (
65
)
66
67
73
-class Eagle2_5_VLProcessor(BaseInternVLProcessor):
74
- """
75
- Custom processor for Eagle2.5-VL model.
76
- Extends BaseInternVLProcessor with Eagle-specific token handling.
77
- """
78
-
79
- def __init__(
80
- self,
81
- config: PretrainedConfig,
82
- tokenizer: TokenizerLike,
83
- *,
84
- min_dynamic_patch: int | None = None,
85
- max_dynamic_patch: int | None = None,
86
- dynamic_image_size: bool | None = None,
87
- ) -> None:
88
- # Skip super().__init__() to avoid config manipulation
89
- # Directly initialize all required attributes
90
- self.config = config
91
- self.tokenizer = tokenizer
92
-
93
- # Image size with force_image_size override
94
- image_size: int = config.vision_config.image_size
95
- if hasattr(config, "force_image_size") and config.force_image_size:
96
- image_size = config.force_image_size
97
-
98
- patch_size: int = config.vision_config.patch_size
99
- downsample_ratio: float = getattr(config, "downsample_ratio", 0.5)
100
-
101
- # Compute num_image_token
102
- self.num_image_token = int(
103
- (image_size // patch_size) ** 2 * (downsample_ratio**2)
104
- )
105
- self.image_size = image_size
106
-
107
- # Dynamic patch settings with defaults
108
- self.min_dynamic_patch = (
109
- min_dynamic_patch
110
- if min_dynamic_patch is not None
111
- else getattr(config, "min_dynamic_patch", 1)
112
- )
113
- self.max_dynamic_patch = (
114
- max_dynamic_patch
115
- if max_dynamic_patch is not None
116
- else getattr(config, "max_dynamic_patch", 12)
117
- )
118
- self.dynamic_image_size = (
119
- dynamic_image_size
120
- if dynamic_image_size is not None
121
- else getattr(config, "dynamic_image_size", True)
122
- )
123
- self.use_thumbnail: bool = getattr(config, "use_thumbnail", True)
124
-
125
- @property
126
- def image_token_id(self) -> int:
127
- """Get the image token ID from config or tokenizer."""
128
- if hasattr(self.config, "image_token_index"):
129
- return self.config.image_token_index
130
- # Fallback to tokenizer vocab - use <IMG_CONTEXT> (ID: 151667)
131
- vocab = self.tokenizer.get_vocab()
132
- if IMG_CONTEXT in vocab:
133
- return vocab[IMG_CONTEXT]
134
- raise ValueError(f"Cannot find image token '{IMG_CONTEXT}' in vocabulary")
135
-
136
- def get_image_repl(
137
- self,
138
- feature_size: int,
139
- num_patches: int | None,
140
- ) -> PromptUpdateDetails[str]:
141
- """Get image replacement string for prompt."""
142
- repl_features = IMG_CONTEXT * feature_size
143
- repl_full = IMG_START + repl_features + IMG_END
144
-
145
- return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
146
-
147
-
68
class Eagle2_5_VLProcessingInfo(BaseInternVLProcessingInfo):
69
"""Processing info for Eagle2.5-VL model."""
70
vllm/model_executor/models/h2ovl.py
+1
-374
@@ -11,7 +11,6 @@
11
from collections.abc import Mapping, Sequence
12
13
import torch
14
-from PIL import Image
14
from transformers import PretrainedConfig
15
16
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -27,391 +26,19 @@ from vllm.multimodal.processing.processor import (
26
ProcessorInputs,
27
PromptReplacement,
28
PromptUpdate,
30
- PromptUpdateDetails,
29
TimingContext,
30
)
33
-from vllm.tokenizers import TokenizerLike
31
+from vllm.transformers_utils.processors.h2ovl import H2OVLProcessor
32
33
from .intern_vit import InternVisionModel
34
from .internvl import (
37
- IMG_CONTEXT,
38
- IMG_END,
39
- IMG_START,
35
BaseInternVLDummyInputsBuilder,
36
BaseInternVLMultiModalProcessor,
37
BaseInternVLProcessingInfo,
43
- BaseInternVLProcessor,
38
InternVLChatModel,
45
- build_transform,
46
- find_closest_aspect_ratio,
47
- get_internvl_target_ratios,
39
)
40
41
51
-def resolve_h2ovl_min_max_num(
52
- *,
53
- min_dynamic_patch: int,
54
- max_dynamic_patch: int,
55
- dynamic_image_size: bool,
56
- use_thumbnail: bool,
57
-) -> tuple[int, int]:
58
- min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
59
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
60
-
61
- if use_thumbnail and max_dynamic_patch != 1:
62
- max_dynamic_patch += 1
63
-
64
- return min_dynamic_patch, max_dynamic_patch
65
-
66
-
67
-def get_h2ovl_target_ratios(
68
- min_num: int,
69
- max_num: int,
70
- *,
71
- prior_aspect_ratio: tuple[int, int] | None,
72
-) -> list[tuple[int, int]]:
73
- target_ratios = get_internvl_target_ratios(min_num, max_num)
74
-
75
- # if prior_aspect_ratio is provided, filter the target ratios
76
- if prior_aspect_ratio is not None:
77
- target_ratios = [
78
- ratio
79
- for ratio in target_ratios
80
- if prior_aspect_ratio[0] % ratio[0] != 0
81
- and prior_aspect_ratio[1] % ratio[1] != 0
82
- ]
83
-
84
- return target_ratios
85
-
86
-
87
-# modified to include blocks generated in second pass
88
-def calculate_h2ovl_targets(
89
- *,
90
- orig_width: int,
91
- orig_height: int,
92
- target_ratios: list[tuple[int, int]],
93
- image_size: int,
94
- use_thumbnail: bool,
95
-) -> tuple[int, int, int, tuple[int, int]]:
96
- aspect_ratio = orig_width / orig_height
97
-
98
- # find the closest aspect ratio to the target
99
- target_aspect_ratio = find_closest_aspect_ratio(
100
- aspect_ratio,
101
- target_ratios,
102
- width=orig_width,
103
- height=orig_height,
104
- image_size=image_size,
105
- )
106
-
107
- # calculate the target width and height
108
- target_width = image_size * target_aspect_ratio[0]
109
- target_height = image_size * target_aspect_ratio[1]
110
- blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
111
-
112
- # add thumbnail image if num_blocks != 1
113
- if use_thumbnail and blocks != 1:
114
- blocks += 1
115
-
116
- return blocks, target_width, target_height, target_aspect_ratio
117
-
118
-
119
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
120
-# refactored to handle prior_aspect_ratio
121
-def dynamic_preprocess_h2ovl(
122
- image: Image.Image,
123
- *,
124
- target_ratios: list[tuple[int, int]],
125
- image_size: int,
126
- use_thumbnail: bool,
127
-) -> tuple[list[Image.Image], tuple[int, int]]:
128
- orig_width, orig_height = image.size
129
-
130
- # calculate the number of blocks without thumbnail
131
- (
132
- blocks,
133
- target_width,
134
- target_height,
135
- target_aspect_ratio,
136
- ) = calculate_h2ovl_targets(
137
- orig_width=orig_width,
138
- orig_height=orig_height,
139
- target_ratios=target_ratios,
140
- image_size=image_size,
141
- use_thumbnail=False,
142
- )
143
-
144
- # resize the image
145
- resized_img = image.resize((target_width, target_height))
146
- processed_images = []
147
- for i in range(blocks):
148
- box = (
149
- (i % (target_width // image_size)) * image_size,
150
- (i // (target_width // image_size)) * image_size,
151
- ((i % (target_width // image_size)) + 1) * image_size,
152
- ((i // (target_width // image_size)) + 1) * image_size,
153
- )
154
- # split the image
155
- split_img = resized_img.crop(box)
156
- processed_images.append(split_img)
157
-
158
- assert len(processed_images) == blocks
159
-
160
- if use_thumbnail and len(processed_images) != 1:
161
- thumbnail_img = image.resize((image_size, image_size))
162
- processed_images.append(thumbnail_img)
163
-
164
- return processed_images, target_aspect_ratio
165
-
166
-
167
-def _preprocess_image(
168
- image: Image.Image,
169
- *,
170
- input_size: int,
171
- min_num: int,
172
- max_num: int,
173
- use_thumbnail: bool,
174
- prior_aspect_ratio: tuple[int, int] | None,
175
-) -> tuple[torch.Tensor, tuple[int, int]]:
176
- target_ratios = get_h2ovl_target_ratios(
177
- min_num,
178
- max_num,
179
- prior_aspect_ratio=prior_aspect_ratio,
180
- )
181
-
182
- transform = build_transform(input_size=input_size)
183
- images, target_aspect_ratio = dynamic_preprocess_h2ovl(
184
- image,
185
- image_size=input_size,
186
- use_thumbnail=use_thumbnail,
187
- target_ratios=target_ratios,
188
- )
189
-
190
- pixel_values = torch.stack([transform(image) for image in images])
191
- return pixel_values, target_aspect_ratio
192
-
193
-
194
-# refactored to use the _preprocess_image function
195
-def image_to_pixel_values_h2ovl(
196
- image: Image.Image,
197
- *,
198
- input_size: int,
199
- min_num: int,
200
- max_num: int,
201
- use_thumbnail: bool,
202
- use_msac: bool,
203
-) -> torch.Tensor:
204
- # when MSAC is turned on, we need to process the image twice
205
- if use_msac:
206
- # first pass
207
- pixel_values1, aspect_ratio1 = _preprocess_image(
208
- image,
209
- input_size=input_size,
210
- min_num=1,
211
- max_num=max_num,
212
- use_thumbnail=True,
213
- prior_aspect_ratio=None,
214
- )
215
- # second pass
216
- pixel_values2, _ = _preprocess_image(
217
- image,
218
- input_size=input_size,
219
- min_num=3,
220
- max_num=max_num,
221
- use_thumbnail=True,
222
- prior_aspect_ratio=aspect_ratio1,
223
- )
224
- # combine pixel values
225
- pixel_values = torch.cat(
226
- [pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0
227
- )
228
-
229
- else:
230
- pixel_values, _ = _preprocess_image(
231
- image,
232
- input_size=input_size,
233
- min_num=min_num,
234
- max_num=max_num,
235
- use_thumbnail=use_thumbnail,
236
- prior_aspect_ratio=None,
237
- )
238
-
239
- return pixel_values
240
-
241
-
242
-class H2OVLProcessor(BaseInternVLProcessor):
243
- def __init__(
244
- self,
245
- config: PretrainedConfig,
246
- tokenizer: TokenizerLike,
247
- *,
248
- min_dynamic_patch: int | None = None,
249
- max_dynamic_patch: int | None = None,
250
- dynamic_image_size: bool | None = None,
251
- use_msac: bool | None = None,
252
- ) -> None:
253
- super().__init__(
254
- config,
255
- tokenizer,
256
- min_dynamic_patch=min_dynamic_patch,
257
- max_dynamic_patch=max_dynamic_patch,
258
- dynamic_image_size=dynamic_image_size,
259
- )
260
-
261
- if use_msac is None:
262
- use_msac = config.use_msac
263
- assert isinstance(use_msac, bool)
264
-
265
- self.use_msac = use_msac
266
-
267
- @property
268
- def image_token_id(self) -> int:
269
- return self.tokenizer.get_vocab()[IMG_CONTEXT]
270
-
271
- def get_image_repl(
272
- self,
273
- feature_size: int,
274
- num_patches: int | None,
275
- ) -> PromptUpdateDetails[str]:
276
- repl_features = IMG_CONTEXT * feature_size
277
- repl_full = IMG_START + repl_features + IMG_END
278
-
279
- return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
280
-
281
- def resolve_min_max_num(
282
- self,
283
- *,
284
- min_dynamic_patch: int | None = None,
285
- max_dynamic_patch: int | None = None,
286
- dynamic_image_size: bool | None = None,
287
- use_thumbnail: bool | None = None,
288
- ) -> tuple[int, int]:
289
- min_dynamic_patch = (
290
- self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
291
- )
292
- max_dynamic_patch = (
293
- self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
294
- )
295
- dynamic_image_size = (
296
- self.dynamic_image_size
297
- if dynamic_image_size is None
298
- else dynamic_image_size
299
- )
300
- use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
301
-
302
- return resolve_h2ovl_min_max_num(
303
- min_dynamic_patch=min_dynamic_patch,
304
- max_dynamic_patch=max_dynamic_patch,
305
- dynamic_image_size=dynamic_image_size,
306
- use_thumbnail=use_thumbnail,
307
- )
308
-
309
- def resolve_target_ratios(
310
- self,
311
- *,
312
- min_dynamic_patch: int | None = None,
313
- max_dynamic_patch: int | None = None,
314
- dynamic_image_size: bool | None = None,
315
- use_thumbnail: bool | None = None,
316
- prior_aspect_ratio: tuple[int, int] | None = None,
317
- override_min_num: int | None = None,
318
- ) -> list[tuple[int, int]]:
319
- min_num, max_num = self.resolve_min_max_num(
320
- min_dynamic_patch=min_dynamic_patch,
321
- max_dynamic_patch=max_dynamic_patch,
322
- dynamic_image_size=dynamic_image_size,
323
- use_thumbnail=use_thumbnail,
324
- )
325
- if override_min_num is not None:
326
- min_num = override_min_num
327
-
328
- return get_h2ovl_target_ratios(
329
- min_num,
330
- max_num,
331
- prior_aspect_ratio=prior_aspect_ratio,
332
- )
333
-
334
- def get_num_image_tokens(
335
- self,
336
- *,
337
- image_width: int,
338
- image_height: int,
339
- use_msac: bool | None = None,
340
- ) -> int:
341
- use_msac = self.use_msac if use_msac is None else use_msac
342
-
343
- use_thumbnail = self.use_thumbnail
344
-
345
- if use_msac:
346
- target_ratios_1 = self.resolve_target_ratios(
347
- use_thumbnail=False, # Applied in calculate_targets
348
- override_min_num=1,
349
- )
350
- num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
351
- orig_width=image_width,
352
- orig_height=image_height,
353
- image_size=self.image_size,
354
- target_ratios=target_ratios_1,
355
- use_thumbnail=True,
356
- )
357
-
358
- target_ratios_2 = self.resolve_target_ratios(
359
- use_thumbnail=False, # Applied in calculate_targets
360
- prior_aspect_ratio=aspect_ratio_1,
361
- override_min_num=3,
362
- )
363
- num_patches_2, _, _, _ = calculate_h2ovl_targets(
364
- orig_width=image_width,
365
- orig_height=image_height,
366
- image_size=self.image_size,
367
- target_ratios=target_ratios_2,
368
- use_thumbnail=True,
369
- )
370
-
371
- num_patches = num_patches_1 + num_patches_2 - 1
372
- else:
373
- target_ratios = self.resolve_target_ratios(
374
- use_thumbnail=False, # Applied in calculate_targets
375
- )
376
- num_patches, _, _, _ = calculate_h2ovl_targets(
377
- orig_width=image_width,
378
- orig_height=image_height,
379
- image_size=self.image_size,
380
- target_ratios=target_ratios,
381
- use_thumbnail=use_thumbnail,
382
- )
383
-
384
- return num_patches * self.num_image_token
385
-
386
- def _images_to_pixel_values_lst(
387
- self,
388
- images: list[Image.Image],
389
- min_dynamic_patch: int | None = None,
390
- max_dynamic_patch: int | None = None,
391
- dynamic_image_size: bool | None = None,
392
- ) -> list[torch.Tensor]:
393
- use_msac = self.use_msac if len(images) == 1 else False
394
-
395
- min_num, max_num = self.resolve_min_max_num(
396
- min_dynamic_patch=min_dynamic_patch,
397
- max_dynamic_patch=max_dynamic_patch,
398
- dynamic_image_size=dynamic_image_size,
399
- use_thumbnail=False, # Applied in image_to_pixel_values
400
- )
401
-
402
- return [
403
- image_to_pixel_values_h2ovl(
404
- image,
405
- input_size=self.image_size,
406
- min_num=min_num,
407
- max_num=max_num,
408
- use_thumbnail=self.use_thumbnail,
409
- use_msac=use_msac,
410
- )
411
- for image in images
412
- ]
413
-
414
-
42
class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
43
def get_hf_processor(self, **kwargs: object) -> H2OVLProcessor:
44
return self.ctx.init_processor(
vllm/model_executor/models/internvl.py
+7
-578
@@ -7,16 +7,13 @@
7
# Copyright (c) 2023 OpenGVLab
8
# Licensed under The MIT License [see LICENSE for details]
9
# --------------------------------------------------------
10
-from abc import ABC, abstractmethod
10
+from abc import abstractmethod
11
from collections.abc import Iterable, Mapping, Sequence
12
-from typing import Annotated, Any, Literal, TypeAlias, TypeVar
12
+from typing import Annotated, Literal, TypeAlias, TypeVar
13
14
-import numpy.typing as npt
14
import torch
15
import torch.nn as nn
17
-import torchvision.transforms as T
18
-from PIL import Image
19
-from transformers import BatchFeature, PretrainedConfig, TensorType
16
+from transformers import BatchFeature, PretrainedConfig
17
18
from vllm.config import VllmConfig
19
from vllm.config.multimodal import BaseDummyOptions
@@ -28,7 +25,6 @@ from vllm.model_executor.models.intern_vit import (
25
)
26
from vllm.model_executor.models.module_mapping import MultiModelKeys
27
from vllm.multimodal import MULTIMODAL_REGISTRY
31
-from vllm.multimodal.image import convert_image_mode
28
from vllm.multimodal.inputs import (
29
MultiModalDataDict,
30
MultiModalFieldConfig,
@@ -46,10 +42,12 @@ from vllm.multimodal.processing import (
42
BaseProcessingInfo,
43
PromptReplacement,
44
PromptUpdate,
49
- PromptUpdateDetails,
45
)
46
from vllm.sequence import IntermediateTensors
52
-from vllm.tokenizers import TokenizerLike
47
+from vllm.transformers_utils.processors.internvl import (
48
+ BaseInternVLProcessor,
49
+ InternVLProcessor,
50
+)
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
53
from .interfaces import (
@@ -60,13 +58,6 @@ from .interfaces import (
58
)
59
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
60
63
-IMG_START = "<img>"
64
-IMG_END = "</img>"
65
-IMG_CONTEXT = "<IMG_CONTEXT>"
66
-
67
-IMAGENET_MEAN = (0.485, 0.456, 0.406)
68
-IMAGENET_STD = (0.229, 0.224, 0.225)
69
-
61
62
class InternVLImagePixelInputs(TensorSchema):
63
"""
@@ -128,568 +119,6 @@ class InternVLVideoEmbeddingInputs(TensorSchema):
119
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
120
121
131
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
132
-def build_transform(input_size: int):
133
- MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
134
- transform = T.Compose(
135
- [
136
- T.Lambda(lambda img: convert_image_mode(img, "RGB")),
137
- T.Resize(
138
- (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
139
- ),
140
- T.ToTensor(),
141
- T.Normalize(mean=MEAN, std=STD),
142
- ]
143
- )
144
- return transform
145
-
146
-
147
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
148
-def find_closest_aspect_ratio(
149
- aspect_ratio: float,
150
- target_ratios: list[tuple[int, int]],
151
- *,
152
- width: int,
153
- height: int,
154
- image_size: int,
155
-) -> tuple[int, int]:
156
- best_ratio_diff = float("inf")
157
- best_ratio = (1, 1)
158
- area = width * height
159
- for ratio in target_ratios:
160
- target_aspect_ratio = ratio[0] / ratio[1]
161
- ratio_diff = abs(aspect_ratio - target_aspect_ratio)
162
- if ratio_diff < best_ratio_diff:
163
- best_ratio_diff = ratio_diff
164
- best_ratio = ratio
165
- elif ratio_diff == best_ratio_diff:
166
- if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
167
- best_ratio = ratio
168
- return best_ratio
169
-
170
-
171
-def resolve_internvl_min_max_num(
172
- *,
173
- min_dynamic_patch: int,
174
- max_dynamic_patch: int,
175
- dynamic_image_size: bool,
176
- use_thumbnail: bool,
177
-) -> tuple[int, int]:
178
- min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
179
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
180
-
181
- if use_thumbnail and max_dynamic_patch != 1:
182
- max_dynamic_patch += 1
183
-
184
- return min_dynamic_patch, max_dynamic_patch
185
-
186
-
187
-def get_internvl_target_ratios(
188
- min_num: int,
189
- max_num: int,
190
-) -> list[tuple[int, int]]:
191
- target_ratios = {
192
- (i, j)
193
- for n in range(min_num, max_num + 1)
194
- for i in range(1, n + 1)
195
- for j in range(1, n + 1)
196
- if min_num <= i * j <= max_num
197
- }
198
- return sorted(target_ratios, key=lambda x: x[0] * x[1])
199
-
200
-
201
-def calculate_internvl_targets(
202
- *,
203
- orig_width: int,
204
- orig_height: int,
205
- target_ratios: list[tuple[int, int]],
206
- image_size: int,
207
- use_thumbnail: bool,
208
-) -> tuple[int, int, int]:
209
- aspect_ratio = orig_width / orig_height
210
-
211
- # find the closest aspect ratio to the target
212
- target_aspect_ratio = find_closest_aspect_ratio(
213
- aspect_ratio,
214
- target_ratios,
215
- width=orig_width,
216
- height=orig_height,
217
- image_size=image_size,
218
- )
219
-
220
- # calculate the target width and height
221
- target_width = image_size * target_aspect_ratio[0]
222
- target_height = image_size * target_aspect_ratio[1]
223
- blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
224
-
225
- # add thumbnail image if num_blocks != 1
226
- if use_thumbnail and blocks != 1:
227
- blocks += 1
228
-
229
- return blocks, target_width, target_height
230
-
231
-
232
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
233
-def dynamic_preprocess_internvl(
234
- image: Image.Image,
235
- *,
236
- target_ratios: list[tuple[int, int]],
237
- image_size: int,
238
- use_thumbnail: bool,
239
-) -> list[Image.Image]:
240
- orig_width, orig_height = image.size
241
-
242
- # calculate the number of blocks without thumbnail
243
- blocks, target_width, target_height = calculate_internvl_targets(
244
- orig_width=orig_width,
245
- orig_height=orig_height,
246
- target_ratios=target_ratios,
247
- image_size=image_size,
248
- use_thumbnail=False,
249
- )
250
-
251
- # resize the image
252
- resized_img = image.resize((target_width, target_height))
253
- processed_images = []
254
- for i in range(blocks):
255
- box = (
256
- (i % (target_width // image_size)) * image_size,
257
- (i // (target_width // image_size)) * image_size,
258
- ((i % (target_width // image_size)) + 1) * image_size,
259
- ((i // (target_width // image_size)) + 1) * image_size,
260
- )
261
- # split the image
262
- split_img = resized_img.crop(box)
263
- processed_images.append(split_img)
264
-
265
- assert len(processed_images) == blocks
266
-
267
- if use_thumbnail and len(processed_images) != 1:
268
- thumbnail_img = image.resize((image_size, image_size))
269
- processed_images.append(thumbnail_img)
270
-
271
- return processed_images
272
-
273
-
274
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
275
-def image_to_pixel_values_internvl(
276
- image: Image.Image,
277
- *,
278
- input_size: int,
279
- min_num: int,
280
- max_num: int,
281
- use_thumbnail: bool,
282
-) -> torch.Tensor:
283
- target_ratios = get_internvl_target_ratios(min_num, max_num)
284
-
285
- transform = build_transform(input_size=input_size)
286
- images = dynamic_preprocess_internvl(
287
- image,
288
- target_ratios=target_ratios,
289
- image_size=input_size,
290
- use_thumbnail=use_thumbnail,
291
- )
292
-
293
- pixel_values = torch.stack([transform(image) for image in images])
294
- return pixel_values
295
-
296
-
297
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
298
-def video_to_pixel_values_internvl(
299
- video: npt.NDArray,
300
- *,
301
- input_size: int,
302
- min_num: int,
303
- max_num: int,
304
- use_thumbnail: bool,
305
-) -> torch.Tensor:
306
- target_ratios = get_internvl_target_ratios(min_num, max_num)
307
-
308
- transform = build_transform(input_size=input_size)
309
- frames_list = list[Image.Image]()
310
- for frame in video:
311
- pil_frame = dynamic_preprocess_internvl(
312
- Image.fromarray(frame, mode="RGB"),
313
- target_ratios=target_ratios,
314
- image_size=input_size,
315
- use_thumbnail=use_thumbnail,
316
- )
317
- assert len(pil_frame) == 1
318
- frames_list.extend(pil_frame)
319
-
320
- pixel_values = torch.stack([transform(image) for image in frames_list])
321
- return pixel_values
322
-
323
-
324
-class BaseInternVLProcessor(ABC):
325
- """
326
- This model doesn't define its own HF processor,
327
- so we implement our own one here.
328
-
329
- The code to insert image tokens is based on:
330
- https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
331
- """
332
-
333
- def __init__(
334
- self,
335
- config: PretrainedConfig,
336
- tokenizer: TokenizerLike,
337
- *,
338
- min_dynamic_patch: int | None = None,
339
- max_dynamic_patch: int | None = None,
340
- dynamic_image_size: bool | None = None,
341
- ) -> None:
342
- super().__init__()
343
-
344
- self.config = config
345
- self.tokenizer = tokenizer
346
-
347
- image_size: int = config.vision_config.image_size
348
- patch_size: int = config.vision_config.patch_size
349
-
350
- if min_dynamic_patch is None:
351
- min_dynamic_patch = config.min_dynamic_patch
352
- assert isinstance(min_dynamic_patch, int)
353
-
354
- if max_dynamic_patch is None:
355
- max_dynamic_patch = config.max_dynamic_patch
356
- assert isinstance(max_dynamic_patch, int)
357
-
358
- if dynamic_image_size is None:
359
- dynamic_image_size = config.dynamic_image_size
360
- assert isinstance(dynamic_image_size, bool)
361
-
362
- self.num_image_token = int(
363
- (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
364
- )
365
- self.image_size = image_size
366
- self.min_dynamic_patch = min_dynamic_patch
367
- self.max_dynamic_patch = max_dynamic_patch
368
- self.dynamic_image_size = dynamic_image_size
369
- self.use_thumbnail: bool = config.use_thumbnail
370
-
371
- @property
372
- @abstractmethod
373
- def image_token_id(self) -> int:
374
- raise NotImplementedError
375
-
376
- @abstractmethod
377
- def get_image_repl(
378
- self,
379
- feature_size: int,
380
- num_patches: int | None,
381
- ) -> PromptUpdateDetails[str]:
382
- raise NotImplementedError
383
-
384
- def resolve_min_max_num(
385
- self,
386
- *,
387
- min_dynamic_patch: int | None = None,
388
- max_dynamic_patch: int | None = None,
389
- dynamic_image_size: bool | None = None,
390
- use_thumbnail: bool | None = None,
391
- ) -> tuple[int, int]:
392
- min_dynamic_patch = (
393
- self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
394
- )
395
- max_dynamic_patch = (
396
- self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
397
- )
398
- dynamic_image_size = (
399
- self.dynamic_image_size
400
- if dynamic_image_size is None
401
- else dynamic_image_size
402
- )
403
- use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
404
-
405
- return resolve_internvl_min_max_num(
406
- min_dynamic_patch=min_dynamic_patch,
407
- max_dynamic_patch=max_dynamic_patch,
408
- dynamic_image_size=dynamic_image_size,
409
- use_thumbnail=use_thumbnail,
410
- )
411
-
412
- def resolve_target_ratios(
413
- self,
414
- *,
415
- min_dynamic_patch: int | None = None,
416
- max_dynamic_patch: int | None = None,
417
- dynamic_image_size: bool | None = None,
418
- use_thumbnail: bool | None = None,
419
- ) -> list[tuple[int, int]]:
420
- min_num, max_num = self.resolve_min_max_num(
421
- min_dynamic_patch=min_dynamic_patch,
422
- max_dynamic_patch=max_dynamic_patch,
423
- dynamic_image_size=dynamic_image_size,
424
- use_thumbnail=use_thumbnail,
425
- )
426
-
427
- return get_internvl_target_ratios(min_num, max_num)
428
-
429
- def get_num_image_tokens(
430
- self,
431
- *,
432
- image_width: int,
433
- image_height: int,
434
- ) -> int:
435
- target_ratios = self.resolve_target_ratios(
436
- use_thumbnail=False, # Applied in calculate_targets
437
- )
438
-
439
- num_patches, _, _ = calculate_internvl_targets(
440
- orig_width=image_width,
441
- orig_height=image_height,
442
- image_size=self.image_size,
443
- target_ratios=target_ratios,
444
- use_thumbnail=self.use_thumbnail,
445
- )
446
-
447
- return num_patches * self.num_image_token
448
-
449
- def _images_to_pixel_values_lst(
450
- self,
451
- images: list[Image.Image],
452
- min_dynamic_patch: int | None = None,
453
- max_dynamic_patch: int | None = None,
454
- dynamic_image_size: bool | None = None,
455
- ) -> list[torch.Tensor]:
456
- min_num, max_num = self.resolve_min_max_num(
457
- min_dynamic_patch=min_dynamic_patch,
458
- max_dynamic_patch=max_dynamic_patch,
459
- dynamic_image_size=dynamic_image_size,
460
- use_thumbnail=False, # Applied in image_to_pixel_values
461
- )
462
-
463
- return [
464
- image_to_pixel_values_internvl(
465
- image,
466
- input_size=self.image_size,
467
- min_num=min_num,
468
- max_num=max_num,
469
- use_thumbnail=self.use_thumbnail,
470
- )
471
- for image in images
472
- ]
473
-
474
- def _preprocess_image(
475
- self,
476
- text: list[str],
477
- images: list[Image.Image],
478
- min_dynamic_patch: int | None = None,
479
- max_dynamic_patch: int | None = None,
480
- dynamic_image_size: bool | None = None,
481
- ) -> tuple[list[str], dict[str, torch.Tensor]]:
482
- if len(images) == 0:
483
- image_inputs = {}
484
- else:
485
- pixel_values_lst = self._images_to_pixel_values_lst(
486
- images,
487
- min_dynamic_patch=min_dynamic_patch,
488
- max_dynamic_patch=max_dynamic_patch,
489
- dynamic_image_size=dynamic_image_size,
490
- )
491
- image_inputs = {
492
- "pixel_values_flat": torch.cat(pixel_values_lst),
493
- "image_num_patches": torch.tensor(
494
- [len(item) for item in pixel_values_lst]
495
- ),
496
- }
497
-
498
- for pixel_values in pixel_values_lst:
499
- num_patches = pixel_values.shape[0]
500
- feature_size = num_patches * self.num_image_token
501
-
502
- image_repl = self.get_image_repl(feature_size, num_patches)
503
- text = [t.replace("<image>", image_repl.full, 1) for t in text]
504
- return text, image_inputs
505
-
506
- def _make_batch_input(self, input_item: Any | list[Any] | None = None):
507
- if input_item is None:
508
- input_item = []
509
- if not isinstance(input_item, list):
510
- input_item = [input_item]
511
- return input_item
512
-
513
- def __call__(
514
- self,
515
- text: str | list[str] | None = None,
516
- images: Image.Image | list[Image.Image] | None = None,
517
- min_dynamic_patch: int | None = None,
518
- max_dynamic_patch: int | None = None,
519
- dynamic_image_size: bool | None = None,
520
- return_tensors: str | TensorType | None = None,
521
- ) -> BatchFeature:
522
- text, images = [self._make_batch_input(x) for x in (text, images)]
523
-
524
- text, image_inputs = self._preprocess_image(
525
- text=text,
526
- images=images,
527
- min_dynamic_patch=min_dynamic_patch,
528
- max_dynamic_patch=max_dynamic_patch,
529
- dynamic_image_size=dynamic_image_size,
530
- )
531
-
532
- text_inputs = self.tokenizer(text)
533
-
534
- combined_outputs = {**text_inputs, **image_inputs}
535
-
536
- return BatchFeature(combined_outputs, tensor_type=return_tensors)
537
-
538
-
539
-class InternVLProcessor(BaseInternVLProcessor):
540
- """
541
- HF Processor for InternVLChatModel with extended video processing logic.
542
-
543
- Code for video processing is adapted from video example:
544
- https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
545
- """
546
-
547
- def __init__(
548
- self,
549
- config: PretrainedConfig,
550
- tokenizer: TokenizerLike,
551
- *,
552
- min_dynamic_patch: int | None = None,
553
- max_dynamic_patch: int | None = None,
554
- dynamic_image_size: bool | None = None,
555
- video_token: str | None = None,
556
- ) -> None:
557
- super().__init__(
558
- config=config,
559
- tokenizer=tokenizer,
560
- min_dynamic_patch=min_dynamic_patch,
561
- max_dynamic_patch=max_dynamic_patch,
562
- dynamic_image_size=dynamic_image_size,
563
- )
564
- # add extra video token for video processing
565
- self.video_token = video_token
566
-
567
- @property
568
- def image_token_id(self) -> int:
569
- return self.tokenizer.get_vocab()[IMG_CONTEXT]
570
-
571
- @property
572
- def video_token_id(self) -> int | None:
573
- if self.video_token is None:
574
- return None
575
- return self.tokenizer.get_vocab().get(self.video_token, None)
576
-
577
- @property
578
- def supports_video(self) -> bool:
579
- return self.video_token_id is not None
580
-
581
- def _videos_to_pixel_values_lst(
582
- self,
583
- videos: list[npt.NDArray],
584
- dynamic_image_size: bool | None = None,
585
- ) -> list[torch.Tensor]:
586
- min_num, max_num = self.resolve_min_max_num(
587
- min_dynamic_patch=1,
588
- max_dynamic_patch=1,
589
- dynamic_image_size=dynamic_image_size,
590
- use_thumbnail=False, # Applied in image_to_pixel_values
591
- )
592
-
593
- return [
594
- video_to_pixel_values_internvl(
595
- video,
596
- input_size=self.image_size,
597
- min_num=min_num,
598
- max_num=max_num,
599
- use_thumbnail=False,
600
- )
601
- for video in videos
602
- ]
603
-
604
- def _preprocess_video(
605
- self,
606
- text: list[str],
607
- videos: list[npt.NDArray],
608
- dynamic_image_size: bool | None = None,
609
- ):
610
- if len(videos) == 0 or not self.supports_video:
611
- video_inputs = {}
612
- else:
613
- pixel_values_lst_video = self._videos_to_pixel_values_lst(
614
- videos,
615
- dynamic_image_size=dynamic_image_size,
616
- )
617
- video_inputs = {
618
- "pixel_values_flat_video": torch.cat(pixel_values_lst_video),
619
- "video_num_patches": torch.tensor(
620
- [len(item) for item in pixel_values_lst_video]
621
- ),
622
- }
623
-
624
- for pixel_values in pixel_values_lst_video:
625
- num_patches = pixel_values.shape[0]
626
-
627
- video_repl = self.get_video_repl(
628
- self.num_image_token, num_patches, self.video_token
629
- )
630
- text = [t.replace("<video>", video_repl.full, 1) for t in text]
631
- return text, video_inputs
632
-
633
- def __call__(
634
- self,
635
- text: str | list[str] | None = None,
636
- images: Image.Image | list[Image.Image] | None = None,
637
- videos: npt.NDArray | list[npt.NDArray] | None = None,
638
- min_dynamic_patch: int | None = None,
639
- max_dynamic_patch: int | None = None,
640
- dynamic_image_size: bool | None = None,
641
- return_tensors: str | TensorType | None = None,
642
- ) -> BatchFeature:
643
- text, images, videos = [
644
- self._make_batch_input(x) for x in (text, images, videos)
645
- ]
646
-
647
- text, image_inputs = self._preprocess_image(
648
- text=text,
649
- images=images,
650
- min_dynamic_patch=min_dynamic_patch,
651
- max_dynamic_patch=max_dynamic_patch,
652
- dynamic_image_size=dynamic_image_size,
653
- )
654
-
655
- text, video_inputs = self._preprocess_video(
656
- text=text,
657
- videos=videos,
658
- dynamic_image_size=dynamic_image_size,
659
- )
660
-
661
- text_inputs = self.tokenizer(text)
662
-
663
- combined_outputs = {**text_inputs, **image_inputs, **video_inputs}
664
-
665
- return BatchFeature(combined_outputs, tensor_type=return_tensors)
666
-
667
- def get_image_repl(
668
- self,
669
- feature_size: int,
670
- num_patches: int | None,
671
- ) -> PromptUpdateDetails[str]:
672
- repl_features = IMG_CONTEXT * feature_size
673
- repl_full = IMG_START + repl_features + IMG_END
674
-
675
- return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
676
-
677
- def get_video_repl(
678
- self,
679
- feature_size: int,
680
- num_patches: int | None = None,
681
- video_context_token: str = IMG_CONTEXT,
682
- ) -> PromptUpdateDetails[str]:
683
- repl_features = video_context_token * self.num_image_token
684
- repl_features_with_sep = IMG_START + repl_features + IMG_END
685
- # num_patches is equal to num_frames
686
- repl_full = "".join(
687
- [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
688
- )
689
-
690
- return PromptUpdateDetails.select_text(repl_full, video_context_token)
691
-
692
-
122
class BaseInternVLProcessingInfo(BaseProcessingInfo):
123
"""Basic image-only ProcessingInfo for InternVL-style models."""
124
vllm/model_executor/models/nano_nemotron_vl.py
+16
-1017
@@ -8,22 +8,15 @@
8
# --------------------------------------------------------
9
10
import copy
11
-import math
11
import warnings
13
-from abc import ABC, abstractmethod
12
+from abc import abstractmethod
13
from collections.abc import Iterable, Mapping, Sequence
15
-from dataclasses import dataclass
14
from functools import cached_property
17
-from typing import Annotated, Any, Literal, TypeAlias, TypeVar
15
+from typing import Annotated, Literal, TypeAlias, TypeVar
16
19
-import einops
20
-import numpy as np
21
-import numpy.typing as npt
22
-import regex as re
17
import torch
18
import torch.nn as nn
25
-from PIL import Image
26
-from transformers import BatchFeature, PretrainedConfig, TensorType
19
+from transformers import BatchFeature
20
21
from vllm.config import VllmConfig
22
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
@@ -38,10 +31,6 @@ from vllm.model_executor.models.interfaces import (
31
SupportsMultiModal,
32
SupportsMultiModalPruning,
33
)
41
-from vllm.model_executor.models.internvl import (
42
- calculate_internvl_targets,
43
- get_internvl_target_ratios,
44
-)
34
from vllm.model_executor.models.module_mapping import MultiModelKeys
35
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
36
from vllm.model_executor.models.parakeet import ParakeetExtractor, ProjectedParakeet
@@ -83,23 +72,28 @@ from vllm.multimodal.processing.processor import (
72
BaseProcessingInfo,
73
PromptReplacement,
74
PromptUpdate,
86
- PromptUpdateDetails,
87
- _seq2tokens,
75
)
76
from vllm.renderers import TokenizeParams
77
from vllm.sequence import IntermediateTensors
91
-from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
78
+from vllm.tokenizers import cached_tokenizer_from_config
79
from vllm.transformers_utils.configs.radio import RadioConfig
80
+from vllm.transformers_utils.processors.nano_nemotron_vl import (
81
+ AUDIO_CONTEXT,
82
+ IMG_CONTEXT,
83
+ IMG_END,
84
+ IMG_START,
85
+ BaseNanoNemotronVLProcessor,
86
+ DynamicResolutionImageTiler,
87
+ NanoNemotronVLProcessor,
88
+ get_internvl_target_ratios,
89
+)
90
from vllm.utils.tensor_schema import TensorSchema, TensorShape
91
92
from .utils import _merge_multimodal_embeddings
93
94
logger = init_logger(__name__)
98
-# Configure PIL to handle large images without warnings
99
-# This prevents DecompressionBombWarning for legitimate large images
100
-Image.MAX_IMAGE_PIXELS = None # Disable the limit entirely
101
-# Alternative: Set a specific higher limit
102
-# Image.MAX_IMAGE_PIXELS = 300000000 # ~300M pixels
95
+
96
+MAX_AUDIO_LEN_S = 10 * 60 # 10 minutes
97
98
99
class NanoNemotronVLAudioFeatureInputs(TensorSchema):
@@ -116,20 +110,6 @@ class NanoNemotronVLAudioFeatureInputs(TensorSchema):
110
audio_feature_lengths: Annotated[torch.Tensor, TensorShape("b")]
111
112
119
-MAX_AUDIO_LEN_S = 10 * 60 # 10 minutes
120
-
121
-IMG_START = "<img>"
122
-IMG_END = "</img>"
123
-IMG_CONTEXT = "<image>"
124
-AUDIO_START = "<so_start>"
125
-AUDIO_END = "<so_end>"
126
-AUDIO_CONTEXT = "<so_embedding>"
127
-
128
-# Profiling
129
-# MAX_FRAMES = 16
130
-DEFAULT_NUM_TILES = 12
131
-
132
-
113
class NanoNemotronVLImagePixelInputs(TensorSchema):
114
"""
115
Dimensions:
@@ -213,987 +193,6 @@ NanoNemotronVLVideoInputs: TypeAlias = (
193
)
194
195
216
-def dynamic_preprocess(
217
- image,
218
- *,
219
- image_size=512,
220
- max_num_tiles=12,
221
- use_thumbnail=True,
222
- idx=0,
223
-):
224
- orig_width, orig_height = image.size
225
-
226
- target_ratios = get_internvl_target_ratios(1, max_num_tiles)
227
-
228
- blocks, target_width, target_height = calculate_internvl_targets(
229
- orig_width=orig_width,
230
- orig_height=orig_height,
231
- target_ratios=target_ratios,
232
- image_size=image_size,
233
- use_thumbnail=False,
234
- )
235
-
236
- image = np.asarray(
237
- image.convert("RGB") if image.mode != "RGB" else image, dtype=np.uint8
238
- )
239
-
240
- image = torch.from_numpy(image).unsqueeze(0) # (1, H, W, 3)
241
- image = image.permute(0, 3, 1, 2) # (1, 3, H, W)
242
-
243
- resized_img = torch.nn.functional.interpolate(
244
- image,
245
- size=(target_height, target_width),
246
- mode="bicubic",
247
- align_corners=False,
248
- antialias=True,
249
- )
250
- B, C, H, W = resized_img.shape
251
- hp, wp = H // image_size, W // image_size
252
- patches = (
253
- resized_img.reshape(B, C, hp, image_size, wp, image_size)
254
- .permute(0, 2, 4, 1, 3, 5)
255
- .reshape(B * hp * wp, C, image_size, image_size)
256
- / 255.0
257
- )
258
-
259
- if use_thumbnail and patches.shape[0] > 1:
260
- thumb = (
261
- torch.nn.functional.interpolate(
262
- image,
263
- size=(image_size, image_size),
264
- mode="bicubic",
265
- align_corners=False,
266
- antialias=True,
267
- )
268
- / 255.0
269
- )
270
- patches = torch.cat([patches, thumb], dim=0)
271
-
272
- return list(patches)
273
-
274
-
275
-def image_to_pixel_values(
276
- image: Image.Image,
277
- *,
278
- input_size: int,
279
- max_num: int,
280
- use_thumbnail: bool,
281
- idx: int,
282
-) -> torch.Tensor:
283
- images = dynamic_preprocess(
284
- image,
285
- image_size=input_size,
286
- max_num_tiles=max_num,
287
- use_thumbnail=use_thumbnail,
288
- idx=idx,
289
- )
290
-
291
- pixel_values = torch.stack(images)
292
- return pixel_values
293
-
294
-
295
-def video_to_pixel_values(
296
- video: npt.NDArray,
297
- *,
298
- input_size: int,
299
- max_num_tiles: int = 1,
300
- use_thumbnail: bool,
301
-) -> torch.Tensor:
302
- assert max_num_tiles == 1, "Video modality always uses one tile"
303
-
304
- # (num_frames, H, W, C) -> (num_frames, C, H, W)
305
- video_tensor = torch.from_numpy(video).permute(0, 3, 1, 2)
306
-
307
- if video_tensor.shape[2] != input_size or video_tensor.shape[3] != input_size:
308
- video_tensor = torch.nn.functional.interpolate(
309
- video_tensor,
310
- size=(input_size, input_size),
311
- mode="bicubic",
312
- align_corners=False,
313
- antialias=True,
314
- )
315
-
316
- video_tensor = video_tensor / 255.0
317
-
318
- return video_tensor
319
-
320
-
321
-def input_conditioner(x, norm_mean, norm_std):
322
- return (x - norm_mean) / norm_std
323
-
324
-
325
-def calculate_timestamps(
326
- indices: list[int] | torch.Tensor,
327
- frame_duration_ms: int,
328
-):
329
- if not isinstance(indices, list):
330
- indices = indices.tolist()
331
-
332
- timestamps = [int(i) * frame_duration_ms / 1000.0 for i in indices]
333
- return timestamps
334
-
335
-
336
-class DynamicResolutionImageTiler:
337
- CONV_MERGING = False
338
- PIXEL_SHUFFLE = True
339
- USE_THUMBNAIL = False
340
-
341
- def __init__(
342
- self,
343
- *,
344
- max_model_len: int,
345
- patch_size: int,
346
- min_num_patches: int,
347
- max_num_patches: int,
348
- downsample_ratio: int,
349
- norm_mean: Sequence[float],
350
- norm_std: Sequence[float],
351
- factor_max: float = 1.0,
352
- use_thumbnail: bool = False,
353
- ) -> None:
354
- assert use_thumbnail is False, "use_thumbnail is not supported"
355
- self._patch_size: int = patch_size
356
- self._max_model_len = max_model_len
357
- self._min_num_patches = min_num_patches
358
- self._max_num_patches = max_num_patches if max_num_patches > 0 else float("inf")
359
- self._factor_max = factor_max
360
- self.norm_mean = torch.tensor(norm_mean).reshape(3, 1, 1)
361
- self.norm_std = torch.tensor(norm_std).reshape(3, 1, 1)
362
- assert downsample_ratio < 1
363
- reduction_factor = 1 / downsample_ratio
364
- assert reduction_factor == 2.0
365
- self._downsample_ratio = int(reduction_factor) ** (
366
- self.PIXEL_SHUFFLE + self.CONV_MERGING
367
- )
368
- assert self._downsample_ratio == 2
369
-
370
- def _get_num_embeddings(self, width: int, height: int) -> int:
371
- num_patches = (width // self._patch_size) * (height // self._patch_size)
372
- num_tokens = num_patches // (self._downsample_ratio**2)
373
- return num_tokens
374
-
375
- def width_and_height_for_max_num_tokens_available(
376
- self,
377
- target_num_tokens_post_shuffle: int,
378
- ) -> tuple[int, int]:
379
- """
380
- TODO: optimize this so it squeezes closer to target number of tokens.
381
- Calculate image dimensions that produce approximately `target` tokens after
382
- pixel_shuffle.
383
-
384
- With pixel_shuffle enabled, each 2x2 patch grid becomes 1 token, so we
385
- need 4*B patches to get B tokens.
386
-
387
- Examples:
388
- >>> PATCH_SIZE = 16
389
- >>> DOWNSAMPLE_RATIO = 0.5
390
- >>> tiler = DynamicResolutionImageTiler(
391
- ... max_model_len=16384,
392
- ... patch_size=PATCH_SIZE,
393
- ... downsample_ratio=DOWNSAMPLE_RATIO,
394
- ... min_num_patches=4,
395
- ... max_num_patches=0,
396
- ... )
397
- >>> width, height = tiler.width_and_height_for_max_num_tokens_available(
398
- ... target_num_tokens_post_shuffle=8192,
399
- ... )
400
- >>> assert width, height == (2880, 2880)
401
- >>> assert (width // PATCH_SIZE) * (
402
- ... height // PATCH_SIZE
403
- ... ) // 2**2 == 8100 # tokens post-shuffle
404
- >>> assert tiler._get_num_embeddings(width=width, height=height) == 8100
405
- """
406
- side_pixels = (
407
- math.isqrt(target_num_tokens_post_shuffle)
408
- * self._downsample_ratio
409
- * self._patch_size
410
- )
411
- assert isinstance(side_pixels, int) and side_pixels % self._patch_size == 0
412
- return side_pixels, side_pixels
413
-
414
- def max_num_tokens_available(self, text_prompt_length: int) -> int:
415
- return self._max_model_len - text_prompt_length - 4
416
-
417
- def _images_to_pixel_values_lst(
418
- self,
419
- text_prompt_length: int,
420
- images: list[Image.Image],
421
- ) -> tuple[list[torch.Tensor], list[int]]:
422
- num_tokens_available = self.max_num_tokens_available(text_prompt_length)
423
- params_per_image = self.compute_params(images, num_tokens_available)
424
-
425
- feature_sizes = []
426
- images = []
427
- for param in params_per_image:
428
- for t in self.apply_params(param):
429
- assert t.ndim == 3, f"{t.ndim=}: expected 3 dim tensor"
430
- images.append(t)
431
- feature_sizes.append(param.num_embeddings)
432
- return images, feature_sizes
433
-
434
- feature_size_cache: dict[Image.Image, int] = {}
435
-
436
- @classmethod
437
- def get_cached_feature_size(cls, image: Image.Image) -> int:
438
- feature_size = cls.feature_size_cache[id(image)]
439
- # hard assert that we only use the feature size once
440
- del cls.feature_size_cache[id(image)]
441
- return feature_size
442
-
443
- @dataclass
444
- class DynamicResolutionParams:
445
- media: Image.Image
446
- num_tiles: int
447
- num_embeddings: int
448
- patch_size: tuple[int, int]
449
-
450
- def apply_params(self, params: DynamicResolutionParams) -> list[torch.Tensor]:
451
- target_size = (
452
- params.patch_size[1] * self._patch_size,
453
- params.patch_size[0] * self._patch_size,
454
- )
455
- image = np.asarray(
456
- params.media.convert("RGB") if params.media.mode != "RGB" else params.media,
457
- dtype=np.uint8,
458
- )
459
- resized_img = (
460
- torch.nn.functional.interpolate(
461
- torch.from_numpy(image).unsqueeze(0).permute(0, 3, 1, 2),
462
- size=target_size,
463
- mode="bicubic",
464
- align_corners=False,
465
- antialias=True,
466
- )
467
- / 255.0
468
- )
469
- return list(resized_img)
470
-
471
- def process_media(
472
- self,
473
- media: Image.Image,
474
- num_tokens_available: int,
475
- ) -> tuple[DynamicResolutionParams, int]:
476
- """Process a single media item and return its parameters.
477
-
478
- Args:
479
- media: The media item to process
480
- num_tokens_available: Number of tokens available for this media
481
- Returns:
482
- DynamicResolutionParams for the media
483
- """
484
- current_num_tokens_available = num_tokens_available
485
- assert isinstance(media, Image.Image), (
486
- "Dynamic resolution is only supported for image media"
487
- )
488
- orig_width, orig_height = media.width, media.height
489
- closest_patch_height = round(orig_height / self._patch_size + 0.5)
490
- closest_patch_width = round(orig_width / self._patch_size + 0.5)
491
- patches = closest_patch_height * closest_patch_width
492
-
493
- factor = min(
494
- math.sqrt(current_num_tokens_available / patches), self._factor_max
495
- )
496
- target_patch_height = math.floor(factor * closest_patch_height)
497
- target_patch_width = math.floor(factor * closest_patch_width)
498
-
499
- # Consider self._min_num_patches if > current_num_tokens_available.
500
- if (
501
- current_num_tokens_available > self._min_num_patches
502
- and target_patch_height * target_patch_width < self._min_num_patches
503
- ):
504
- up_factor = math.sqrt(
505
- self._min_num_patches / (target_patch_height * target_patch_width)
506
- )
507
- target_patch_height = math.ceil(up_factor * target_patch_height)
508
- target_patch_width = math.ceil(up_factor * target_patch_width)
509
-
510
- # Round patch grid to be divisible by 2 (pixel-shuffle OR conv-merging)
511
- # or by 4 when BOTH are enabled (two successive 2x reductions)
512
- if self.PIXEL_SHUFFLE or self.CONV_MERGING:
513
- required_divisor = 4 if (self.PIXEL_SHUFFLE and self.CONV_MERGING) else 2
514
-
515
- rem_h = target_patch_height % required_divisor
516
- if rem_h != 0:
517
- inc_h = required_divisor - rem_h
518
- if (
519
- target_patch_height + inc_h
520
- ) * target_patch_width <= current_num_tokens_available:
521
- target_patch_height += inc_h
522
- else:
523
- target_patch_height = max(
524
- required_divisor, target_patch_height - rem_h
525
- )
526
-
527
- rem_w = target_patch_width % required_divisor
528
- if rem_w != 0:
529
- inc_w = required_divisor - rem_w
530
- if (
531
- target_patch_height * (target_patch_width + inc_w)
532
- <= current_num_tokens_available
533
- ):
534
- target_patch_width += inc_w
535
- else:
536
- target_patch_width = max(
537
- required_divisor, target_patch_width - rem_w
538
- )
539
-
540
- # Calculate embeddings for the main dynamic resolution image
541
- num_embeddings = self._get_num_embeddings(
542
- target_patch_width * self._patch_size,
543
- target_patch_height * self._patch_size,
544
- )
545
-
546
- token_count = target_patch_width * target_patch_height
547
-
548
- # Add thumbnail embeddings if enabled and image area is below threshold
549
- num_tiles = 1 # Base dynamic resolution image
550
-
551
- return self.DynamicResolutionParams(
552
- media=media,
553
- num_tiles=num_tiles,
554
- num_embeddings=num_embeddings,
555
- patch_size=(target_patch_width, target_patch_height),
556
- ), token_count
557
-
558
- def compute_params(
559
- self,
560
- media_list: list[Image.Image],
561
- num_tokens_available: int | None = None,
562
- ) -> list[DynamicResolutionParams]:
563
- """Compute parameters for all media with iterative token budgeting.
564
-
565
- Args:
566
- media_list: List of media items to process
567
- num_tokens_available: Total number of tokens available across all media
568
- Returns:
569
- List of ImageTilingParams for each media item
570
- """
571
- num_tokens_available = (
572
- num_tokens_available
573
- * (4 if self.PIXEL_SHUFFLE else 1)
574
- * (4 if self.CONV_MERGING else 1)
575
- )
576
- # When the number of available token is too small,
577
- # allow self._min_num_patches per media and let the sample be truncated.
578
- num_tokens_available = max(
579
- num_tokens_available, self._min_num_patches * len(media_list)
580
- )
581
-
582
- # Clip the number of tokens available per media to >min and <max patches.
583
- num_tokens_available_per_media = [
584
- max(min(num_tokens_available, self._max_num_patches), self._min_num_patches)
585
- for _ in range(len(media_list))
586
- ]
587
-
588
- # prevent infinite loop in any case
589
- for _ in range(10):
590
- # Step 1: Process each media with current token budget
591
- params = []
592
- token_counts = []
593
-
594
- for media, tokens_for_media in zip(
595
- media_list, num_tokens_available_per_media
596
- ):
597
- param, token_count = self.process_media(media, tokens_for_media)
598
- params.append(param)
599
- token_counts.append(token_count)
600
- self.feature_size_cache[id(param.media)] = param.num_embeddings
601
-
602
- # Step 2: Check if total tokens is within budget
603
- total_tokens = sum(token_counts)
604
-
605
- if total_tokens <= num_tokens_available:
606
- # We're within budget, return the params
607
- return params
608
-
609
- # Step 3: We're over budget, need to scale down
610
- # Calculate scaling factor to get under budget
611
- scaling_factor = num_tokens_available / total_tokens
612
-
613
- # Recalculate token budgets for each media based on scaling
614
- # Each media gets a proportional share of the total budget
615
- scaled_down_num_tokens_available_per_media = [
616
- max(self._min_num_patches, int(token_count * scaling_factor))
617
- for token_count in token_counts
618
- ]
619
- scaled_down = any(
620
- [
621
- scaled_down_num_tokens_available_per_media[i]
622
- < num_tokens_available_per_media[i]
623
- for i in range(len(num_tokens_available_per_media))
624
- ]
625
- )
626
- # If there wasn't scaling down, we're stuck with min_num_patches per media,
627
- # else try with the scaled down num_tokens_available_per_media.
628
- if not scaled_down:
629
- num_tokens_available_per_media = [self._min_num_patches] * len(
630
- media_list
631
- )
632
- else:
633
- num_tokens_available_per_media = (
634
- scaled_down_num_tokens_available_per_media
635
- )
636
- ctx = f"{params=} {total_tokens=} {num_tokens_available=}"
637
- raise ValueError(
638
- f"Should be unreachable - `return params` above must be reached: {ctx}"
639
- )
640
-
641
- @staticmethod
642
- def stack(images: list[torch.Tensor], patch_size: int) -> torch.Tensor:
643
- assert len(images) > 0, "No images to stack"
644
-
645
- def rearrange_img(x):
646
- py = x.shape[-2] // patch_size
647
- px = x.shape[-1] // patch_size
648
- x = einops.rearrange(
649
- x,
650
- "c (py yy) (px xx) -> (py px) (c yy xx)",
651
- py=py,
652
- yy=patch_size,
653
- px=px,
654
- xx=patch_size,
655
- )
656
- return x
657
-
658
- imgs = [rearrange_img(img) for img in images]
659
- pixel_values_flat = torch.cat(imgs, dim=0).unsqueeze(0)
660
- return pixel_values_flat
661
-
662
-
663
-class BaseNanoNemotronVLProcessor(ABC):
664
- """
665
- This model doesn't define its own HF processor,
666
- so we implement our own one here.
667
-
668
- The code to insert image tokens is based on:
669
- https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
670
- """
671
-
672
- def __init__(
673
- self,
674
- config: PretrainedConfig,
675
- tokenizer: TokenizerLike,
676
- *args,
677
- max_model_len: int,
678
- max_num_tiles: int | None = None,
679
- **kwargs,
680
- ) -> None:
681
- super().__init__()
682
-
683
- self.config = config
684
- self.tokenizer = tokenizer
685
-
686
- self.max_num_tiles = max_num_tiles or DEFAULT_NUM_TILES
687
- image_size: int = config.force_image_size
688
- patch_size: int = config.patch_size
689
- downsample_ratio: int = config.downsample_ratio
690
-
691
- self.num_image_token = int(
692
- (image_size // patch_size) ** 2 * (downsample_ratio**2)
693
- )
694
- self.image_size = image_size
695
- self.use_thumbnail: bool = config.use_thumbnail
696
- self.norm_mean = torch.Tensor(config.norm_mean).reshape(1, 3, 1, 1)
697
- self.norm_std = torch.Tensor(config.norm_std).reshape(1, 3, 1, 1)
698
-
699
- self.dynamic_tiler: DynamicResolutionImageTiler | None = None
700
- if self.use_dynamic_resolution(config):
701
- self.dynamic_tiler = DynamicResolutionImageTiler(
702
- max_model_len=max_model_len,
703
- patch_size=patch_size,
704
- downsample_ratio=downsample_ratio,
705
- min_num_patches=config.vision_config.args["min_num_patches"],
706
- max_num_patches=config.vision_config.args["max_num_patches"],
707
- norm_mean=config.norm_mean,
708
- norm_std=config.norm_std,
709
- )
710
-
711
- @staticmethod
712
- def use_dynamic_resolution(config: PretrainedConfig) -> bool:
713
- return "min_num_patches" in config.vision_config.args
714
-
715
- @property
716
- @abstractmethod
717
- def image_token_id(self) -> int:
718
- raise NotImplementedError
719
-
720
- @abstractmethod
721
- def get_image_repl(
722
- self,
723
- feature_size: int,
724
- num_patches: int | None,
725
- ) -> PromptUpdateDetails[str]:
726
- raise NotImplementedError
727
-
728
- def get_num_image_tokens(
729
- self,
730
- *,
731
- image_width: int,
732
- image_height: int,
733
- max_num_tiles: int,
734
- ) -> int:
735
- target_ratios = get_internvl_target_ratios(1, max_num_tiles)
736
-
737
- num_patches, _, _ = calculate_internvl_targets(
738
- orig_width=image_width,
739
- orig_height=image_height,
740
- target_ratios=target_ratios,
741
- image_size=self.image_size,
742
- use_thumbnail=self.use_thumbnail,
743
- )
744
-
745
- return num_patches * self.num_image_token
746
-
747
- def _images_to_pixel_values_lst(
748
- self,
749
- images: list[Image.Image],
750
- max_num_tiles: int,
751
- ) -> list[torch.Tensor]:
752
- return [
753
- image_to_pixel_values(
754
- image,
755
- input_size=self.image_size,
756
- max_num=max_num_tiles,
757
- use_thumbnail=self.use_thumbnail,
758
- idx=idx,
759
- )
760
- for idx, image in enumerate(images)
761
- ]
762
-
763
- def _preprocess_image(
764
- self,
765
- text: list[str],
766
- images: list[Image.Image],
767
- max_num_tiles: int,
768
- ) -> tuple[list[str], dict[str, Any]]:
769
- if len(images) == 0:
770
- image_inputs = {}
771
- return text, image_inputs
772
-
773
- if tiler := self.dynamic_tiler:
774
- sans_images = text[0].replace("<image>", "")
775
- text_prompt_length = len(
776
- self.tokenizer(sans_images, add_special_tokens=False).input_ids
777
- )
778
- pixel_values_lst, num_tokens_per_image = tiler._images_to_pixel_values_lst(
779
- text_prompt_length=text_prompt_length,
780
- images=images,
781
- )
782
- imgs_sizes = [(pv.shape[-2], pv.shape[-1]) for pv in pixel_values_lst]
783
- normalized = [
784
- input_conditioner(img, tiler.norm_mean, tiler.norm_std)
785
- for img in pixel_values_lst
786
- ]
787
- image_num_patches = torch.tensor([1] * len(num_tokens_per_image))
788
- image_inputs = {
789
- "pixel_values_flat": normalized,
790
- "imgs_sizes": imgs_sizes,
791
- "num_tokens_per_image": num_tokens_per_image,
792
- }
793
- else:
794
- pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles)
795
- image_num_patches = torch.tensor([len(item) for item in pixel_values_lst])
796
- pixel_values_flat = input_conditioner(
797
- torch.cat(pixel_values_lst), self.norm_mean, self.norm_std
798
- )
799
- image_inputs = {
800
- "pixel_values_flat": pixel_values_flat,
801
- "image_num_patches": image_num_patches,
802
- }
803
- num_tokens_per_image = [
804
- self.num_image_token * len(item) for item in pixel_values_lst
805
- ]
806
-
807
- assert len(text) == 1, (
808
- "hf_processor is called on the output of get_dummy_text, "
809
- "which should be a single string"
810
- )
811
- parts = [x for x in re.split(r"(<image>)", text[0]) if x]
812
- assert parts.count("<image>") == len(pixel_values_lst), (
813
- "the number of <image> tokens in the text should be the "
814
- "same as the number of images"
815
- )
816
-
817
- for i, (feature_size, num_patches) in enumerate(
818
- zip(num_tokens_per_image, image_num_patches, strict=True)
819
- ):
820
- image_repl = self.get_image_repl(feature_size, num_patches)
821
- parts[i] = parts[i].replace("<image>", image_repl.full)
822
- text = ["".join(parts)]
823
-
824
- return text, image_inputs
825
-
826
- def _make_batch_input(self, input_item: Any | list[Any] | None = None):
827
- if input_item is None:
828
- input_item = []
829
- if not isinstance(input_item, list):
830
- input_item = [input_item]
831
- return input_item
832
-
833
- @abstractmethod
834
- def __call__(
835
- self,
836
- text: str | list[str] | None = None,
837
- images: Image.Image | list[Image.Image] | None = None,
838
- return_tensors: str | TensorType | None = None,
839
- max_num_tiles: int | None = None,
840
- ) -> BatchFeature:
841
- raise NotImplementedError
842
-
843
-
844
-class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
845
- """
846
- HF Processor with extended video processing logic.
847
- Code for video processing is adapted from video example:
848
- https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
849
- """
850
-
851
- def __init__(
852
- self,
853
- config: PretrainedConfig,
854
- tokenizer: TokenizerLike,
855
- *,
856
- max_model_len: int,
857
- max_num_tiles: int | None = None,
858
- video_token: str | None = None,
859
- video_pruning_rate: float | None = None,
860
- ) -> None:
861
- super().__init__(
862
- config=config,
863
- tokenizer=tokenizer,
864
- max_model_len=max_model_len,
865
- max_num_tiles=max_num_tiles,
866
- )
867
- # add extra video token for video processing
868
- self.video_token = video_token
869
- self.video_pruning_rate = video_pruning_rate
870
-
871
- self.audio_extractor: ParakeetExtractor | None = None
872
- raw_sound_config = getattr(config, "sound_config", None)
873
- if raw_sound_config is not None:
874
- self.audio_extractor = ParakeetExtractor(raw_sound_config)
875
-
876
- # Pre-tokenize special tokens for video processing
877
- # to avoid repeated tokenization
878
- self._img_start_token_ids = tokenizer.encode(
879
- IMG_START, add_special_tokens=False
880
- )
881
- self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)
882
- self._img_context_token_ids = tokenizer.encode(
883
- IMG_CONTEXT, add_special_tokens=False
884
- )
885
-
886
- @property
887
- def supports_video(self) -> bool:
888
- return self.video_token_id is not None
889
-
890
- @property
891
- def video_token_id(self) -> int | None:
892
- if self.video_token is None:
893
- return None
894
- return self.tokenizer.get_vocab().get(self.video_token, None)
895
-
896
- @property
897
- def image_token_id(self) -> int:
898
- return self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT)
899
-
900
- def _videos_to_pixel_values_lst(
901
- self,
902
- videos: list[npt.NDArray],
903
- max_num_tiles: int,
904
- ) -> list[torch.Tensor]:
905
- return [
906
- video_to_pixel_values(
907
- video,
908
- input_size=self.image_size,
909
- max_num_tiles=max_num_tiles,
910
- use_thumbnail=self.use_thumbnail,
911
- )
912
- for video in videos
913
- ]
914
-
915
- def _preprocess_video(
916
- self,
917
- text: list[str],
918
- videos: list[tuple[npt.NDArray, dict[str, Any]]],
919
- max_num_tiles: int,
920
- ):
921
- if len(videos) == 0 or not self.supports_video:
922
- video_inputs = {}
923
- else:
924
- videos_lst = [v[0] for v in videos]
925
- video_metadata_lst = [v[1] for v in videos]
926
- pixel_values_lst_video = self._videos_to_pixel_values_lst(
927
- videos_lst,
928
- max_num_tiles=max_num_tiles,
929
- )
930
-
931
- # We use frame duration in milliseconds (as integer) to ensure
932
- # we have consistent timestamps calculation. At preprocessing
933
- # fps parameter is given in fp32, while at inference it is bf16
934
- # which leads to inaccurate timestamp calculation and causes
935
- # timestamp values to differ.In rare cases this causes
936
- # mismatching number of output tokens for tokenized frame prefixes
937
- frame_duration_ms_lst = [
938
- int(1000.0 / metadata["fps"]) for metadata in video_metadata_lst
939
- ]
940
- frames_indices_lst = [
941
- metadata["frames_indices"] for metadata in video_metadata_lst
942
- ]
943
- video_num_patches = torch.tensor(
944
- [len(item) for item in pixel_values_lst_video]
945
- )
946
- video_inputs = {
947
- "pixel_values_flat_video": input_conditioner(
948
- torch.cat(pixel_values_lst_video), self.norm_mean, self.norm_std
949
- ),
950
- "video_num_patches": video_num_patches,
951
- "frames_indices": frames_indices_lst,
952
- "frame_duration_ms": torch.tensor(frame_duration_ms_lst),
953
- }
954
-
955
- image_size: int = self.config.force_image_size
956
- patch_size: int = self.config.patch_size
957
- downsample_ratio = self.config.downsample_ratio
958
- tokens_in_single_frame = int(
959
- (image_size * image_size // patch_size**2) * (downsample_ratio**2)
960
- )
961
-
962
- for pixel_values, video_metadata, frames_indices, frame_duration_ms in zip(
963
- pixel_values_lst_video,
964
- video_metadata_lst,
965
- frames_indices_lst,
966
- frame_duration_ms_lst,
967
- ):
968
- num_frames = pixel_values.shape[0]
969
-
970
- if (
971
- self.video_pruning_rate is not None
972
- and self.video_pruning_rate > 0.0
973
- ):
974
- # Start of EVS-specific code
975
- num_tokens = compute_retained_tokens_count(
976
- tokens_per_frame=tokens_in_single_frame,
977
- num_frames=num_frames,
978
- q=self.video_pruning_rate,
979
- )
980
-
981
- # Here we just need placeholders that won't actually be replaced -
982
- # we just need to make sure the total number of tokens is correct
983
- # assign all tokens to the first frame
984
- tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
985
-
986
- # End of EVS-specific code
987
- else:
988
- tokens_per_frame = [tokens_in_single_frame] * num_frames
989
-
990
- video_repl = self.get_video_repl(
991
- tokens_per_frame=tokens_per_frame,
992
- frames_indices=frames_indices,
993
- frame_duration_ms=frame_duration_ms,
994
- tokenizer=self.tokenizer,
995
- img_start_token_ids=self._img_start_token_ids,
996
- img_end_token_ids=self._img_end_token_ids,
997
- img_context_token_ids=self._img_context_token_ids,
998
- )
999
-
1000
- # video_repl.full is a list of token IDs
1001
- # Convert token IDs back to text for the HF processor flow
1002
- video_repl_text = self.tokenizer.decode(
1003
- video_repl.full, skip_special_tokens=False
1004
- )
1005
- text = [t.replace("<video>", video_repl_text, 1) for t in text]
1006
-
1007
- return text, video_inputs
1008
-
1009
- def _preprocess_audio(
1010
- self,
1011
- text: list[str],
1012
- audios: list[npt.NDArray],
1013
- ):
1014
- if len(audios) == 0:
1015
- return text, {}
1016
- assert self.audio_extractor is not None
1017
-
1018
- extractor = self.audio_extractor
1019
-
1020
- parts = [x for x in re.split(f"({re.escape(AUDIO_CONTEXT)})", text[0]) if x]
1021
- token_count = parts.count(AUDIO_CONTEXT)
1022
- if token_count != len(audios):
1023
- raise ValueError(
1024
- "Number of audio tokens in text does not match the number "
1025
- f"of audios (tokens={token_count}, audios={len(audios)})."
1026
- )
1027
- audio_index = 0
1028
- for idx, part in enumerate(parts):
1029
- if part == AUDIO_CONTEXT:
1030
- audio_repl = self.get_audio_repl(audios[audio_index])
1031
- parts[idx] = audio_repl.full
1032
- audio_index += 1
1033
- text = ["".join(parts)]
1034
- audio_inputs = extractor(
1035
- audios,
1036
- sampling_rate=extractor.sampling_rate,
1037
- return_tensors="pt",
1038
- )
1039
- input_audio_features = audio_inputs.input_features
1040
- feature_attention_mask = audio_inputs.attention_mask
1041
- audio_feature_lengths = feature_attention_mask.sum(dim=1)
1042
- audio_inputs = {
1043
- "input_audio_features": input_audio_features,
1044
- "feature_attention_mask": feature_attention_mask,
1045
- "audio_feature_lengths": audio_feature_lengths,
1046
- }
1047
-
1048
- return text, audio_inputs
1049
-
1050
- def __call__(
1051
- self,
1052
- text: str | list[str] | None = None,
1053
- images: Image.Image | list[Image.Image] | None = None,
1054
- videos: list[tuple[npt.NDArray, dict[str, Any]]] | None = None,
1055
- audios: AudioItem | list[AudioItem] | None = None,
1056
- return_tensors: str | TensorType | None = None,
1057
- max_num_tiles: int | None = None,
1058
- ) -> BatchFeature:
1059
- # Use default if not provided
1060
- if max_num_tiles is None:
1061
- max_num_tiles = self.max_num_tiles
1062
-
1063
- text, images, videos, audios = [
1064
- self._make_batch_input(x) for x in (text, images, videos, audios)
1065
- ]
1066
-
1067
- text, image_inputs = self._preprocess_image(
1068
- text=text,
1069
- images=images,
1070
- max_num_tiles=max_num_tiles,
1071
- )
1072
-
1073
- text, video_inputs = self._preprocess_video(
1074
- text=text,
1075
- videos=videos,
1076
- max_num_tiles=1,
1077
- )
1078
-
1079
- text, audio_inputs = self._preprocess_audio(
1080
- text=text,
1081
- audios=audios,
1082
- )
1083
-
1084
- text_inputs = self.tokenizer(text, add_special_tokens=False)
1085
-
1086
- combined_inputs = {**text_inputs, **video_inputs, **audio_inputs}
1087
-
1088
- if self.dynamic_tiler is None:
1089
- batch = BatchFeature(
1090
- {**combined_inputs, **image_inputs},
1091
- tensor_type=return_tensors,
1092
- )
1093
- else:
1094
- batch = BatchFeature(combined_inputs, tensor_type=return_tensors)
1095
- # allow images to be exempt from the BatchFeature validation:
1096
- # We will .stack() them in _parse_and_validate_image_input
1097
- batch.update(image_inputs)
1098
- return batch
1099
-
1100
- def get_image_repl(
1101
- self,
1102
- feature_size: int,
1103
- num_patches: int | None,
1104
- ) -> PromptUpdateDetails[str]:
1105
- repl_features = IMG_CONTEXT * feature_size
1106
- repl_full = IMG_START + repl_features + IMG_END
1107
-
1108
- return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
1109
-
1110
- def get_audio_repl(
1111
- self,
1112
- audio: npt.NDArray,
1113
- ) -> PromptUpdateDetails[str]:
1114
- assert self.audio_extractor is not None
1115
- num_tokens = self.audio_extractor.audio_token_count(len(audio))
1116
- repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
1117
- return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
1118
-
1119
- @classmethod
1120
- def get_video_repl(
1121
- cls,
1122
- *,
1123
- tokens_per_frame: list[int],
1124
- frames_indices: list[int],
1125
- frame_duration_ms: int,
1126
- tokenizer: TokenizerLike,
1127
- img_start_token_ids: list[int],
1128
- img_end_token_ids: list[int],
1129
- img_context_token_ids: list[int],
1130
- ) -> PromptUpdateDetails[list[int]]:
1131
- """
1132
- Build prompt replacement for a video.
1133
- The replacement returned is not actually used to replace the placeholder
1134
- tokens - it's just used to make sure we allocate the correct number
1135
- of tokens.
1136
- Actual replacement is done in embed_multimodal of
1137
- NemotronH_Nano_VL_V2
1138
- (specifically in _process_video_input -> _create_final_video_embeddings).
1139
- There, we create the final embeddings with text embeddings for indicator tokens
1140
- and video embeddings for video tokens.
1141
- This is a single function that handles all cases - non EVS, EVS dummy, EVS real.
1142
- The differentiation is done via tokens_per_frame parameter.
1143
- - non EVS case - constant value same value across all frames
1144
- - EVS dummy - Doesn't matter how tokens are distributed between frames - just
1145
- make sure the total number of tokens is correct.
1146
- - EVS real (called from get_real_video_repl_for_evs) - different value per frame
1147
- Args:
1148
- tokens_per_frame (list[int]): number of tokens per frame
1149
- frames_indices (list[int]): frame indices
1150
- frame_duration_ms (int): duration of each frame in milliseconds
1151
- tokenizer (TokenizerLike): tokenizer to use for tokenizing frame separators
1152
- img_start_token_ids (list[int]): pre-tokenized IMG_START tokens
1153
- img_end_token_ids (list[int]): pre-tokenized IMG_END tokens
1154
- img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens
1155
- """
1156
- # TODO: Add support of frame_duration_ms to be None
1157
- # At preprocessing step we should allow absent / metadata without
1158
- # frames_indices field.
1159
- timestamps_enabled = frame_duration_ms is not None
1160
-
1161
- if timestamps_enabled:
1162
- timestamps = calculate_timestamps(frames_indices, frame_duration_ms)
1163
-
1164
- assert len(timestamps) == len(tokens_per_frame), (
1165
- "timestamps and tokens_per_frame must have the same length"
1166
- )
1167
- frame_separators = [
1168
- f"Frame {i + 1} sampled at {timestamp:.2f} seconds: "
1169
- for i, timestamp in enumerate(timestamps)
1170
- ]
1171
- else:
1172
- frame_separators = [
1173
- f"Frame {i + 1}: " for i, _ in enumerate(tokens_per_frame)
1174
- ]
1175
-
1176
- # Tokenize frame separator independently
1177
- frame_separators_tokenized = [
1178
- _seq2tokens(tokenizer, sep) for sep in frame_separators
1179
- ]
1180
-
1181
- # Tokenize each component independently to avoid tokenizer merging tokens
1182
- # across boundaries. This ensures consistent tokenization regardless of
1183
- # num_tokens_per_frame values.
1184
- all_token_ids = []
1185
- for i, num_tokens in enumerate(tokens_per_frame):
1186
- frame_sep_token_ids = frame_separators_tokenized[i]
1187
- all_token_ids.extend(frame_sep_token_ids)
1188
-
1189
- # Add pre-tokenized special tokens
1190
- all_token_ids.extend(img_start_token_ids)
1191
- all_token_ids.extend(img_context_token_ids * num_tokens)
1192
- all_token_ids.extend(img_end_token_ids)
1193
-
1194
- return PromptUpdateDetails.from_seq(all_token_ids)
1195
-
1196
-
196
class BaseNanoNemotronVLProcessingInfo(BaseProcessingInfo):
197
"""Basic image-only ProcessingInfo for InternVL-style models."""
198
vllm/model_executor/models/nemotron_parse.py
+1
-232
@@ -11,18 +11,13 @@ import math
11
from collections.abc import Iterable, Mapping, Sequence
12
from typing import Annotated, Literal
13
14
-import numpy as np
14
import torch
15
import torch.nn as nn
16
from einops import rearrange
18
-from PIL import Image
19
-from timm.data.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
20
-from torchvision import transforms as T
17
from transformers import (
18
BartConfig,
19
BatchFeature,
20
PretrainedConfig,
25
- TensorType,
21
)
22
23
from vllm.config import CacheConfig, VllmConfig
@@ -59,13 +54,12 @@ from vllm.multimodal.processing import (
54
PromptUpdate,
55
)
56
from vllm.renderers import TokenizeParams
62
-from vllm.tokenizers import TokenizerLike
57
from vllm.transformers_utils.configs.radio import RadioConfig
58
+from vllm.transformers_utils.processors.nemotron_parse import NemotronParseProcessor
59
from vllm.utils.tensor_schema import TensorSchema, TensorShape
60
from vllm.v1.attention.backend import AttentionType
61
62
logger = init_logger(__name__)
68
-DEFAULT_FINAL_IMAGE_SIZE = (2048, 1648)
63
64
65
class BartScaledWordEmbedding(VocabParallelEmbedding):
@@ -372,231 +366,6 @@ class NemotronParsePixelInputs(TensorSchema):
366
data: Annotated[torch.Tensor, TensorShape("b", 3, "h", "w")]
367
368
375
-class NemotronParseImageProcessor:
376
- """
377
- NemotronParse Image Processor
378
- """
379
-
380
- def __init__(
381
- self,
382
- final_size: tuple = DEFAULT_FINAL_IMAGE_SIZE,
383
- **kwargs,
384
- ):
385
- # Ensure final_size is properly formatted
386
- if isinstance(final_size, (list, tuple)) and len(final_size) >= 2:
387
- self.final_size = (int(final_size[0]), int(final_size[1]))
388
- elif isinstance(final_size, (int, float)):
389
- self.final_size = (int(final_size), int(final_size))
390
- else:
391
- self.final_size = DEFAULT_FINAL_IMAGE_SIZE # Default fallback
392
-
393
- self.norm_mean = torch.Tensor(OPENAI_CLIP_MEAN).reshape(1, 3, 1, 1)
394
- self.norm_std = torch.Tensor(OPENAI_CLIP_STD).reshape(1, 3, 1, 1)
395
-
396
- # Create transforms
397
- self._create_transforms()
398
-
399
- def _create_transforms(self):
400
- """Create transform objects."""
401
- try:
402
- import albumentations as A
403
- except ImportError as err:
404
- raise ImportError(
405
- "The package `albumentations` is required to use "
406
- "NemotronParse model. Please install it with `pip install "
407
- "albumentations`."
408
- ) from err
409
-
410
- # Ensure final_size is a tuple of integers
411
- if isinstance(self.final_size, (list, tuple)):
412
- self.target_height, self.target_width = (
413
- int(self.final_size[0]),
414
- int(self.final_size[1]),
415
- )
416
- else:
417
- self.target_height = self.target_width = int(self.final_size)
418
-
419
- import cv2
420
-
421
- self.transform = A.Compose(
422
- [
423
- A.PadIfNeeded(
424
- min_height=self.target_height,
425
- min_width=self.target_width,
426
- border_mode=cv2.BORDER_CONSTANT,
427
- fill=[255, 255, 255],
428
- p=1.0,
429
- ),
430
- ]
431
- )
432
-
433
- self.torch_transform = T.Compose(
434
- [
435
- T.ToTensor(),
436
- ]
437
- )
438
-
439
- def _resize_with_aspect_ratio(self, image: np.ndarray) -> np.ndarray:
440
- """Resize image maintaining aspect ratio (exact replica of original
441
- LongestMaxSizeHW)."""
442
- height, width = image.shape[:2]
443
- max_size_height = self.target_height
444
- max_size_width = self.target_width
445
-
446
- # Original LongestMaxSizeHW algorithm from custom_augmentations.py
447
- aspect_ratio = width / height
448
- new_height = height
449
- new_width = width
450
-
451
- # If height too big then scale image down
452
- if height > max_size_height:
453
- new_height = max_size_height
454
- new_width = int(new_height * aspect_ratio)
455
-
456
- # If width too big, scale image down further
457
- if new_width > max_size_width:
458
- new_width = max_size_width
459
- new_height = int(new_width / aspect_ratio)
460
-
461
- # Use cv2.INTER_LINEAR like the original
462
- import cv2
463
-
464
- return cv2.resize(
465
- image, (new_width, new_height), interpolation=cv2.INTER_LINEAR
466
- )
467
-
468
- def _pad_to_size(self, image: np.ndarray) -> np.ndarray:
469
- """Pad image to target size with white padding (matches A.PadIfNeeded
470
- behavior)."""
471
- h, w = image.shape[:2]
472
- min_height, min_width = self.target_height, self.target_width
473
-
474
- # Only pad if image is smaller than target (matches A.PadIfNeeded logic)
475
- pad_h = max(0, min_height - h)
476
- pad_w = max(0, min_width - w)
477
-
478
- if pad_h == 0 and pad_w == 0:
479
- return image
480
-
481
- # A.PadIfNeeded pads to bottom-right with constant value
482
- if len(image.shape) == 3:
483
- # Color image - pad bottom and right with white (255, 255, 255)
484
- padded = np.pad(
485
- image,
486
- ((0, pad_h), (0, pad_w), (0, 0)),
487
- mode="constant",
488
- constant_values=255,
489
- )
490
- else:
491
- # Grayscale image - pad with white (255)
492
- padded = np.pad(
493
- image, ((0, pad_h), (0, pad_w)), mode="constant", constant_values=255
494
- )
495
-
496
- return padded
497
-
498
- def preprocess(
499
- self,
500
- images: Image.Image | list[Image.Image],
501
- **kwargs,
502
- ) -> dict[str, torch.Tensor]:
503
- """
504
- Preprocess an image or batch of images for the NemotronParse model.
505
-
506
- Args:
507
- images: Input image(s)
508
- """
509
- # Ensure images is a list
510
- if not isinstance(images, list):
511
- images = [images]
512
-
513
- # Convert PIL images to numpy arrays if needed
514
- processed_images = []
515
- for image in images:
516
- if isinstance(image, Image.Image):
517
- image = np.asarray(image)
518
- processed_images.append(image)
519
-
520
- # Apply NemotronParse-specific transforms
521
- pixel_values = []
522
- for image in processed_images:
523
- # Manual resize with aspect ratio preservation
524
- # (replaces LongestMaxSizeHW)
525
- processed_image = self._resize_with_aspect_ratio(image)
526
-
527
- # Apply remaining albumentations transforms if available
528
- if self.transform is not None:
529
- transformed = self.transform(image=processed_image)
530
- processed_image = transformed["image"]
531
- else:
532
- # Fallback: just pad to target size
533
- processed_image = self._pad_to_size(processed_image)
534
-
535
- # Convert to tensor
536
- pixel_values_tensor = self.torch_transform(processed_image)
537
-
538
- # Handle grayscale images
539
- if pixel_values_tensor.shape[0] == 1:
540
- pixel_values_tensor = pixel_values_tensor.expand(3, -1, -1)
541
-
542
- pixel_values.append(pixel_values_tensor)
543
-
544
- # Stack into batch
545
- pixel_values = torch.stack(pixel_values)
546
-
547
- # Normalize pixel values
548
- normalized_values = (pixel_values - self.norm_mean) / self.norm_std
549
- return {"pixel_values": normalized_values}
550
-
551
- def __call__(
552
- self, images: Image.Image | list[Image.Image], **kwargs
553
- ) -> dict[str, torch.Tensor]:
554
- return self.preprocess(images, **kwargs)
555
-
556
-
557
-class NemotronParseProcessor:
558
- """
559
- NemotronParse Processor
560
- """
561
-
562
- def __init__(
563
- self,
564
- config: PretrainedConfig,
565
- tokenizer: TokenizerLike,
566
- **kwargs,
567
- ) -> None:
568
- super().__init__()
569
-
570
- self.config = config
571
- self.tokenizer = tokenizer
572
-
573
- self.image_processor = NemotronParseImageProcessor(final_size=config.image_size)
574
-
575
- def _make_batch_input(self, input_item=None):
576
- if input_item is None:
577
- input_item = []
578
- if not isinstance(input_item, list):
579
- input_item = [input_item]
580
- return input_item
581
-
582
- def __call__(
583
- self,
584
- text: str | None = None,
585
- images: Image.Image | list[Image.Image] | None = None,
586
- return_tensors: str | TensorType | None = None,
587
- **kwargs,
588
- ) -> BatchFeature:
589
- text, images = [self._make_batch_input(x) for x in (text, images)]
590
- image_inputs = {} if len(images) == 0 else self.image_processor(images)
591
-
592
- text_inputs = self.tokenizer(text, add_special_tokens=False, **kwargs)
593
- combined_outputs = BatchFeature(
594
- data={**text_inputs, **image_inputs},
595
- tensor_type=return_tensors,
596
- )
597
- return combined_outputs
598
-
599
-
369
class NemotronParseProcessingInfo(BaseProcessingInfo):
370
def get_hf_config(self):
371
return self.ctx.get_hf_config()
vllm/model_executor/models/nemotron_vl.py
+4
-404
@@ -1,22 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
5
-# --------------------------------------------------------
6
-# InternVL
7
-# Copyright (c) 2023 OpenGVLab
8
-# Licensed under The MIT License [see LICENSE for details]
9
-# --------------------------------------------------------
3
import math
11
-from abc import ABC
4
from collections.abc import Iterable
5
6
import torch
7
import torch.nn as nn
16
-import torchvision.transforms as T
17
-from PIL import Image
8
from transformers import AutoModel, PretrainedConfig
19
-from transformers.image_processing_utils_fast import BaseImageProcessorFast
9
10
from vllm.config import VllmConfig
11
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -30,16 +19,16 @@ from vllm.model_executor.models.internvl import (
19
InternVLImageEmbeddingInputs,
20
InternVLImageInputs,
21
InternVLImagePixelInputs,
33
- InternVLProcessor,
22
)
23
from vllm.model_executor.models.module_mapping import MultiModelKeys
24
from vllm.model_executor.models.siglip import SiglipVisionModel
25
from vllm.multimodal import MULTIMODAL_REGISTRY
38
-from vllm.multimodal.image import convert_image_mode
39
-from vllm.multimodal.processing import PromptUpdateDetails
26
from vllm.sequence import IntermediateTensors
41
-from vllm.tokenizers import TokenizerLike
27
from vllm.transformers_utils.processor import cached_image_processor_from_config
28
+from vllm.transformers_utils.processors.nemotron_vl import (
29
+ LlamaNemotronVLEmbedProcessor,
30
+ NemotronVLProcessor,
31
+)
32
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
33
34
from .interfaces import (
@@ -58,310 +47,6 @@ from .utils import (
47
)
48
49
61
-def build_transform(input_size: int):
62
- return T.Compose(
63
- [
64
- T.Lambda(lambda img: convert_image_mode(img, "RGB")),
65
- T.Resize(
66
- (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
67
- ),
68
- T.ToTensor(),
69
- ]
70
- )
71
-
72
-
73
-# adapted from https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1
74
-def find_closest_aspect_ratio(
75
- aspect_ratio: float,
76
- target_ratios: list[tuple[int, int]],
77
- *,
78
- width: int,
79
- height: int,
80
- image_size: int,
81
-) -> tuple[int, int]:
82
- best_factor = float("-inf")
83
- best_ratio = (1, 1)
84
- area = width * height
85
-
86
- for rw, rh in target_ratios:
87
- target_aspect_ratio = rw / rh
88
- size_factor = min((rw * rh * image_size * image_size) / area, 0.6)
89
- ratio_closeness = min(
90
- target_aspect_ratio / aspect_ratio, aspect_ratio / target_aspect_ratio
91
- )
92
- factor = size_factor * ratio_closeness
93
-
94
- if factor > best_factor:
95
- best_factor = factor
96
- best_ratio = (rw, rh)
97
-
98
- return best_ratio
99
-
100
-
101
-def calculate_nemotron_vl_targets(
102
- *,
103
- orig_width: int,
104
- orig_height: int,
105
- target_ratios: list[tuple[int, int]],
106
- image_size: int,
107
- use_thumbnail: bool,
108
-) -> tuple[int, int, int]:
109
- aspect_ratio = orig_width / orig_height
110
-
111
- # find the closest aspect ratio to the target
112
- target_aspect_ratio = find_closest_aspect_ratio(
113
- aspect_ratio,
114
- target_ratios,
115
- width=orig_width,
116
- height=orig_height,
117
- image_size=image_size,
118
- )
119
-
120
- # calculate the target width and height
121
- target_width = image_size * target_aspect_ratio[0]
122
- target_height = image_size * target_aspect_ratio[1]
123
- blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
124
-
125
- # add thumbnail image if num_blocks != 1
126
- if use_thumbnail and blocks != 1:
127
- blocks += 1
128
-
129
- return blocks, target_width, target_height
130
-
131
-
132
-def dynamic_preprocess_nemotron_vl(
133
- image: Image.Image,
134
- *,
135
- target_ratios: list[tuple[int, int]],
136
- image_size: int,
137
- use_thumbnail: bool,
138
-) -> list[Image.Image]:
139
- orig_width, orig_height = image.size
140
-
141
- # calculate the number of blocks without thumbnail
142
- blocks, target_width, target_height = calculate_nemotron_vl_targets(
143
- orig_width=orig_width,
144
- orig_height=orig_height,
145
- target_ratios=target_ratios,
146
- image_size=image_size,
147
- use_thumbnail=False,
148
- )
149
-
150
- # resize the image
151
- resized_img = image.resize((target_width, target_height))
152
- processed_images = []
153
- for i in range(blocks):
154
- box = (
155
- (i % (target_width // image_size)) * image_size,
156
- (i // (target_width // image_size)) * image_size,
157
- ((i % (target_width // image_size)) + 1) * image_size,
158
- ((i // (target_width // image_size)) + 1) * image_size,
159
- )
160
- # split the image
161
- split_img = resized_img.crop(box)
162
- processed_images.append(split_img)
163
-
164
- assert len(processed_images) == blocks
165
-
166
- if use_thumbnail and len(processed_images) != 1:
167
- thumbnail_img = image.resize((image_size, image_size))
168
- processed_images.append(thumbnail_img)
169
-
170
- return processed_images
171
-
172
-
173
-def get_nemotron_vl_target_ratios(
174
- min_num: int,
175
- max_num: int,
176
-) -> list[tuple[int, int]]:
177
- target_ratios = {
178
- (i, j)
179
- for n in range(min_num, max_num + 1)
180
- for i in range(1, n + 1)
181
- for j in range(1, n + 1)
182
- if min_num <= i * j <= max_num
183
- }
184
- return sorted(target_ratios, key=lambda x: x[0] * x[1])
185
-
186
-
187
-def image_to_pixel_values_nemotron_vl(
188
- image: Image.Image,
189
- *,
190
- input_size: int,
191
- min_num: int,
192
- max_num: int,
193
- use_thumbnail: bool,
194
- transform: T.Compose | None = None,
195
-) -> torch.Tensor:
196
- target_ratios = get_nemotron_vl_target_ratios(min_num, max_num)
197
-
198
- if transform is None:
199
- transform = build_transform(input_size=input_size)
200
-
201
- images = dynamic_preprocess_nemotron_vl(
202
- image,
203
- target_ratios=target_ratios,
204
- image_size=input_size,
205
- use_thumbnail=use_thumbnail,
206
- )
207
-
208
- pixel_values = torch.stack([transform(image) for image in images])
209
- return pixel_values
210
-
211
-
212
-class NemotronVLProcessor(InternVLProcessor):
213
- IMG_START = "<img>"
214
- IMG_END = "</img>"
215
- IMG_CONTEXT = "<image>"
216
-
217
- def __init__(
218
- self,
219
- config: PretrainedConfig,
220
- tokenizer: TokenizerLike,
221
- image_processor: BaseImageProcessorFast | None = None,
222
- *,
223
- min_dynamic_patch: int | None = None,
224
- max_dynamic_patch: int | None = None,
225
- dynamic_image_size: bool | None = None,
226
- ) -> None:
227
- ABC.__init__(self)
228
- self.config = config
229
- self.tokenizer = tokenizer
230
- self.image_processor = image_processor
231
- image_size: int = config.force_image_size
232
- patch_size: int = config.patch_size
233
-
234
- if min_dynamic_patch is None:
235
- min_dynamic_patch = 1
236
- assert isinstance(min_dynamic_patch, int)
237
-
238
- if max_dynamic_patch is None:
239
- max_dynamic_patch = self.image_processor.max_num_tiles
240
- assert isinstance(max_dynamic_patch, int)
241
-
242
- if dynamic_image_size is None:
243
- dynamic_image_size = True
244
- assert isinstance(dynamic_image_size, bool)
245
-
246
- self.num_image_token = int(
247
- (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
248
- )
249
- self.image_size = image_size
250
- self.min_dynamic_patch = min_dynamic_patch
251
- self.max_dynamic_patch = max_dynamic_patch
252
- self.dynamic_image_size = dynamic_image_size
253
-
254
- if image_processor is not None:
255
- self.use_thumbnail = image_processor.use_thumbnail
256
- else:
257
- self.use_thumbnail = getattr(config, "use_thumbnail", True)
258
-
259
- @property
260
- def image_token_id(self) -> int:
261
- return self.tokenizer.get_vocab()[self.IMG_CONTEXT]
262
-
263
- def _get_transform(self) -> T.Compose:
264
- return build_transform(input_size=self.image_size)
265
-
266
- def get_num_image_tokens(
267
- self,
268
- *,
269
- image_width: int,
270
- image_height: int,
271
- ) -> int:
272
- target_ratios = self.resolve_target_ratios(
273
- use_thumbnail=False, # Applied in calculate_targets
274
- )
275
-
276
- num_patches, _, _ = calculate_nemotron_vl_targets(
277
- orig_width=image_width,
278
- orig_height=image_height,
279
- image_size=self.image_size,
280
- target_ratios=target_ratios,
281
- use_thumbnail=self.use_thumbnail,
282
- )
283
-
284
- return num_patches * self.num_image_token
285
-
286
- def _images_to_pixel_values_lst(
287
- self,
288
- images: list[Image.Image],
289
- min_dynamic_patch: int | None = None,
290
- max_dynamic_patch: int | None = None,
291
- dynamic_image_size: bool | None = None,
292
- ) -> list[torch.Tensor]:
293
- min_num, max_num = self.resolve_min_max_num(
294
- min_dynamic_patch=min_dynamic_patch,
295
- max_dynamic_patch=max_dynamic_patch,
296
- dynamic_image_size=dynamic_image_size,
297
- use_thumbnail=False, # Applied in image_to_pixel_values
298
- )
299
-
300
- return [
301
- image_to_pixel_values_nemotron_vl(
302
- image,
303
- input_size=self.image_size,
304
- min_num=min_num,
305
- max_num=max_num,
306
- use_thumbnail=self.use_thumbnail,
307
- transform=self._get_transform(),
308
- )
309
- for image in images
310
- ]
311
-
312
- def _replace_image_tokens(
313
- self,
314
- text: list[str],
315
- pixel_values_lst: list[torch.Tensor],
316
- ) -> list[str]:
317
- """Replace <image> placeholders with image tokens."""
318
- for pixel_values in pixel_values_lst:
319
- num_patches = pixel_values.shape[0]
320
- feature_size = num_patches * self.num_image_token
321
- image_repl = self.get_image_repl(feature_size, num_patches)
322
- # Use temporary placeholder to avoid replacing tokens we just inserted
323
- NVL_IMAGE_CONTEXT = image_repl.full.replace("<image>", "<NVL_IMG_CONTEXT>")
324
- text = [t.replace("<image>", NVL_IMAGE_CONTEXT, 1) for t in text]
325
- return [t.replace("<NVL_IMG_CONTEXT>", self.IMG_CONTEXT) for t in text]
326
-
327
- def _preprocess_image(
328
- self,
329
- text: list[str],
330
- images: list[Image.Image],
331
- min_dynamic_patch: int | None = None,
332
- max_dynamic_patch: int | None = None,
333
- dynamic_image_size: bool | None = None,
334
- ) -> tuple[list[str], dict[str, torch.Tensor]]:
335
- if len(images) == 0:
336
- image_inputs = {}
337
- else:
338
- pixel_values_lst = self._images_to_pixel_values_lst(
339
- images,
340
- min_dynamic_patch=min_dynamic_patch,
341
- max_dynamic_patch=max_dynamic_patch,
342
- dynamic_image_size=dynamic_image_size,
343
- )
344
- image_inputs = {
345
- "pixel_values_flat": torch.cat(pixel_values_lst),
346
- "image_num_patches": torch.tensor(
347
- [len(item) for item in pixel_values_lst]
348
- ),
349
- }
350
-
351
- text = self._replace_image_tokens(text, pixel_values_lst)
352
- return text, image_inputs
353
-
354
- def get_image_repl(
355
- self,
356
- feature_size: int,
357
- num_patches: int | None,
358
- ) -> PromptUpdateDetails[str]:
359
- repl_features = self.IMG_CONTEXT * feature_size
360
- repl_full = self.IMG_START + repl_features + self.IMG_END
361
-
362
- return PromptUpdateDetails.select_text(repl_full, self.IMG_CONTEXT)
363
-
364
-
50
class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
51
"""Processing info for Nemotron VL models."""
52
@@ -700,91 +385,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
385
# - Pooler output instead of generative logits
386
# --------------------------------------------------------
387
703
-# SigLIP normalization constants
704
-SIGLIP_MEAN = (0.5, 0.5, 0.5)
705
-SIGLIP_STD = (0.5, 0.5, 0.5)
706
-
707
-
708
-def build_siglip_transform(input_size: int):
709
- """Build transform for SigLIP vision encoder with normalization.
710
-
711
- Extends the base transform from nemotron_vl with SigLIP-specific normalization.
712
- """
713
- base_transform = build_transform(input_size=input_size)
714
- return T.Compose(
715
- [
716
- base_transform,
717
- T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD),
718
- ]
719
- )
720
-
721
-
722
-class LlamaNemotronVLEmbedProcessor(NemotronVLProcessor):
723
- """
724
- Processor for LlamaNemotronVL embedding model.
725
-
726
- Inherits from NemotronVLProcessor and specializes it for embedding tasks:
727
- - Uses SigLIP transform with normalization instead of base transform
728
- - Uses different image context token (<IMG_CONTEXT> vs <image>)
729
- """
730
-
731
- IMG_CONTEXT = "<IMG_CONTEXT>"
732
-
733
- def __init__(
734
- self,
735
- config: PretrainedConfig,
736
- tokenizer: TokenizerLike,
737
- processor_config: dict,
738
- *,
739
- min_dynamic_patch: int | None = None,
740
- max_dynamic_patch: int | None = None,
741
- dynamic_image_size: bool | None = None,
742
- ) -> None:
743
- if min_dynamic_patch is None:
744
- min_dynamic_patch = processor_config.get(
745
- "min_input_tiles",
746
- getattr(config, "min_dynamic_patch", 1),
747
- )
748
- if max_dynamic_patch is None:
749
- max_dynamic_patch = processor_config.get(
750
- "max_input_tiles",
751
- getattr(config, "max_dynamic_patch", 1),
752
- )
753
- if dynamic_image_size is None:
754
- dynamic_image_size = processor_config.get(
755
- "dynamic_image_size",
756
- getattr(config, "dynamic_image_size", True),
757
- )
758
- super().__init__(
759
- config=config,
760
- tokenizer=tokenizer,
761
- image_processor=None,
762
- min_dynamic_patch=min_dynamic_patch,
763
- max_dynamic_patch=max_dynamic_patch,
764
- dynamic_image_size=dynamic_image_size,
765
- )
766
-
767
- def _get_transform(self) -> T.Compose:
768
- """Override to add SigLIP normalization."""
769
- return build_siglip_transform(input_size=self.image_size)
770
-
771
- def _replace_image_tokens(
772
- self,
773
- text: list[str],
774
- pixel_values_lst: list[torch.Tensor],
775
- ) -> list[str]:
776
- """Override with simpler token replacement for embedding model.
777
-
778
- No temporary placeholder needed because IMG_CONTEXT is <IMG_CONTEXT>,
779
- not <image>, so there's no collision risk.
780
- """
781
- for pixel_values in pixel_values_lst:
782
- num_patches = pixel_values.shape[0]
783
- feature_size = num_patches * self.num_image_token
784
- image_repl = self.get_image_repl(feature_size, num_patches)
785
- text = [t.replace("<image>", image_repl.full, 1) for t in text]
786
- return text
787
-
388
389
class LlamaNemotronVLEmbedProcessingInfo(NemotronVLProcessingInfo):
390
"""Processing info for LlamaNemotronVL embedding model."""
vllm/model_executor/models/nvlm_d.py
+1
-33
@@ -27,48 +27,16 @@ from vllm.multimodal.processing import (
27
PromptUpdate,
28
PromptUpdateDetails,
29
)
30
+from vllm.transformers_utils.processors.nvlm_d import IMG_PAD, NVLMProcessor
31
32
from .intern_vit import InternVisionModel
33
from .internvl import (
34
BaseInternVLDummyInputsBuilder,
35
BaseInternVLMultiModalProcessor,
36
BaseInternVLProcessingInfo,
36
- BaseInternVLProcessor,
37
InternVLChatModel,
38
)
39
40
-IMG_PAD = "<|vision_pad|>"
41
-
42
-
43
-class NVLMProcessor(BaseInternVLProcessor):
44
- @property
45
- def image_token_id(self) -> int:
46
- return self.tokenizer.get_vocab()[IMG_PAD]
47
-
48
- def get_image_repl(
49
- self,
50
- feature_size: int,
51
- num_patches: int | None,
52
- ) -> PromptUpdateDetails[str]:
53
- if num_patches is None:
54
- raise NotImplementedError("Embedding inputs are not supported")
55
-
56
- tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
57
- if self.use_thumbnail:
58
- tile_pos_identifiers += ["<tile_global_thumbnail>"]
59
-
60
- context_size = feature_size // num_patches
61
- features = "".join(
62
- identifier + IMG_PAD * context_size for identifier in tile_pos_identifiers
63
- )
64
-
65
- # We include the start and end as well because "<Image><tile" is
66
- # tokenized as ["<Image", "><", "tile"], resulting in assertion error
67
- # when trying to find "<tile" as a subsequence of "<Image><tile"
68
- repl = "<Image>" + features + "</Image>"
69
-
70
- return PromptUpdateDetails.select_text(repl, IMG_PAD)
71
-
40
41
class NVLMProcessingInfo(BaseInternVLProcessingInfo):
42
def get_hf_processor(self, **kwargs: object) -> NVLMProcessor:
vllm/model_executor/models/skyworkr1v.py
+2
-377
@@ -12,9 +12,7 @@ from typing import Annotated, Literal, TypeAlias
12
13
import torch
14
import torch.nn as nn
15
-import torchvision.transforms as T
16
-from PIL import Image
17
-from transformers import BatchFeature, PretrainedConfig, TensorType
15
+from transformers import BatchFeature, PretrainedConfig
16
17
from vllm.config import VllmConfig
18
from vllm.config.multimodal import BaseDummyOptions
@@ -26,7 +24,6 @@ from vllm.model_executor.models.intern_vit import (
24
InternVisionPatchModel,
25
)
26
from vllm.multimodal import MULTIMODAL_REGISTRY
29
-from vllm.multimodal.image import convert_image_mode
27
from vllm.multimodal.inputs import (
28
MultiModalDataDict,
29
MultiModalFieldConfig,
@@ -44,22 +41,14 @@ from vllm.multimodal.processing import (
41
BaseProcessingInfo,
42
PromptReplacement,
43
PromptUpdate,
47
- PromptUpdateDetails,
44
)
45
from vllm.sequence import IntermediateTensors
50
-from vllm.tokenizers import TokenizerLike
46
+from vllm.transformers_utils.processors.skyworkr1v import SkyworkR1VProcessor
47
from vllm.utils.tensor_schema import TensorSchema, TensorShape
48
49
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
50
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
51
56
-IMG_START = "<img>"
57
-IMG_END = "</img>"
58
-IMG_CONTEXT = "<IMG_CONTEXT>"
59
-
60
-IMAGENET_MEAN = (0.485, 0.456, 0.406)
61
-IMAGENET_STD = (0.229, 0.224, 0.225)
62
-
52
53
class SkyworkR1VImagePixelInputs(TensorSchema):
54
"""
@@ -106,370 +95,6 @@ SkyworkR1VImageInputs: TypeAlias = (
95
)
96
97
109
-# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
110
-def build_transform(input_size: int):
111
- MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
112
- return T.Compose(
113
- [
114
- T.Lambda(lambda img: convert_image_mode(img, "RGB")),
115
- T.Resize(
116
- (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
117
- ),
118
- T.ToTensor(),
119
- T.Normalize(mean=MEAN, std=STD),
120
- ]
121
- )
122
-
123
-
124
-# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
125
-def find_closest_aspect_ratio(
126
- aspect_ratio: float,
127
- target_ratios: list[tuple[int, int]],
128
- *,
129
- width: int,
130
- height: int,
131
- image_size: int,
132
-) -> tuple[int, int]:
133
- best_ratio_diff = float("inf")
134
- best_ratio = (1, 1)
135
- area = width * height
136
- for ratio in target_ratios:
137
- target_aspect_ratio = ratio[0] / ratio[1]
138
- ratio_diff = abs(aspect_ratio - target_aspect_ratio)
139
- if ratio_diff < best_ratio_diff:
140
- best_ratio_diff = ratio_diff
141
- best_ratio = ratio
142
- elif ratio_diff == best_ratio_diff:
143
- if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
144
- best_ratio = ratio
145
- return best_ratio
146
-
147
-
148
-def resolve_skyworkr1v_min_max_num(
149
- *,
150
- min_dynamic_patch: int,
151
- max_dynamic_patch: int,
152
- dynamic_image_size: bool,
153
- use_thumbnail: bool,
154
-) -> tuple[int, int]:
155
- min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
156
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
157
-
158
- if use_thumbnail and max_dynamic_patch != 1:
159
- max_dynamic_patch += 1
160
-
161
- return min_dynamic_patch, max_dynamic_patch
162
-
163
-
164
-def get_skyworkr1v_target_ratios(
165
- min_num: int,
166
- max_num: int,
167
-) -> list[tuple[int, int]]:
168
- target_ratios = {
169
- (i, j)
170
- for n in range(min_num, max_num + 1)
171
- for i in range(1, n + 1)
172
- for j in range(1, n + 1)
173
- if min_num <= i * j <= max_num
174
- }
175
- return sorted(target_ratios, key=lambda x: x[0] * x[1])
176
-
177
-
178
-def calculate_skyworkr1v_targets(
179
- *,
180
- orig_width: int,
181
- orig_height: int,
182
- target_ratios: list[tuple[int, int]],
183
- image_size: int,
184
- use_thumbnail: bool,
185
-) -> tuple[int, int, int]:
186
- aspect_ratio = orig_width / orig_height
187
-
188
- # find the closest aspect ratio to the target
189
- target_aspect_ratio = find_closest_aspect_ratio(
190
- aspect_ratio,
191
- target_ratios,
192
- width=orig_width,
193
- height=orig_height,
194
- image_size=image_size,
195
- )
196
-
197
- # calculate the target width and height
198
- target_width = image_size * target_aspect_ratio[0]
199
- target_height = image_size * target_aspect_ratio[1]
200
- blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
201
-
202
- # add thumbnail image if num_blocks != 1
203
- if use_thumbnail and blocks != 1:
204
- blocks += 1
205
-
206
- return blocks, target_width, target_height
207
-
208
-
209
-def dynamic_preprocess_skyworkr1v(
210
- image: Image.Image,
211
- *,
212
- target_ratios: list[tuple[int, int]],
213
- image_size: int,
214
- use_thumbnail: bool,
215
-) -> list[Image.Image]:
216
- orig_width, orig_height = image.size
217
-
218
- # calculate the number of blocks without thumbnail
219
- blocks, target_width, target_height = calculate_skyworkr1v_targets(
220
- orig_width=orig_width,
221
- orig_height=orig_height,
222
- target_ratios=target_ratios,
223
- image_size=image_size,
224
- use_thumbnail=False,
225
- )
226
-
227
- # resize the image
228
- resized_img = image.resize((target_width, target_height))
229
- processed_images = []
230
- for i in range(blocks):
231
- box = (
232
- (i % (target_width // image_size)) * image_size,
233
- (i // (target_width // image_size)) * image_size,
234
- ((i % (target_width // image_size)) + 1) * image_size,
235
- ((i // (target_width // image_size)) + 1) * image_size,
236
- )
237
- # split the image
238
- split_img = resized_img.crop(box)
239
- processed_images.append(split_img)
240
-
241
- assert len(processed_images) == blocks
242
-
243
- if use_thumbnail and len(processed_images) != 1:
244
- thumbnail_img = image.resize((image_size, image_size))
245
- processed_images.append(thumbnail_img)
246
-
247
- return processed_images
248
-
249
-
250
-# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B
251
-def image_to_pixel_values_skyworkr1v(
252
- image: Image.Image,
253
- *,
254
- input_size: int,
255
- min_num: int,
256
- max_num: int,
257
- use_thumbnail: bool,
258
-) -> torch.Tensor:
259
- target_ratios = get_skyworkr1v_target_ratios(min_num, max_num)
260
-
261
- transform = build_transform(input_size=input_size)
262
- images = dynamic_preprocess_skyworkr1v(
263
- image,
264
- target_ratios=target_ratios,
265
- image_size=input_size,
266
- use_thumbnail=use_thumbnail,
267
- )
268
-
269
- pixel_values = torch.stack([transform(image) for image in images])
270
- return pixel_values
271
-
272
-
273
-class SkyworkR1VProcessor:
274
- """
275
- This model doesn't define its own HF processor,
276
- so we implement our own one here.
277
-
278
- The code to insert image tokens is based on:
279
- https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py#L252
280
- """
281
-
282
- def __init__(
283
- self,
284
- config: PretrainedConfig,
285
- tokenizer: TokenizerLike,
286
- *,
287
- min_dynamic_patch: int | None = None,
288
- max_dynamic_patch: int | None = None,
289
- dynamic_image_size: bool | None = None,
290
- ) -> None:
291
- super().__init__()
292
-
293
- self.config = config
294
- self.tokenizer = tokenizer
295
-
296
- image_size: int = config.vision_config.image_size
297
- patch_size: int = config.vision_config.patch_size
298
-
299
- if min_dynamic_patch is None:
300
- min_dynamic_patch = config.min_dynamic_patch
301
- assert isinstance(min_dynamic_patch, int)
302
-
303
- if max_dynamic_patch is None:
304
- max_dynamic_patch = config.max_dynamic_patch
305
- assert isinstance(max_dynamic_patch, int)
306
-
307
- if dynamic_image_size is None:
308
- dynamic_image_size = config.dynamic_image_size
309
- assert isinstance(dynamic_image_size, bool)
310
-
311
- self.num_image_token = int(
312
- (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
313
- )
314
- self.image_size = image_size
315
- self.min_dynamic_patch = min_dynamic_patch
316
- self.max_dynamic_patch = max_dynamic_patch
317
- self.dynamic_image_size = dynamic_image_size
318
- self.use_thumbnail: bool = config.use_thumbnail
319
-
320
- @property
321
- def image_token_id(self) -> int:
322
- return self.tokenizer.get_vocab()[IMG_CONTEXT]
323
-
324
- def get_image_repl(
325
- self,
326
- feature_size: int,
327
- num_patches: int | None,
328
- ) -> PromptUpdateDetails[str]:
329
- repl_features = IMG_CONTEXT * feature_size
330
- repl_full = IMG_START + repl_features + IMG_END
331
-
332
- return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
333
-
334
- def resolve_min_max_num(
335
- self,
336
- *,
337
- min_dynamic_patch: int | None = None,
338
- max_dynamic_patch: int | None = None,
339
- dynamic_image_size: bool | None = None,
340
- use_thumbnail: bool | None = None,
341
- ) -> tuple[int, int]:
342
- min_dynamic_patch = (
343
- self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
344
- )
345
- max_dynamic_patch = (
346
- self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
347
- )
348
- dynamic_image_size = (
349
- self.dynamic_image_size
350
- if dynamic_image_size is None
351
- else dynamic_image_size
352
- )
353
- use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
354
-
355
- return resolve_skyworkr1v_min_max_num(
356
- min_dynamic_patch=min_dynamic_patch,
357
- max_dynamic_patch=max_dynamic_patch,
358
- dynamic_image_size=dynamic_image_size,
359
- use_thumbnail=use_thumbnail,
360
- )
361
-
362
- def resolve_target_ratios(
363
- self,
364
- *,
365
- min_dynamic_patch: int | None = None,
366
- max_dynamic_patch: int | None = None,
367
- dynamic_image_size: bool | None = None,
368
- use_thumbnail: bool | None = None,
369
- ) -> list[tuple[int, int]]:
370
- min_num, max_num = self.resolve_min_max_num(
371
- min_dynamic_patch=min_dynamic_patch,
372
- max_dynamic_patch=max_dynamic_patch,
373
- dynamic_image_size=dynamic_image_size,
374
- use_thumbnail=use_thumbnail,
375
- )
376
-
377
- return get_skyworkr1v_target_ratios(min_num, max_num)
378
-
379
- def get_num_image_tokens(
380
- self,
381
- *,
382
- image_width: int,
383
- image_height: int,
384
- ) -> int:
385
- target_ratios = self.resolve_target_ratios(
386
- use_thumbnail=False, # Applied in calculate_targets
387
- )
388
-
389
- num_patches, _, _ = calculate_skyworkr1v_targets(
390
- orig_width=image_width,
391
- orig_height=image_height,
392
- image_size=self.image_size,
393
- target_ratios=target_ratios,
394
- use_thumbnail=self.use_thumbnail,
395
- )
396
-
397
- return num_patches * self.num_image_token
398
-
399
- def _images_to_pixel_values_lst(
400
- self,
401
- images: list[Image.Image],
402
- min_dynamic_patch: int | None = None,
403
- max_dynamic_patch: int | None = None,
404
- dynamic_image_size: bool | None = None,
405
- ) -> list[torch.Tensor]:
406
- min_num, max_num = self.resolve_min_max_num(
407
- min_dynamic_patch=min_dynamic_patch,
408
- max_dynamic_patch=max_dynamic_patch,
409
- dynamic_image_size=dynamic_image_size,
410
- use_thumbnail=False, # Applied in image_to_pixel_values
411
- )
412
-
413
- return [
414
- image_to_pixel_values_skyworkr1v(
415
- image,
416
- input_size=self.image_size,
417
- min_num=min_num,
418
- max_num=max_num,
419
- use_thumbnail=self.use_thumbnail,
420
- )
421
- for image in images
422
- ]
423
-
424
- def __call__(
425
- self,
426
- text: str | list[str] | None = None,
427
- images: Image.Image | list[Image.Image] | None = None,
428
- min_dynamic_patch: int | None = None,
429
- max_dynamic_patch: int | None = None,
430
- dynamic_image_size: bool | None = None,
431
- return_tensors: str | TensorType | None = None,
432
- ) -> BatchFeature:
433
- if text is None:
434
- text = []
435
- if not isinstance(text, list):
436
- text = [text]
437
- if images is None:
438
- images = []
439
- if not isinstance(images, list):
440
- images = [images]
441
-
442
- if len(images) == 0:
443
- image_inputs = {}
444
- else:
445
- pixel_values_lst = self._images_to_pixel_values_lst(
446
- images,
447
- min_dynamic_patch=min_dynamic_patch,
448
- max_dynamic_patch=max_dynamic_patch,
449
- dynamic_image_size=dynamic_image_size,
450
- )
451
- image_inputs = {
452
- "pixel_values_flat": torch.cat(pixel_values_lst),
453
- "image_num_patches": torch.tensor(
454
- [len(item) for item in pixel_values_lst]
455
- ),
456
- }
457
-
458
- for pixel_values in pixel_values_lst:
459
- num_patches = pixel_values.shape[0]
460
- feature_size = num_patches * self.num_image_token
461
-
462
- image_repl = self.get_image_repl(feature_size, num_patches)
463
-
464
- text = [t.replace("<image>", image_repl.full, 1) for t in text]
465
-
466
- text_inputs = self.tokenizer(text)
467
-
468
- combined_outputs = {**text_inputs, **image_inputs}
469
-
470
- return BatchFeature(combined_outputs, tensor_type=return_tensors)
471
-
472
-
98
class SkyworkR1VProcessingInfo(BaseProcessingInfo):
99
def get_hf_processor(self, **kwargs: object) -> SkyworkR1VProcessor:
100
return self.ctx.init_processor(
vllm/transformers_utils/processors/__init__.py
+18
@@ -13,35 +13,53 @@ import importlib
13
__all__ = [
14
"BagelProcessor",
15
"DeepseekVLV2Processor",
16
+ "Eagle2_5_VLProcessor",
17
"FireRedASR2Processor",
18
"FunASRProcessor",
19
"GLM4VProcessor",
20
+ "H2OVLProcessor",
21
"HunYuanVLProcessor",
22
"HunYuanVLImageProcessor",
23
+ "InternVLProcessor",
24
"KimiAudioProcessor",
25
"MistralCommonPixtralProcessor",
26
"MistralCommonVoxtralProcessor",
27
+ "NanoNemotronVLProcessor",
28
+ "NemotronParseProcessor",
29
+ "NemotronVLProcessor",
30
+ "LlamaNemotronVLEmbedProcessor",
31
+ "NVLMProcessor",
32
"OvisProcessor",
33
"Ovis2_5Processor",
34
"QwenVLProcessor",
35
"Qwen3ASRProcessor",
36
+ "SkyworkR1VProcessor",
37
]
38
39
_CLASS_TO_MODULE: dict[str, str] = {
40
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
41
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
42
+ "Eagle2_5_VLProcessor": "vllm.transformers_utils.processors.eagle2_5_vl",
43
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
44
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
45
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
46
+ "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
47
"HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
48
"HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
49
+ "InternVLProcessor": "vllm.transformers_utils.processors.internvl",
50
"KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
51
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
52
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
53
+ "NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
54
+ "NemotronParseProcessor": "vllm.transformers_utils.processors.nemotron_parse",
55
+ "NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
56
+ "LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
57
+ "NVLMProcessor": "vllm.transformers_utils.processors.nvlm_d",
58
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
59
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
60
"QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
61
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
62
+ "SkyworkR1VProcessor": "vllm.transformers_utils.processors.skyworkr1v",
63
}
64
65
vllm/transformers_utils/processors/eagle2_5_vl.py
new
+85
@@ -0,0 +1,85 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# Adapted from NVIDIA Eagle2.5-VL model
4
+# https://huggingface.co/nvidia/Eagle2.5-8B
5
+from transformers import PretrainedConfig
6
+
7
+from vllm.multimodal.processing import PromptUpdateDetails
8
+from vllm.tokenizers import TokenizerLike
9
+
10
+from .internvl import IMG_CONTEXT, IMG_END, IMG_START, BaseInternVLProcessor
11
+
12
+
13
+class Eagle2_5_VLProcessor(BaseInternVLProcessor):
14
+ """
15
+ Custom processor for Eagle2.5-VL model.
16
+ Extends BaseInternVLProcessor with Eagle-specific token handling.
17
+ """
18
+
19
+ def __init__(
20
+ self,
21
+ config: PretrainedConfig,
22
+ tokenizer: TokenizerLike,
23
+ *,
24
+ min_dynamic_patch: int | None = None,
25
+ max_dynamic_patch: int | None = None,
26
+ dynamic_image_size: bool | None = None,
27
+ ) -> None:
28
+ # Skip super().__init__() to avoid config manipulation
29
+ # Directly initialize all required attributes
30
+ self.config = config
31
+ self.tokenizer = tokenizer
32
+
33
+ # Image size with force_image_size override
34
+ image_size: int = config.vision_config.image_size
35
+ if hasattr(config, "force_image_size") and config.force_image_size:
36
+ image_size = config.force_image_size
37
+
38
+ patch_size: int = config.vision_config.patch_size
39
+ downsample_ratio: float = getattr(config, "downsample_ratio", 0.5)
40
+
41
+ # Compute num_image_token
42
+ self.num_image_token = int(
43
+ (image_size // patch_size) ** 2 * (downsample_ratio**2)
44
+ )
45
+ self.image_size = image_size
46
+
47
+ # Dynamic patch settings with defaults
48
+ self.min_dynamic_patch = (
49
+ min_dynamic_patch
50
+ if min_dynamic_patch is not None
51
+ else getattr(config, "min_dynamic_patch", 1)
52
+ )
53
+ self.max_dynamic_patch = (
54
+ max_dynamic_patch
55
+ if max_dynamic_patch is not None
56
+ else getattr(config, "max_dynamic_patch", 12)
57
+ )
58
+ self.dynamic_image_size = (
59
+ dynamic_image_size
60
+ if dynamic_image_size is not None
61
+ else getattr(config, "dynamic_image_size", True)
62
+ )
63
+ self.use_thumbnail: bool = getattr(config, "use_thumbnail", True)
64
+
65
+ @property
66
+ def image_token_id(self) -> int:
67
+ """Get the image token ID from config or tokenizer."""
68
+ if hasattr(self.config, "image_token_index"):
69
+ return self.config.image_token_index
70
+ # Fallback to tokenizer vocab - use <IMG_CONTEXT> (ID: 151667)
71
+ vocab = self.tokenizer.get_vocab()
72
+ if IMG_CONTEXT in vocab:
73
+ return vocab[IMG_CONTEXT]
74
+ raise ValueError(f"Cannot find image token '{IMG_CONTEXT}' in vocabulary")
75
+
76
+ def get_image_repl(
77
+ self,
78
+ feature_size: int,
79
+ num_patches: int | None,
80
+ ) -> PromptUpdateDetails[str]:
81
+ """Get image replacement string for prompt."""
82
+ repl_features = IMG_CONTEXT * feature_size
83
+ repl_full = IMG_START + repl_features + IMG_END
84
+
85
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
vllm/transformers_utils/processors/h2ovl.py
new
+390
@@ -0,0 +1,390 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py
5
+# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py
6
+# --------------------------------------------------------
7
+# H2OVL-Mississippi
8
+# Copyright (c) 2024 H2O.AI
9
+# Licensed under Apache 2.0 License [see LICENSE for details]
10
+# --------------------------------------------------------
11
+import torch
12
+from PIL import Image
13
+from transformers import PretrainedConfig
14
+
15
+from vllm.multimodal.processing import PromptUpdateDetails
16
+from vllm.tokenizers import TokenizerLike
17
+
18
+from .internvl import (
19
+ IMG_CONTEXT,
20
+ IMG_END,
21
+ IMG_START,
22
+ BaseInternVLProcessor,
23
+ build_transform,
24
+ find_closest_aspect_ratio,
25
+ get_internvl_target_ratios,
26
+)
27
+
28
+
29
+def resolve_h2ovl_min_max_num(
30
+ *,
31
+ min_dynamic_patch: int,
32
+ max_dynamic_patch: int,
33
+ dynamic_image_size: bool,
34
+ use_thumbnail: bool,
35
+) -> tuple[int, int]:
36
+ min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
37
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
38
+
39
+ if use_thumbnail and max_dynamic_patch != 1:
40
+ max_dynamic_patch += 1
41
+
42
+ return min_dynamic_patch, max_dynamic_patch
43
+
44
+
45
+def get_h2ovl_target_ratios(
46
+ min_num: int,
47
+ max_num: int,
48
+ *,
49
+ prior_aspect_ratio: tuple[int, int] | None,
50
+) -> list[tuple[int, int]]:
51
+ target_ratios = get_internvl_target_ratios(min_num, max_num)
52
+
53
+ # if prior_aspect_ratio is provided, filter the target ratios
54
+ if prior_aspect_ratio is not None:
55
+ target_ratios = [
56
+ ratio
57
+ for ratio in target_ratios
58
+ if prior_aspect_ratio[0] % ratio[0] != 0
59
+ and prior_aspect_ratio[1] % ratio[1] != 0
60
+ ]
61
+
62
+ return target_ratios
63
+
64
+
65
+# modified to include blocks generated in second pass
66
+def calculate_h2ovl_targets(
67
+ *,
68
+ orig_width: int,
69
+ orig_height: int,
70
+ target_ratios: list[tuple[int, int]],
71
+ image_size: int,
72
+ use_thumbnail: bool,
73
+) -> tuple[int, int, int, tuple[int, int]]:
74
+ aspect_ratio = orig_width / orig_height
75
+
76
+ # find the closest aspect ratio to the target
77
+ target_aspect_ratio = find_closest_aspect_ratio(
78
+ aspect_ratio,
79
+ target_ratios,
80
+ width=orig_width,
81
+ height=orig_height,
82
+ image_size=image_size,
83
+ )
84
+
85
+ # calculate the target width and height
86
+ target_width = image_size * target_aspect_ratio[0]
87
+ target_height = image_size * target_aspect_ratio[1]
88
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
89
+
90
+ # add thumbnail image if num_blocks != 1
91
+ if use_thumbnail and blocks != 1:
92
+ blocks += 1
93
+
94
+ return blocks, target_width, target_height, target_aspect_ratio
95
+
96
+
97
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
98
+# refactored to handle prior_aspect_ratio
99
+def dynamic_preprocess_h2ovl(
100
+ image: Image.Image,
101
+ *,
102
+ target_ratios: list[tuple[int, int]],
103
+ image_size: int,
104
+ use_thumbnail: bool,
105
+) -> tuple[list[Image.Image], tuple[int, int]]:
106
+ orig_width, orig_height = image.size
107
+
108
+ # calculate the number of blocks without thumbnail
109
+ (
110
+ blocks,
111
+ target_width,
112
+ target_height,
113
+ target_aspect_ratio,
114
+ ) = calculate_h2ovl_targets(
115
+ orig_width=orig_width,
116
+ orig_height=orig_height,
117
+ target_ratios=target_ratios,
118
+ image_size=image_size,
119
+ use_thumbnail=False,
120
+ )
121
+
122
+ # resize the image
123
+ resized_img = image.resize((target_width, target_height))
124
+ processed_images = []
125
+ for i in range(blocks):
126
+ box = (
127
+ (i % (target_width // image_size)) * image_size,
128
+ (i // (target_width // image_size)) * image_size,
129
+ ((i % (target_width // image_size)) + 1) * image_size,
130
+ ((i // (target_width // image_size)) + 1) * image_size,
131
+ )
132
+ # split the image
133
+ split_img = resized_img.crop(box)
134
+ processed_images.append(split_img)
135
+
136
+ assert len(processed_images) == blocks
137
+
138
+ if use_thumbnail and len(processed_images) != 1:
139
+ thumbnail_img = image.resize((image_size, image_size))
140
+ processed_images.append(thumbnail_img)
141
+
142
+ return processed_images, target_aspect_ratio
143
+
144
+
145
+def _preprocess_image(
146
+ image: Image.Image,
147
+ *,
148
+ input_size: int,
149
+ min_num: int,
150
+ max_num: int,
151
+ use_thumbnail: bool,
152
+ prior_aspect_ratio: tuple[int, int] | None,
153
+) -> tuple[torch.Tensor, tuple[int, int]]:
154
+ target_ratios = get_h2ovl_target_ratios(
155
+ min_num,
156
+ max_num,
157
+ prior_aspect_ratio=prior_aspect_ratio,
158
+ )
159
+
160
+ transform = build_transform(input_size=input_size)
161
+ images, target_aspect_ratio = dynamic_preprocess_h2ovl(
162
+ image,
163
+ image_size=input_size,
164
+ use_thumbnail=use_thumbnail,
165
+ target_ratios=target_ratios,
166
+ )
167
+
168
+ pixel_values = torch.stack([transform(image) for image in images])
169
+ return pixel_values, target_aspect_ratio
170
+
171
+
172
+# refactored to use the _preprocess_image function
173
+def image_to_pixel_values_h2ovl(
174
+ image: Image.Image,
175
+ *,
176
+ input_size: int,
177
+ min_num: int,
178
+ max_num: int,
179
+ use_thumbnail: bool,
180
+ use_msac: bool,
181
+) -> torch.Tensor:
182
+ # when MSAC is turned on, we need to process the image twice
183
+ if use_msac:
184
+ # first pass
185
+ pixel_values1, aspect_ratio1 = _preprocess_image(
186
+ image,
187
+ input_size=input_size,
188
+ min_num=1,
189
+ max_num=max_num,
190
+ use_thumbnail=True,
191
+ prior_aspect_ratio=None,
192
+ )
193
+ # second pass
194
+ pixel_values2, _ = _preprocess_image(
195
+ image,
196
+ input_size=input_size,
197
+ min_num=3,
198
+ max_num=max_num,
199
+ use_thumbnail=True,
200
+ prior_aspect_ratio=aspect_ratio1,
201
+ )
202
+ # combine pixel values
203
+ pixel_values = torch.cat(
204
+ [pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0
205
+ )
206
+
207
+ else:
208
+ pixel_values, _ = _preprocess_image(
209
+ image,
210
+ input_size=input_size,
211
+ min_num=min_num,
212
+ max_num=max_num,
213
+ use_thumbnail=use_thumbnail,
214
+ prior_aspect_ratio=None,
215
+ )
216
+
217
+ return pixel_values
218
+
219
+
220
+class H2OVLProcessor(BaseInternVLProcessor):
221
+ def __init__(
222
+ self,
223
+ config: PretrainedConfig,
224
+ tokenizer: TokenizerLike,
225
+ *,
226
+ min_dynamic_patch: int | None = None,
227
+ max_dynamic_patch: int | None = None,
228
+ dynamic_image_size: bool | None = None,
229
+ use_msac: bool | None = None,
230
+ ) -> None:
231
+ super().__init__(
232
+ config,
233
+ tokenizer,
234
+ min_dynamic_patch=min_dynamic_patch,
235
+ max_dynamic_patch=max_dynamic_patch,
236
+ dynamic_image_size=dynamic_image_size,
237
+ )
238
+
239
+ if use_msac is None:
240
+ use_msac = config.use_msac
241
+ assert isinstance(use_msac, bool)
242
+
243
+ self.use_msac = use_msac
244
+
245
+ @property
246
+ def image_token_id(self) -> int:
247
+ return self.tokenizer.get_vocab()[IMG_CONTEXT]
248
+
249
+ def get_image_repl(
250
+ self,
251
+ feature_size: int,
252
+ num_patches: int | None,
253
+ ) -> PromptUpdateDetails[str]:
254
+ repl_features = IMG_CONTEXT * feature_size
255
+ repl_full = IMG_START + repl_features + IMG_END
256
+
257
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
258
+
259
+ def resolve_min_max_num(
260
+ self,
261
+ *,
262
+ min_dynamic_patch: int | None = None,
263
+ max_dynamic_patch: int | None = None,
264
+ dynamic_image_size: bool | None = None,
265
+ use_thumbnail: bool | None = None,
266
+ ) -> tuple[int, int]:
267
+ min_dynamic_patch = (
268
+ self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
269
+ )
270
+ max_dynamic_patch = (
271
+ self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
272
+ )
273
+ dynamic_image_size = (
274
+ self.dynamic_image_size
275
+ if dynamic_image_size is None
276
+ else dynamic_image_size
277
+ )
278
+ use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
279
+
280
+ return resolve_h2ovl_min_max_num(
281
+ min_dynamic_patch=min_dynamic_patch,
282
+ max_dynamic_patch=max_dynamic_patch,
283
+ dynamic_image_size=dynamic_image_size,
284
+ use_thumbnail=use_thumbnail,
285
+ )
286
+
287
+ def resolve_target_ratios(
288
+ self,
289
+ *,
290
+ min_dynamic_patch: int | None = None,
291
+ max_dynamic_patch: int | None = None,
292
+ dynamic_image_size: bool | None = None,
293
+ use_thumbnail: bool | None = None,
294
+ prior_aspect_ratio: tuple[int, int] | None = None,
295
+ override_min_num: int | None = None,
296
+ ) -> list[tuple[int, int]]:
297
+ min_num, max_num = self.resolve_min_max_num(
298
+ min_dynamic_patch=min_dynamic_patch,
299
+ max_dynamic_patch=max_dynamic_patch,
300
+ dynamic_image_size=dynamic_image_size,
301
+ use_thumbnail=use_thumbnail,
302
+ )
303
+ if override_min_num is not None:
304
+ min_num = override_min_num
305
+
306
+ return get_h2ovl_target_ratios(
307
+ min_num,
308
+ max_num,
309
+ prior_aspect_ratio=prior_aspect_ratio,
310
+ )
311
+
312
+ def get_num_image_tokens(
313
+ self,
314
+ *,
315
+ image_width: int,
316
+ image_height: int,
317
+ use_msac: bool | None = None,
318
+ ) -> int:
319
+ use_msac = self.use_msac if use_msac is None else use_msac
320
+
321
+ use_thumbnail = self.use_thumbnail
322
+
323
+ if use_msac:
324
+ target_ratios_1 = self.resolve_target_ratios(
325
+ use_thumbnail=False, # Applied in calculate_targets
326
+ override_min_num=1,
327
+ )
328
+ num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
329
+ orig_width=image_width,
330
+ orig_height=image_height,
331
+ image_size=self.image_size,
332
+ target_ratios=target_ratios_1,
333
+ use_thumbnail=True,
334
+ )
335
+
336
+ target_ratios_2 = self.resolve_target_ratios(
337
+ use_thumbnail=False, # Applied in calculate_targets
338
+ prior_aspect_ratio=aspect_ratio_1,
339
+ override_min_num=3,
340
+ )
341
+ num_patches_2, _, _, _ = calculate_h2ovl_targets(
342
+ orig_width=image_width,
343
+ orig_height=image_height,
344
+ image_size=self.image_size,
345
+ target_ratios=target_ratios_2,
346
+ use_thumbnail=True,
347
+ )
348
+
349
+ num_patches = num_patches_1 + num_patches_2 - 1
350
+ else:
351
+ target_ratios = self.resolve_target_ratios(
352
+ use_thumbnail=False, # Applied in calculate_targets
353
+ )
354
+ num_patches, _, _, _ = calculate_h2ovl_targets(
355
+ orig_width=image_width,
356
+ orig_height=image_height,
357
+ image_size=self.image_size,
358
+ target_ratios=target_ratios,
359
+ use_thumbnail=use_thumbnail,
360
+ )
361
+
362
+ return num_patches * self.num_image_token
363
+
364
+ def _images_to_pixel_values_lst(
365
+ self,
366
+ images: list[Image.Image],
367
+ min_dynamic_patch: int | None = None,
368
+ max_dynamic_patch: int | None = None,
369
+ dynamic_image_size: bool | None = None,
370
+ ) -> list[torch.Tensor]:
371
+ use_msac = self.use_msac if len(images) == 1 else False
372
+
373
+ min_num, max_num = self.resolve_min_max_num(
374
+ min_dynamic_patch=min_dynamic_patch,
375
+ max_dynamic_patch=max_dynamic_patch,
376
+ dynamic_image_size=dynamic_image_size,
377
+ use_thumbnail=False, # Applied in image_to_pixel_values
378
+ )
379
+
380
+ return [
381
+ image_to_pixel_values_h2ovl(
382
+ image,
383
+ input_size=self.image_size,
384
+ min_num=min_num,
385
+ max_num=max_num,
386
+ use_thumbnail=self.use_thumbnail,
387
+ use_msac=use_msac,
388
+ )
389
+ for image in images
390
+ ]
vllm/transformers_utils/processors/internvl.py
new
+603
@@ -0,0 +1,603 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
5
+# --------------------------------------------------------
6
+# InternVL
7
+# Copyright (c) 2023 OpenGVLab
8
+# Licensed under The MIT License [see LICENSE for details]
9
+# --------------------------------------------------------
10
+from abc import ABC, abstractmethod
11
+from typing import Any, TypeVar
12
+
13
+import numpy.typing as npt
14
+import torch
15
+import torchvision.transforms as T
16
+from PIL import Image
17
+from transformers import BatchFeature, PretrainedConfig, TensorType
18
+
19
+from vllm.multimodal.image import convert_image_mode
20
+from vllm.multimodal.processing import PromptUpdateDetails
21
+from vllm.tokenizers import TokenizerLike
22
+
23
+_T = TypeVar("_T")
24
+
25
+IMG_START = "<img>"
26
+IMG_END = "</img>"
27
+IMG_CONTEXT = "<IMG_CONTEXT>"
28
+
29
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
30
+IMAGENET_STD = (0.229, 0.224, 0.225)
31
+
32
+
33
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
34
+def build_transform(input_size: int):
35
+ MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
36
+ transform = T.Compose(
37
+ [
38
+ T.Lambda(lambda img: convert_image_mode(img, "RGB")),
39
+ T.Resize(
40
+ (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
41
+ ),
42
+ T.ToTensor(),
43
+ T.Normalize(mean=MEAN, std=STD),
44
+ ]
45
+ )
46
+ return transform
47
+
48
+
49
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
50
+def find_closest_aspect_ratio(
51
+ aspect_ratio: float,
52
+ target_ratios: list[tuple[int, int]],
53
+ *,
54
+ width: int,
55
+ height: int,
56
+ image_size: int,
57
+) -> tuple[int, int]:
58
+ best_ratio_diff = float("inf")
59
+ best_ratio = (1, 1)
60
+ area = width * height
61
+ for ratio in target_ratios:
62
+ target_aspect_ratio = ratio[0] / ratio[1]
63
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
64
+ if ratio_diff < best_ratio_diff:
65
+ best_ratio_diff = ratio_diff
66
+ best_ratio = ratio
67
+ elif ratio_diff == best_ratio_diff:
68
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
69
+ best_ratio = ratio
70
+ return best_ratio
71
+
72
+
73
+def resolve_internvl_min_max_num(
74
+ *,
75
+ min_dynamic_patch: int,
76
+ max_dynamic_patch: int,
77
+ dynamic_image_size: bool,
78
+ use_thumbnail: bool,
79
+) -> tuple[int, int]:
80
+ min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
81
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
82
+
83
+ if use_thumbnail and max_dynamic_patch != 1:
84
+ max_dynamic_patch += 1
85
+
86
+ return min_dynamic_patch, max_dynamic_patch
87
+
88
+
89
+def get_internvl_target_ratios(
90
+ min_num: int,
91
+ max_num: int,
92
+) -> list[tuple[int, int]]:
93
+ target_ratios = {
94
+ (i, j)
95
+ for n in range(min_num, max_num + 1)
96
+ for i in range(1, n + 1)
97
+ for j in range(1, n + 1)
98
+ if min_num <= i * j <= max_num
99
+ }
100
+ return sorted(target_ratios, key=lambda x: x[0] * x[1])
101
+
102
+
103
+def calculate_internvl_targets(
104
+ *,
105
+ orig_width: int,
106
+ orig_height: int,
107
+ target_ratios: list[tuple[int, int]],
108
+ image_size: int,
109
+ use_thumbnail: bool,
110
+) -> tuple[int, int, int]:
111
+ aspect_ratio = orig_width / orig_height
112
+
113
+ # find the closest aspect ratio to the target
114
+ target_aspect_ratio = find_closest_aspect_ratio(
115
+ aspect_ratio,
116
+ target_ratios,
117
+ width=orig_width,
118
+ height=orig_height,
119
+ image_size=image_size,
120
+ )
121
+
122
+ # calculate the target width and height
123
+ target_width = image_size * target_aspect_ratio[0]
124
+ target_height = image_size * target_aspect_ratio[1]
125
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
126
+
127
+ # add thumbnail image if num_blocks != 1
128
+ if use_thumbnail and blocks != 1:
129
+ blocks += 1
130
+
131
+ return blocks, target_width, target_height
132
+
133
+
134
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
135
+def dynamic_preprocess_internvl(
136
+ image: Image.Image,
137
+ *,
138
+ target_ratios: list[tuple[int, int]],
139
+ image_size: int,
140
+ use_thumbnail: bool,
141
+) -> list[Image.Image]:
142
+ orig_width, orig_height = image.size
143
+
144
+ # calculate the number of blocks without thumbnail
145
+ blocks, target_width, target_height = calculate_internvl_targets(
146
+ orig_width=orig_width,
147
+ orig_height=orig_height,
148
+ target_ratios=target_ratios,
149
+ image_size=image_size,
150
+ use_thumbnail=False,
151
+ )
152
+
153
+ # resize the image
154
+ resized_img = image.resize((target_width, target_height))
155
+ processed_images = []
156
+ for i in range(blocks):
157
+ box = (
158
+ (i % (target_width // image_size)) * image_size,
159
+ (i // (target_width // image_size)) * image_size,
160
+ ((i % (target_width // image_size)) + 1) * image_size,
161
+ ((i // (target_width // image_size)) + 1) * image_size,
162
+ )
163
+ # split the image
164
+ split_img = resized_img.crop(box)
165
+ processed_images.append(split_img)
166
+
167
+ assert len(processed_images) == blocks
168
+
169
+ if use_thumbnail and len(processed_images) != 1:
170
+ thumbnail_img = image.resize((image_size, image_size))
171
+ processed_images.append(thumbnail_img)
172
+
173
+ return processed_images
174
+
175
+
176
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
177
+def image_to_pixel_values_internvl(
178
+ image: Image.Image,
179
+ *,
180
+ input_size: int,
181
+ min_num: int,
182
+ max_num: int,
183
+ use_thumbnail: bool,
184
+) -> torch.Tensor:
185
+ target_ratios = get_internvl_target_ratios(min_num, max_num)
186
+
187
+ transform = build_transform(input_size=input_size)
188
+ images = dynamic_preprocess_internvl(
189
+ image,
190
+ target_ratios=target_ratios,
191
+ image_size=input_size,
192
+ use_thumbnail=use_thumbnail,
193
+ )
194
+
195
+ pixel_values = torch.stack([transform(image) for image in images])
196
+ return pixel_values
197
+
198
+
199
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
200
+def video_to_pixel_values_internvl(
201
+ video: npt.NDArray,
202
+ *,
203
+ input_size: int,
204
+ min_num: int,
205
+ max_num: int,
206
+ use_thumbnail: bool,
207
+) -> torch.Tensor:
208
+ target_ratios = get_internvl_target_ratios(min_num, max_num)
209
+
210
+ transform = build_transform(input_size=input_size)
211
+ frames_list = list[Image.Image]()
212
+ for frame in video:
213
+ pil_frame = dynamic_preprocess_internvl(
214
+ Image.fromarray(frame, mode="RGB"),
215
+ target_ratios=target_ratios,
216
+ image_size=input_size,
217
+ use_thumbnail=use_thumbnail,
218
+ )
219
+ assert len(pil_frame) == 1
220
+ frames_list.extend(pil_frame)
221
+
222
+ pixel_values = torch.stack([transform(image) for image in frames_list])
223
+ return pixel_values
224
+
225
+
226
+class BaseInternVLProcessor(ABC):
227
+ """
228
+ This model doesn't define its own HF processor,
229
+ so we implement our own one here.
230
+
231
+ The code to insert image tokens is based on:
232
+ https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
233
+ """
234
+
235
+ def __init__(
236
+ self,
237
+ config: PretrainedConfig,
238
+ tokenizer: TokenizerLike,
239
+ *,
240
+ min_dynamic_patch: int | None = None,
241
+ max_dynamic_patch: int | None = None,
242
+ dynamic_image_size: bool | None = None,
243
+ ) -> None:
244
+ super().__init__()
245
+
246
+ self.config = config
247
+ self.tokenizer = tokenizer
248
+
249
+ image_size: int = config.vision_config.image_size
250
+ patch_size: int = config.vision_config.patch_size
251
+
252
+ if min_dynamic_patch is None:
253
+ min_dynamic_patch = config.min_dynamic_patch
254
+ assert isinstance(min_dynamic_patch, int)
255
+
256
+ if max_dynamic_patch is None:
257
+ max_dynamic_patch = config.max_dynamic_patch
258
+ assert isinstance(max_dynamic_patch, int)
259
+
260
+ if dynamic_image_size is None:
261
+ dynamic_image_size = config.dynamic_image_size
262
+ assert isinstance(dynamic_image_size, bool)
263
+
264
+ self.num_image_token = int(
265
+ (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
266
+ )
267
+ self.image_size = image_size
268
+ self.min_dynamic_patch = min_dynamic_patch
269
+ self.max_dynamic_patch = max_dynamic_patch
270
+ self.dynamic_image_size = dynamic_image_size
271
+ self.use_thumbnail: bool = config.use_thumbnail
272
+
273
+ @property
274
+ @abstractmethod
275
+ def image_token_id(self) -> int:
276
+ raise NotImplementedError
277
+
278
+ @abstractmethod
279
+ def get_image_repl(
280
+ self,
281
+ feature_size: int,
282
+ num_patches: int | None,
283
+ ) -> PromptUpdateDetails[str]:
284
+ raise NotImplementedError
285
+
286
+ def resolve_min_max_num(
287
+ self,
288
+ *,
289
+ min_dynamic_patch: int | None = None,
290
+ max_dynamic_patch: int | None = None,
291
+ dynamic_image_size: bool | None = None,
292
+ use_thumbnail: bool | None = None,
293
+ ) -> tuple[int, int]:
294
+ min_dynamic_patch = (
295
+ self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
296
+ )
297
+ max_dynamic_patch = (
298
+ self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
299
+ )
300
+ dynamic_image_size = (
301
+ self.dynamic_image_size
302
+ if dynamic_image_size is None
303
+ else dynamic_image_size
304
+ )
305
+ use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
306
+
307
+ return resolve_internvl_min_max_num(
308
+ min_dynamic_patch=min_dynamic_patch,
309
+ max_dynamic_patch=max_dynamic_patch,
310
+ dynamic_image_size=dynamic_image_size,
311
+ use_thumbnail=use_thumbnail,
312
+ )
313
+
314
+ def resolve_target_ratios(
315
+ self,
316
+ *,
317
+ min_dynamic_patch: int | None = None,
318
+ max_dynamic_patch: int | None = None,
319
+ dynamic_image_size: bool | None = None,
320
+ use_thumbnail: bool | None = None,
321
+ ) -> list[tuple[int, int]]:
322
+ min_num, max_num = self.resolve_min_max_num(
323
+ min_dynamic_patch=min_dynamic_patch,
324
+ max_dynamic_patch=max_dynamic_patch,
325
+ dynamic_image_size=dynamic_image_size,
326
+ use_thumbnail=use_thumbnail,
327
+ )
328
+
329
+ return get_internvl_target_ratios(min_num, max_num)
330
+
331
+ def get_num_image_tokens(
332
+ self,
333
+ *,
334
+ image_width: int,
335
+ image_height: int,
336
+ ) -> int:
337
+ target_ratios = self.resolve_target_ratios(
338
+ use_thumbnail=False, # Applied in calculate_targets
339
+ )
340
+
341
+ num_patches, _, _ = calculate_internvl_targets(
342
+ orig_width=image_width,
343
+ orig_height=image_height,
344
+ image_size=self.image_size,
345
+ target_ratios=target_ratios,
346
+ use_thumbnail=self.use_thumbnail,
347
+ )
348
+
349
+ return num_patches * self.num_image_token
350
+
351
+ def _images_to_pixel_values_lst(
352
+ self,
353
+ images: list[Image.Image],
354
+ min_dynamic_patch: int | None = None,
355
+ max_dynamic_patch: int | None = None,
356
+ dynamic_image_size: bool | None = None,
357
+ ) -> list[torch.Tensor]:
358
+ min_num, max_num = self.resolve_min_max_num(
359
+ min_dynamic_patch=min_dynamic_patch,
360
+ max_dynamic_patch=max_dynamic_patch,
361
+ dynamic_image_size=dynamic_image_size,
362
+ use_thumbnail=False, # Applied in image_to_pixel_values
363
+ )
364
+
365
+ return [
366
+ image_to_pixel_values_internvl(
367
+ image,
368
+ input_size=self.image_size,
369
+ min_num=min_num,
370
+ max_num=max_num,
371
+ use_thumbnail=self.use_thumbnail,
372
+ )
373
+ for image in images
374
+ ]
375
+
376
+ def _preprocess_image(
377
+ self,
378
+ text: list[str],
379
+ images: list[Image.Image],
380
+ min_dynamic_patch: int | None = None,
381
+ max_dynamic_patch: int | None = None,
382
+ dynamic_image_size: bool | None = None,
383
+ ) -> tuple[list[str], dict[str, torch.Tensor]]:
384
+ if len(images) == 0:
385
+ image_inputs = {}
386
+ else:
387
+ pixel_values_lst = self._images_to_pixel_values_lst(
388
+ images,
389
+ min_dynamic_patch=min_dynamic_patch,
390
+ max_dynamic_patch=max_dynamic_patch,
391
+ dynamic_image_size=dynamic_image_size,
392
+ )
393
+ image_inputs = {
394
+ "pixel_values_flat": torch.cat(pixel_values_lst),
395
+ "image_num_patches": torch.tensor(
396
+ [len(item) for item in pixel_values_lst]
397
+ ),
398
+ }
399
+
400
+ for pixel_values in pixel_values_lst:
401
+ num_patches = pixel_values.shape[0]
402
+ feature_size = num_patches * self.num_image_token
403
+
404
+ image_repl = self.get_image_repl(feature_size, num_patches)
405
+ text = [t.replace("<image>", image_repl.full, 1) for t in text]
406
+ return text, image_inputs
407
+
408
+ def _make_batch_input(self, input_item: _T | list[_T] | None = None) -> list[_T]:
409
+ if input_item is None:
410
+ input_item = []
411
+ if not isinstance(input_item, list):
412
+ input_item = [input_item]
413
+ return input_item
414
+
415
+ def __call__(
416
+ self,
417
+ text: str | list[str] | None = None,
418
+ images: Image.Image | list[Image.Image] | None = None,
419
+ *,
420
+ min_dynamic_patch: int | None = None,
421
+ max_dynamic_patch: int | None = None,
422
+ dynamic_image_size: bool | None = None,
423
+ return_tensors: str | TensorType | None = None,
424
+ **kwargs,
425
+ ) -> BatchFeature:
426
+ text = self._make_batch_input(text)
427
+ images = self._make_batch_input(images)
428
+
429
+ text, image_inputs = self._preprocess_image(
430
+ text=text,
431
+ images=images,
432
+ min_dynamic_patch=min_dynamic_patch,
433
+ max_dynamic_patch=max_dynamic_patch,
434
+ dynamic_image_size=dynamic_image_size,
435
+ )
436
+
437
+ text_inputs = self.tokenizer(text)
438
+
439
+ combined_outputs = {**text_inputs, **image_inputs}
440
+
441
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
442
+
443
+
444
+class InternVLProcessor(BaseInternVLProcessor):
445
+ """
446
+ HF Processor for InternVLChatModel with extended video processing logic.
447
+
448
+ Code for video processing is adapted from video example:
449
+ https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
450
+ """
451
+
452
+ def __init__(
453
+ self,
454
+ config: PretrainedConfig,
455
+ tokenizer: TokenizerLike,
456
+ *,
457
+ min_dynamic_patch: int | None = None,
458
+ max_dynamic_patch: int | None = None,
459
+ dynamic_image_size: bool | None = None,
460
+ video_token: str | None = None,
461
+ ) -> None:
462
+ super().__init__(
463
+ config=config,
464
+ tokenizer=tokenizer,
465
+ min_dynamic_patch=min_dynamic_patch,
466
+ max_dynamic_patch=max_dynamic_patch,
467
+ dynamic_image_size=dynamic_image_size,
468
+ )
469
+ # add extra video token for video processing
470
+ self.video_token = video_token
471
+
472
+ @property
473
+ def image_token_id(self) -> int:
474
+ return self.tokenizer.get_vocab()[IMG_CONTEXT]
475
+
476
+ @property
477
+ def video_token_id(self) -> int | None:
478
+ if self.video_token is None:
479
+ return None
480
+ return self.tokenizer.get_vocab().get(self.video_token, None)
481
+
482
+ @property
483
+ def supports_video(self) -> bool:
484
+ return self.video_token_id is not None
485
+
486
+ def _videos_to_pixel_values_lst(
487
+ self,
488
+ videos: list[npt.NDArray],
489
+ dynamic_image_size: bool | None = None,
490
+ ) -> list[torch.Tensor]:
491
+ min_num, max_num = self.resolve_min_max_num(
492
+ min_dynamic_patch=1,
493
+ max_dynamic_patch=1,
494
+ dynamic_image_size=dynamic_image_size,
495
+ use_thumbnail=False, # Applied in image_to_pixel_values
496
+ )
497
+
498
+ return [
499
+ video_to_pixel_values_internvl(
500
+ video,
501
+ input_size=self.image_size,
502
+ min_num=min_num,
503
+ max_num=max_num,
504
+ use_thumbnail=False,
505
+ )
506
+ for video in videos
507
+ ]
508
+
509
+ def _preprocess_video(
510
+ self,
511
+ text: list[str],
512
+ videos: list[npt.NDArray],
513
+ dynamic_image_size: bool | None = None,
514
+ ) -> tuple[list[str], dict[str, Any]]:
515
+ if len(videos) == 0 or not self.supports_video:
516
+ return text, {}
517
+
518
+ video_token = self.video_token
519
+ assert video_token is not None
520
+
521
+ pixel_values_lst_video = self._videos_to_pixel_values_lst(
522
+ videos,
523
+ dynamic_image_size=dynamic_image_size,
524
+ )
525
+ video_inputs = {
526
+ "pixel_values_flat_video": torch.cat(pixel_values_lst_video),
527
+ "video_num_patches": torch.tensor(
528
+ [len(item) for item in pixel_values_lst_video]
529
+ ),
530
+ }
531
+
532
+ for pixel_values in pixel_values_lst_video:
533
+ num_patches = pixel_values.shape[0]
534
+
535
+ video_repl = self.get_video_repl(
536
+ self.num_image_token, num_patches, video_token
537
+ )
538
+ text = [t.replace("<video>", video_repl.full, 1) for t in text]
539
+ return text, video_inputs
540
+
541
+ def __call__(
542
+ self,
543
+ text: str | list[str] | None = None,
544
+ images: Image.Image | list[Image.Image] | None = None,
545
+ videos: npt.NDArray | list[npt.NDArray] | None = None,
546
+ *,
547
+ min_dynamic_patch: int | None = None,
548
+ max_dynamic_patch: int | None = None,
549
+ dynamic_image_size: bool | None = None,
550
+ return_tensors: str | TensorType | None = None,
551
+ **kwargs,
552
+ ) -> BatchFeature:
553
+ text = self._make_batch_input(text)
554
+ images = self._make_batch_input(images)
555
+ videos = self._make_batch_input(videos)
556
+
557
+ text, image_inputs = self._preprocess_image(
558
+ text=text,
559
+ images=images,
560
+ min_dynamic_patch=min_dynamic_patch,
561
+ max_dynamic_patch=max_dynamic_patch,
562
+ dynamic_image_size=dynamic_image_size,
563
+ )
564
+
565
+ text, video_inputs = self._preprocess_video(
566
+ text=text,
567
+ videos=videos,
568
+ dynamic_image_size=dynamic_image_size,
569
+ )
570
+
571
+ text_inputs = self.tokenizer(text)
572
+
573
+ combined_outputs = {**text_inputs, **image_inputs, **video_inputs}
574
+
575
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)
576
+
577
+ def get_image_repl(
578
+ self,
579
+ feature_size: int,
580
+ num_patches: int | None,
581
+ ) -> PromptUpdateDetails[str]:
582
+ repl_features = IMG_CONTEXT * feature_size
583
+ repl_full = IMG_START + repl_features + IMG_END
584
+
585
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
586
+
587
+ def get_video_repl(
588
+ self,
589
+ feature_size: int,
590
+ num_patches: int | None,
591
+ video_context_token: str = IMG_CONTEXT,
592
+ ) -> PromptUpdateDetails[str]:
593
+ if num_patches is None:
594
+ raise NotImplementedError("Embedding inputs are not supported")
595
+
596
+ repl_features = video_context_token * self.num_image_token
597
+ repl_features_with_sep = IMG_START + repl_features + IMG_END
598
+ # num_patches is equal to num_frames
599
+ repl_full = "".join(
600
+ [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
601
+ )
602
+
603
+ return PromptUpdateDetails.select_text(repl_full, video_context_token)
vllm/transformers_utils/processors/nano_nemotron_vl.py
new
+1032
@@ -0,0 +1,1032 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# --------------------------------------------------------
4
+# Adapted from
5
+# https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/internvl.py
6
+# under Apache-2.0 License
7
+# LICENSE is in root directory.
8
+# --------------------------------------------------------
9
+
10
+import math
11
+from abc import ABC, abstractmethod
12
+from collections.abc import Sequence
13
+from dataclasses import dataclass
14
+from typing import Any, TypeVar
15
+
16
+import einops
17
+import numpy as np
18
+import numpy.typing as npt
19
+import regex as re
20
+import torch
21
+from PIL import Image
22
+from transformers import BatchFeature, PretrainedConfig, TensorType
23
+
24
+from vllm.model_executor.models.parakeet import ParakeetExtractor
25
+from vllm.multimodal.evs import compute_retained_tokens_count
26
+from vllm.multimodal.inputs import AudioItem
27
+from vllm.multimodal.processing.processor import PromptUpdateDetails, _seq2tokens
28
+from vllm.tokenizers import TokenizerLike
29
+
30
+from .internvl import calculate_internvl_targets, get_internvl_target_ratios
31
+
32
+_T = TypeVar("_T")
33
+
34
+
35
+IMG_START = "<img>"
36
+IMG_END = "</img>"
37
+IMG_CONTEXT = "<image>"
38
+AUDIO_START = "<so_start>"
39
+AUDIO_END = "<so_end>"
40
+AUDIO_CONTEXT = "<so_embedding>"
41
+
42
+# Profiling
43
+# MAX_FRAMES = 16
44
+DEFAULT_NUM_TILES = 12
45
+
46
+
47
+def calculate_timestamps(
48
+ indices: list[int] | torch.Tensor,
49
+ frame_duration_ms: int,
50
+):
51
+ if not isinstance(indices, list):
52
+ indices = indices.tolist()
53
+
54
+ timestamps = [int(i) * frame_duration_ms / 1000.0 for i in indices]
55
+ return timestamps
56
+
57
+
58
+def input_conditioner(x: torch.Tensor, norm_mean: torch.Tensor, norm_std: torch.Tensor):
59
+ return (x - norm_mean) / norm_std
60
+
61
+
62
+def dynamic_preprocess(
63
+ image,
64
+ *,
65
+ image_size=512,
66
+ max_num_tiles=12,
67
+ use_thumbnail=True,
68
+ idx=0,
69
+):
70
+ orig_width, orig_height = image.size
71
+
72
+ target_ratios = get_internvl_target_ratios(1, max_num_tiles)
73
+
74
+ blocks, target_width, target_height = calculate_internvl_targets(
75
+ orig_width=orig_width,
76
+ orig_height=orig_height,
77
+ target_ratios=target_ratios,
78
+ image_size=image_size,
79
+ use_thumbnail=False,
80
+ )
81
+
82
+ image = np.asarray(
83
+ image.convert("RGB") if image.mode != "RGB" else image, dtype=np.uint8
84
+ )
85
+
86
+ image = torch.from_numpy(image).unsqueeze(0) # (1, H, W, 3)
87
+ image = image.permute(0, 3, 1, 2) # (1, 3, H, W)
88
+
89
+ resized_img = torch.nn.functional.interpolate(
90
+ image,
91
+ size=(target_height, target_width),
92
+ mode="bicubic",
93
+ align_corners=False,
94
+ antialias=True,
95
+ )
96
+ B, C, H, W = resized_img.shape
97
+ hp, wp = H // image_size, W // image_size
98
+ patches = (
99
+ resized_img.reshape(B, C, hp, image_size, wp, image_size)
100
+ .permute(0, 2, 4, 1, 3, 5)
101
+ .reshape(B * hp * wp, C, image_size, image_size)
102
+ / 255.0
103
+ )
104
+
105
+ if use_thumbnail and patches.shape[0] > 1:
106
+ thumb = (
107
+ torch.nn.functional.interpolate(
108
+ image,
109
+ size=(image_size, image_size),
110
+ mode="bicubic",
111
+ align_corners=False,
112
+ antialias=True,
113
+ )
114
+ / 255.0
115
+ )
116
+ patches = torch.cat([patches, thumb], dim=0)
117
+
118
+ return list(patches)
119
+
120
+
121
+def image_to_pixel_values(
122
+ image: Image.Image,
123
+ *,
124
+ input_size: int,
125
+ max_num: int,
126
+ use_thumbnail: bool,
127
+ idx: int,
128
+) -> torch.Tensor:
129
+ images = dynamic_preprocess(
130
+ image,
131
+ image_size=input_size,
132
+ max_num_tiles=max_num,
133
+ use_thumbnail=use_thumbnail,
134
+ idx=idx,
135
+ )
136
+
137
+ pixel_values = torch.stack(images)
138
+ return pixel_values
139
+
140
+
141
+def video_to_pixel_values(
142
+ video: npt.NDArray,
143
+ *,
144
+ input_size: int,
145
+ max_num_tiles: int = 1,
146
+ use_thumbnail: bool,
147
+) -> torch.Tensor:
148
+ assert max_num_tiles == 1, "Video modality always uses one tile"
149
+
150
+ # (num_frames, H, W, C) -> (num_frames, C, H, W)
151
+ video_tensor = torch.from_numpy(video).permute(0, 3, 1, 2)
152
+
153
+ if video_tensor.shape[2] != input_size or video_tensor.shape[3] != input_size:
154
+ video_tensor = torch.nn.functional.interpolate(
155
+ video_tensor,
156
+ size=(input_size, input_size),
157
+ mode="bicubic",
158
+ align_corners=False,
159
+ antialias=True,
160
+ )
161
+
162
+ video_tensor = video_tensor / 255.0
163
+
164
+ return video_tensor
165
+
166
+
167
+class DynamicResolutionImageTiler:
168
+ CONV_MERGING = False
169
+ PIXEL_SHUFFLE = True
170
+ USE_THUMBNAIL = False
171
+
172
+ def __init__(
173
+ self,
174
+ *,
175
+ max_model_len: int,
176
+ patch_size: int,
177
+ min_num_patches: int,
178
+ max_num_patches: int,
179
+ downsample_ratio: int,
180
+ norm_mean: Sequence[float],
181
+ norm_std: Sequence[float],
182
+ factor_max: float = 1.0,
183
+ use_thumbnail: bool = False,
184
+ ) -> None:
185
+ assert use_thumbnail is False, "use_thumbnail is not supported"
186
+ self._patch_size: int = patch_size
187
+ self._max_model_len = max_model_len
188
+ self._min_num_patches = min_num_patches
189
+ self._max_num_patches = max_num_patches if max_num_patches > 0 else float("inf")
190
+ self._factor_max = factor_max
191
+ self.norm_mean = torch.tensor(norm_mean).reshape(3, 1, 1)
192
+ self.norm_std = torch.tensor(norm_std).reshape(3, 1, 1)
193
+ assert downsample_ratio < 1
194
+ reduction_factor = 1 / downsample_ratio
195
+ assert reduction_factor == 2.0
196
+ self._downsample_ratio = int(reduction_factor) ** (
197
+ self.PIXEL_SHUFFLE + self.CONV_MERGING
198
+ )
199
+ assert self._downsample_ratio == 2
200
+
201
+ def _get_num_embeddings(self, width: int, height: int) -> int:
202
+ num_patches = (width // self._patch_size) * (height // self._patch_size)
203
+ num_tokens = num_patches // (self._downsample_ratio**2)
204
+ return num_tokens
205
+
206
+ def width_and_height_for_max_num_tokens_available(
207
+ self,
208
+ target_num_tokens_post_shuffle: int,
209
+ ) -> tuple[int, int]:
210
+ """
211
+ TODO: optimize this so it squeezes closer to target number of tokens.
212
+ Calculate image dimensions that produce approximately `target` tokens after
213
+ pixel_shuffle.
214
+
215
+ With pixel_shuffle enabled, each 2x2 patch grid becomes 1 token, so we
216
+ need 4*B patches to get B tokens.
217
+
218
+ Examples:
219
+ >>> PATCH_SIZE = 16
220
+ >>> DOWNSAMPLE_RATIO = 0.5
221
+ >>> tiler = DynamicResolutionImageTiler(
222
+ ... max_model_len=16384,
223
+ ... patch_size=PATCH_SIZE,
224
+ ... downsample_ratio=DOWNSAMPLE_RATIO,
225
+ ... min_num_patches=4,
226
+ ... max_num_patches=0,
227
+ ... )
228
+ >>> width, height = tiler.width_and_height_for_max_num_tokens_available(
229
+ ... target_num_tokens_post_shuffle=8192,
230
+ ... )
231
+ >>> assert width, height == (2880, 2880)
232
+ >>> assert (width // PATCH_SIZE) * (
233
+ ... height // PATCH_SIZE
234
+ ... ) // 2**2 == 8100 # tokens post-shuffle
235
+ >>> assert tiler._get_num_embeddings(width=width, height=height) == 8100
236
+ """
237
+ side_pixels = (
238
+ math.isqrt(target_num_tokens_post_shuffle)
239
+ * self._downsample_ratio
240
+ * self._patch_size
241
+ )
242
+ assert isinstance(side_pixels, int) and side_pixels % self._patch_size == 0
243
+ return side_pixels, side_pixels
244
+
245
+ def max_num_tokens_available(self, text_prompt_length: int) -> int:
246
+ return self._max_model_len - text_prompt_length - 4
247
+
248
+ def _images_to_pixel_values_lst(
249
+ self,
250
+ text_prompt_length: int,
251
+ images: list[Image.Image],
252
+ ) -> tuple[list[torch.Tensor], list[int]]:
253
+ num_tokens_available = self.max_num_tokens_available(text_prompt_length)
254
+ params_per_image = self.compute_params(images, num_tokens_available)
255
+
256
+ feature_sizes = []
257
+ images = []
258
+ for param in params_per_image:
259
+ for t in self.apply_params(param):
260
+ assert t.ndim == 3, f"{t.ndim=}: expected 3 dim tensor"
261
+ images.append(t)
262
+ feature_sizes.append(param.num_embeddings)
263
+ return images, feature_sizes
264
+
265
+ feature_size_cache: dict[Image.Image, int] = {}
266
+
267
+ @classmethod
268
+ def get_cached_feature_size(cls, image: Image.Image) -> int:
269
+ feature_size = cls.feature_size_cache[id(image)]
270
+ # hard assert that we only use the feature size once
271
+ del cls.feature_size_cache[id(image)]
272
+ return feature_size
273
+
274
+ @dataclass
275
+ class DynamicResolutionParams:
276
+ media: Image.Image
277
+ num_tiles: int
278
+ num_embeddings: int
279
+ patch_size: tuple[int, int]
280
+
281
+ def apply_params(self, params: DynamicResolutionParams) -> list[torch.Tensor]:
282
+ target_size = (
283
+ params.patch_size[1] * self._patch_size,
284
+ params.patch_size[0] * self._patch_size,
285
+ )
286
+ image = np.asarray(
287
+ params.media.convert("RGB") if params.media.mode != "RGB" else params.media,
288
+ dtype=np.uint8,
289
+ )
290
+ resized_img = (
291
+ torch.nn.functional.interpolate(
292
+ torch.from_numpy(image).unsqueeze(0).permute(0, 3, 1, 2),
293
+ size=target_size,
294
+ mode="bicubic",
295
+ align_corners=False,
296
+ antialias=True,
297
+ )
298
+ / 255.0
299
+ )
300
+ return list(resized_img)
301
+
302
+ def process_media(
303
+ self,
304
+ media: Image.Image,
305
+ num_tokens_available: int,
306
+ ) -> tuple[DynamicResolutionParams, int]:
307
+ """Process a single media item and return its parameters.
308
+
309
+ Args:
310
+ media: The media item to process
311
+ num_tokens_available: Number of tokens available for this media
312
+ Returns:
313
+ DynamicResolutionParams for the media
314
+ """
315
+ current_num_tokens_available = num_tokens_available
316
+ assert isinstance(media, Image.Image), (
317
+ "Dynamic resolution is only supported for image media"
318
+ )
319
+ orig_width, orig_height = media.width, media.height
320
+ closest_patch_height = round(orig_height / self._patch_size + 0.5)
321
+ closest_patch_width = round(orig_width / self._patch_size + 0.5)
322
+ patches = closest_patch_height * closest_patch_width
323
+
324
+ factor = min(
325
+ math.sqrt(current_num_tokens_available / patches), self._factor_max
326
+ )
327
+ target_patch_height = math.floor(factor * closest_patch_height)
328
+ target_patch_width = math.floor(factor * closest_patch_width)
329
+
330
+ # Consider self._min_num_patches if > current_num_tokens_available.
331
+ if (
332
+ current_num_tokens_available > self._min_num_patches
333
+ and target_patch_height * target_patch_width < self._min_num_patches
334
+ ):
335
+ up_factor = math.sqrt(
336
+ self._min_num_patches / (target_patch_height * target_patch_width)
337
+ )
338
+ target_patch_height = math.ceil(up_factor * target_patch_height)
339
+ target_patch_width = math.ceil(up_factor * target_patch_width)
340
+
341
+ # Round patch grid to be divisible by 2 (pixel-shuffle OR conv-merging)
342
+ # or by 4 when BOTH are enabled (two successive 2x reductions)
343
+ if self.PIXEL_SHUFFLE or self.CONV_MERGING:
344
+ required_divisor = 4 if (self.PIXEL_SHUFFLE and self.CONV_MERGING) else 2
345
+
346
+ rem_h = target_patch_height % required_divisor
347
+ if rem_h != 0:
348
+ inc_h = required_divisor - rem_h
349
+ if (
350
+ target_patch_height + inc_h
351
+ ) * target_patch_width <= current_num_tokens_available:
352
+ target_patch_height += inc_h
353
+ else:
354
+ target_patch_height = max(
355
+ required_divisor, target_patch_height - rem_h
356
+ )
357
+
358
+ rem_w = target_patch_width % required_divisor
359
+ if rem_w != 0:
360
+ inc_w = required_divisor - rem_w
361
+ if (
362
+ target_patch_height * (target_patch_width + inc_w)
363
+ <= current_num_tokens_available
364
+ ):
365
+ target_patch_width += inc_w
366
+ else:
367
+ target_patch_width = max(
368
+ required_divisor, target_patch_width - rem_w
369
+ )
370
+
371
+ # Calculate embeddings for the main dynamic resolution image
372
+ num_embeddings = self._get_num_embeddings(
373
+ target_patch_width * self._patch_size,
374
+ target_patch_height * self._patch_size,
375
+ )
376
+
377
+ token_count = target_patch_width * target_patch_height
378
+
379
+ # Add thumbnail embeddings if enabled and image area is below threshold
380
+ num_tiles = 1 # Base dynamic resolution image
381
+
382
+ return self.DynamicResolutionParams(
383
+ media=media,
384
+ num_tiles=num_tiles,
385
+ num_embeddings=num_embeddings,
386
+ patch_size=(target_patch_width, target_patch_height),
387
+ ), token_count
388
+
389
+ def compute_params(
390
+ self,
391
+ media_list: list[Image.Image],
392
+ num_tokens_available: int,
393
+ ) -> list[DynamicResolutionParams]:
394
+ """Compute parameters for all media with iterative token budgeting.
395
+
396
+ Args:
397
+ media_list: List of media items to process
398
+ num_tokens_available: Total number of tokens available across all media
399
+ Returns:
400
+ List of ImageTilingParams for each media item
401
+ """
402
+ num_tokens_available = (
403
+ num_tokens_available
404
+ * (4 if self.PIXEL_SHUFFLE else 1)
405
+ * (4 if self.CONV_MERGING else 1)
406
+ )
407
+ # When the number of available token is too small,
408
+ # allow self._min_num_patches per media and let the sample be truncated.
409
+ num_tokens_available = max(
410
+ num_tokens_available, self._min_num_patches * len(media_list)
411
+ )
412
+
413
+ # Clip the number of tokens available per media to >min and <max patches.
414
+ num_tokens_available_per_media = [
415
+ int(
416
+ max(
417
+ min(num_tokens_available, self._max_num_patches),
418
+ self._min_num_patches,
419
+ )
420
+ )
421
+ for _ in range(len(media_list))
422
+ ]
423
+
424
+ # prevent infinite loop in any case
425
+ for _ in range(10):
426
+ # Step 1: Process each media with current token budget
427
+ params = []
428
+ token_counts = []
429
+
430
+ for media, tokens_for_media in zip(
431
+ media_list, num_tokens_available_per_media
432
+ ):
433
+ param, token_count = self.process_media(media, tokens_for_media)
434
+ params.append(param)
435
+ token_counts.append(token_count)
436
+ self.feature_size_cache[id(param.media)] = param.num_embeddings
437
+
438
+ # Step 2: Check if total tokens is within budget
439
+ total_tokens = sum(token_counts)
440
+
441
+ if total_tokens <= num_tokens_available:
442
+ # We're within budget, return the params
443
+ return params
444
+
445
+ # Step 3: We're over budget, need to scale down
446
+ # Calculate scaling factor to get under budget
447
+ scaling_factor = num_tokens_available / total_tokens
448
+
449
+ # Recalculate token budgets for each media based on scaling
450
+ # Each media gets a proportional share of the total budget
451
+ scaled_down_num_tokens_available_per_media = [
452
+ max(self._min_num_patches, int(token_count * scaling_factor))
453
+ for token_count in token_counts
454
+ ]
455
+ scaled_down = any(
456
+ [
457
+ scaled_down_num_tokens_available_per_media[i]
458
+ < num_tokens_available_per_media[i]
459
+ for i in range(len(num_tokens_available_per_media))
460
+ ]
461
+ )
462
+ # If there wasn't scaling down, we're stuck with min_num_patches per media,
463
+ # else try with the scaled down num_tokens_available_per_media.
464
+ if not scaled_down:
465
+ num_tokens_available_per_media = [self._min_num_patches] * len(
466
+ media_list
467
+ )
468
+ else:
469
+ num_tokens_available_per_media = (
470
+ scaled_down_num_tokens_available_per_media
471
+ )
472
+ ctx = f"{params=} {total_tokens=} {num_tokens_available=}"
473
+ raise ValueError(
474
+ f"Should be unreachable - `return params` above must be reached: {ctx}"
475
+ )
476
+
477
+ @staticmethod
478
+ def stack(images: list[torch.Tensor], patch_size: int) -> torch.Tensor:
479
+ assert len(images) > 0, "No images to stack"
480
+
481
+ def rearrange_img(x):
482
+ py = x.shape[-2] // patch_size
483
+ px = x.shape[-1] // patch_size
484
+ x = einops.rearrange(
485
+ x,
486
+ "c (py yy) (px xx) -> (py px) (c yy xx)",
487
+ py=py,
488
+ yy=patch_size,
489
+ px=px,
490
+ xx=patch_size,
491
+ )
492
+ return x
493
+
494
+ imgs = [rearrange_img(img) for img in images]
495
+ pixel_values_flat = torch.cat(imgs, dim=0).unsqueeze(0)
496
+ return pixel_values_flat
497
+
498
+
499
+class BaseNanoNemotronVLProcessor(ABC):
500
+ """
501
+ This model doesn't define its own HF processor,
502
+ so we implement our own one here.
503
+
504
+ The code to insert image tokens is based on:
505
+ https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
506
+ """
507
+
508
+ def __init__(
509
+ self,
510
+ config: PretrainedConfig,
511
+ tokenizer: TokenizerLike,
512
+ *args,
513
+ max_model_len: int,
514
+ max_num_tiles: int | None = None,
515
+ **kwargs,
516
+ ) -> None:
517
+ super().__init__()
518
+
519
+ self.config = config
520
+ self.tokenizer = tokenizer
521
+
522
+ self.max_num_tiles = max_num_tiles or DEFAULT_NUM_TILES
523
+ image_size: int = config.force_image_size
524
+ patch_size: int = config.patch_size
525
+ downsample_ratio: int = config.downsample_ratio
526
+
527
+ self.num_image_token = int(
528
+ (image_size // patch_size) ** 2 * (downsample_ratio**2)
529
+ )
530
+ self.image_size = image_size
531
+ self.use_thumbnail: bool = config.use_thumbnail
532
+ self.norm_mean = torch.Tensor(config.norm_mean).reshape(1, 3, 1, 1)
533
+ self.norm_std = torch.Tensor(config.norm_std).reshape(1, 3, 1, 1)
534
+
535
+ self.dynamic_tiler: DynamicResolutionImageTiler | None = None
536
+ if self.use_dynamic_resolution(config):
537
+ self.dynamic_tiler = DynamicResolutionImageTiler(
538
+ max_model_len=max_model_len,
539
+ patch_size=patch_size,
540
+ downsample_ratio=downsample_ratio,
541
+ min_num_patches=config.vision_config.args["min_num_patches"],
542
+ max_num_patches=config.vision_config.args["max_num_patches"],
543
+ norm_mean=config.norm_mean,
544
+ norm_std=config.norm_std,
545
+ )
546
+
547
+ @staticmethod
548
+ def use_dynamic_resolution(config: PretrainedConfig) -> bool:
549
+ return "min_num_patches" in config.vision_config.args
550
+
551
+ @property
552
+ @abstractmethod
553
+ def image_token_id(self) -> int:
554
+ raise NotImplementedError
555
+
556
+ @abstractmethod
557
+ def get_image_repl(
558
+ self,
559
+ feature_size: int,
560
+ num_patches: int | None,
561
+ ) -> PromptUpdateDetails[str]:
562
+ raise NotImplementedError
563
+
564
+ def get_num_image_tokens(
565
+ self,
566
+ *,
567
+ image_width: int,
568
+ image_height: int,
569
+ max_num_tiles: int,
570
+ ) -> int:
571
+ target_ratios = get_internvl_target_ratios(1, max_num_tiles)
572
+
573
+ num_patches, _, _ = calculate_internvl_targets(
574
+ orig_width=image_width,
575
+ orig_height=image_height,
576
+ target_ratios=target_ratios,
577
+ image_size=self.image_size,
578
+ use_thumbnail=self.use_thumbnail,
579
+ )
580
+
581
+ return num_patches * self.num_image_token
582
+
583
+ def _images_to_pixel_values_lst(
584
+ self,
585
+ images: list[Image.Image],
586
+ max_num_tiles: int,
587
+ ) -> list[torch.Tensor]:
588
+ return [
589
+ image_to_pixel_values(
590
+ image,
591
+ input_size=self.image_size,
592
+ max_num=max_num_tiles,
593
+ use_thumbnail=self.use_thumbnail,
594
+ idx=idx,
595
+ )
596
+ for idx, image in enumerate(images)
597
+ ]
598
+
599
+ def _preprocess_image(
600
+ self,
601
+ text: list[str],
602
+ images: list[Image.Image],
603
+ max_num_tiles: int,
604
+ ) -> tuple[list[str], dict[str, Any]]:
605
+ if len(images) == 0:
606
+ return text, {}
607
+
608
+ image_inputs: dict[str, Any]
609
+ if tiler := self.dynamic_tiler:
610
+ sans_images = text[0].replace("<image>", "")
611
+ text_prompt_length = len(
612
+ self.tokenizer(sans_images, add_special_tokens=False).input_ids
613
+ )
614
+ pixel_values_lst, num_tokens_per_image = tiler._images_to_pixel_values_lst(
615
+ text_prompt_length=text_prompt_length,
616
+ images=images,
617
+ )
618
+ imgs_sizes = [(pv.shape[-2], pv.shape[-1]) for pv in pixel_values_lst]
619
+ normalized = [
620
+ input_conditioner(img, tiler.norm_mean, tiler.norm_std)
621
+ for img in pixel_values_lst
622
+ ]
623
+ image_num_patches = torch.tensor([1] * len(num_tokens_per_image))
624
+ image_inputs = {
625
+ "pixel_values_flat": normalized,
626
+ "imgs_sizes": imgs_sizes,
627
+ "num_tokens_per_image": num_tokens_per_image,
628
+ }
629
+ else:
630
+ pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles)
631
+ image_num_patches = torch.tensor([len(item) for item in pixel_values_lst])
632
+ pixel_values_flat = input_conditioner(
633
+ torch.cat(pixel_values_lst), self.norm_mean, self.norm_std
634
+ )
635
+ image_inputs = {
636
+ "pixel_values_flat": pixel_values_flat,
637
+ "image_num_patches": image_num_patches,
638
+ }
639
+ num_tokens_per_image = [
640
+ self.num_image_token * len(item) for item in pixel_values_lst
641
+ ]
642
+
643
+ assert len(text) == 1, (
644
+ "hf_processor is called on the output of get_dummy_text, "
645
+ "which should be a single string"
646
+ )
647
+ parts = [x for x in re.split(r"(<image>)", text[0]) if x]
648
+ assert parts.count("<image>") == len(pixel_values_lst), (
649
+ "the number of <image> tokens in the text should be the "
650
+ "same as the number of images"
651
+ )
652
+
653
+ for i, (feature_size, num_patches) in enumerate(
654
+ zip(num_tokens_per_image, image_num_patches, strict=True)
655
+ ):
656
+ image_repl = self.get_image_repl(feature_size, num_patches)
657
+ parts[i] = parts[i].replace("<image>", image_repl.full)
658
+ text = ["".join(parts)]
659
+
660
+ return text, image_inputs
661
+
662
+ def _make_batch_input(self, input_item: _T | list[_T] | None = None) -> list[_T]:
663
+ if input_item is None:
664
+ input_item = []
665
+ if not isinstance(input_item, list):
666
+ input_item = [input_item]
667
+ return input_item
668
+
669
+ @abstractmethod
670
+ def __call__(
671
+ self,
672
+ text: str | list[str] | None = None,
673
+ images: Image.Image | list[Image.Image] | None = None,
674
+ *,
675
+ return_tensors: str | TensorType | None = None,
676
+ max_num_tiles: int | None = None,
677
+ **kwargs,
678
+ ) -> BatchFeature:
679
+ raise NotImplementedError
680
+
681
+
682
+class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
683
+ """
684
+ HF Processor with extended video processing logic.
685
+ Code for video processing is adapted from video example:
686
+ https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
687
+ """
688
+
689
+ def __init__(
690
+ self,
691
+ config: PretrainedConfig,
692
+ tokenizer: TokenizerLike,
693
+ *,
694
+ max_model_len: int,
695
+ max_num_tiles: int | None = None,
696
+ video_token: str | None = None,
697
+ video_pruning_rate: float | None = None,
698
+ ) -> None:
699
+ super().__init__(
700
+ config=config,
701
+ tokenizer=tokenizer,
702
+ max_model_len=max_model_len,
703
+ max_num_tiles=max_num_tiles,
704
+ )
705
+ # add extra video token for video processing
706
+ self.video_token = video_token
707
+ self.video_pruning_rate = video_pruning_rate
708
+
709
+ self.audio_extractor: ParakeetExtractor | None = None
710
+ raw_sound_config = getattr(config, "sound_config", None)
711
+ if raw_sound_config is not None:
712
+ self.audio_extractor = ParakeetExtractor(raw_sound_config)
713
+
714
+ # Pre-tokenize special tokens for video processing
715
+ # to avoid repeated tokenization
716
+ self._img_start_token_ids = tokenizer.encode(
717
+ IMG_START, add_special_tokens=False
718
+ )
719
+ self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)
720
+ self._img_context_token_ids = tokenizer.encode(
721
+ IMG_CONTEXT, add_special_tokens=False
722
+ )
723
+
724
+ @property
725
+ def supports_video(self) -> bool:
726
+ return self.video_token_id is not None
727
+
728
+ @property
729
+ def video_token_id(self) -> int | None:
730
+ if self.video_token is None:
731
+ return None
732
+ return self.tokenizer.get_vocab().get(self.video_token, None)
733
+
734
+ @property
735
+ def image_token_id(self) -> int:
736
+ return self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT)
737
+
738
+ def _videos_to_pixel_values_lst(
739
+ self,
740
+ videos: list[npt.NDArray],
741
+ max_num_tiles: int,
742
+ ) -> list[torch.Tensor]:
743
+ return [
744
+ video_to_pixel_values(
745
+ video,
746
+ input_size=self.image_size,
747
+ max_num_tiles=max_num_tiles,
748
+ use_thumbnail=self.use_thumbnail,
749
+ )
750
+ for video in videos
751
+ ]
752
+
753
+ def _preprocess_video(
754
+ self,
755
+ text: list[str],
756
+ videos: list[tuple[npt.NDArray, dict[str, Any]]],
757
+ max_num_tiles: int,
758
+ ) -> tuple[list[str], dict[str, Any]]:
759
+ if len(videos) == 0 or not self.supports_video:
760
+ return text, {}
761
+
762
+ videos_lst = [v[0] for v in videos]
763
+ video_metadata_lst = [v[1] for v in videos]
764
+ pixel_values_lst_video = self._videos_to_pixel_values_lst(
765
+ videos_lst,
766
+ max_num_tiles=max_num_tiles,
767
+ )
768
+
769
+ # We use frame duration in milliseconds (as integer) to ensure
770
+ # we have consistent timestamps calculation. At preprocessing
771
+ # fps parameter is given in fp32, while at inference it is bf16
772
+ # which leads to inaccurate timestamp calculation and causes
773
+ # timestamp values to differ.In rare cases this causes
774
+ # mismatching number of output tokens for tokenized frame prefixes
775
+ frame_duration_ms_lst = [
776
+ int(1000.0 / metadata["fps"]) for metadata in video_metadata_lst
777
+ ]
778
+ frames_indices_lst = [
779
+ metadata["frames_indices"] for metadata in video_metadata_lst
780
+ ]
781
+ video_num_patches = torch.tensor([len(item) for item in pixel_values_lst_video])
782
+ video_inputs = {
783
+ "pixel_values_flat_video": input_conditioner(
784
+ torch.cat(pixel_values_lst_video), self.norm_mean, self.norm_std
785
+ ),
786
+ "video_num_patches": video_num_patches,
787
+ "frames_indices": frames_indices_lst,
788
+ "frame_duration_ms": torch.tensor(frame_duration_ms_lst),
789
+ }
790
+
791
+ image_size: int = self.config.force_image_size
792
+ patch_size: int = self.config.patch_size
793
+ downsample_ratio = self.config.downsample_ratio
794
+ tokens_in_single_frame = int(
795
+ (image_size * image_size // patch_size**2) * (downsample_ratio**2)
796
+ )
797
+
798
+ for pixel_values, video_metadata, frames_indices, frame_duration_ms in zip(
799
+ pixel_values_lst_video,
800
+ video_metadata_lst,
801
+ frames_indices_lst,
802
+ frame_duration_ms_lst,
803
+ ):
804
+ num_frames = pixel_values.shape[0]
805
+
806
+ if self.video_pruning_rate is not None and self.video_pruning_rate > 0.0:
807
+ # Start of EVS-specific code
808
+ num_tokens = compute_retained_tokens_count(
809
+ tokens_per_frame=tokens_in_single_frame,
810
+ num_frames=num_frames,
811
+ q=self.video_pruning_rate,
812
+ )
813
+
814
+ # Here we just need placeholders that won't actually be replaced -
815
+ # we just need to make sure the total number of tokens is correct
816
+ # assign all tokens to the first frame
817
+ tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
818
+
819
+ # End of EVS-specific code
820
+ else:
821
+ tokens_per_frame = [tokens_in_single_frame] * num_frames
822
+
823
+ video_repl = self.get_video_repl(
824
+ tokens_per_frame=tokens_per_frame,
825
+ frames_indices=frames_indices,
826
+ frame_duration_ms=frame_duration_ms,
827
+ tokenizer=self.tokenizer,
828
+ img_start_token_ids=self._img_start_token_ids,
829
+ img_end_token_ids=self._img_end_token_ids,
830
+ img_context_token_ids=self._img_context_token_ids,
831
+ )
832
+
833
+ # video_repl.full is a list of token IDs
834
+ # Convert token IDs back to text for the HF processor flow
835
+ video_repl_text = self.tokenizer.decode(
836
+ video_repl.full, skip_special_tokens=False
837
+ )
838
+ text = [t.replace("<video>", video_repl_text, 1) for t in text]
839
+
840
+ return text, video_inputs
841
+
842
+ def _preprocess_audio(
843
+ self,
844
+ text: list[str],
845
+ audios: list[npt.NDArray],
846
+ ) -> tuple[list[str], dict[str, Any]]:
847
+ if len(audios) == 0:
848
+ return text, {}
849
+
850
+ assert self.audio_extractor is not None
851
+ extractor = self.audio_extractor
852
+
853
+ parts = [x for x in re.split(f"({re.escape(AUDIO_CONTEXT)})", text[0]) if x]
854
+ token_count = parts.count(AUDIO_CONTEXT)
855
+ if token_count != len(audios):
856
+ raise ValueError(
857
+ "Number of audio tokens in text does not match the number "
858
+ f"of audios (tokens={token_count}, audios={len(audios)})."
859
+ )
860
+ audio_index = 0
861
+ for idx, part in enumerate(parts):
862
+ if part == AUDIO_CONTEXT:
863
+ audio_repl = self.get_audio_repl(audios[audio_index])
864
+ parts[idx] = audio_repl.full
865
+ audio_index += 1
866
+ text = ["".join(parts)]
867
+ audio_inputs = extractor(
868
+ audios,
869
+ sampling_rate=extractor.sampling_rate,
870
+ return_tensors="pt",
871
+ )
872
+ input_audio_features = audio_inputs.input_features
873
+ feature_attention_mask = audio_inputs.attention_mask
874
+ audio_feature_lengths = feature_attention_mask.sum(dim=1)
875
+ audio_inputs = {
876
+ "input_audio_features": input_audio_features,
877
+ "feature_attention_mask": feature_attention_mask,
878
+ "audio_feature_lengths": audio_feature_lengths,
879
+ }
880
+
881
+ return text, audio_inputs
882
+
883
+ def __call__(
884
+ self,
885
+ text: str | list[str] | None = None,
886
+ images: Image.Image | list[Image.Image] | None = None,
887
+ videos: tuple[npt.NDArray, dict[str, Any]]
888
+ | list[tuple[npt.NDArray, dict[str, Any]]]
889
+ | None = None,
890
+ audios: AudioItem | list[AudioItem] | None = None,
891
+ *,
892
+ return_tensors: str | TensorType | None = None,
893
+ max_num_tiles: int | None = None,
894
+ **kwargs,
895
+ ) -> BatchFeature:
896
+ # Use default if not provided
897
+ if max_num_tiles is None:
898
+ max_num_tiles = self.max_num_tiles
899
+
900
+ text = self._make_batch_input(text)
901
+ images = self._make_batch_input(images)
902
+ videos = self._make_batch_input(videos)
903
+ audios = self._make_batch_input(audios)
904
+
905
+ text, image_inputs = self._preprocess_image(
906
+ text=text,
907
+ images=images,
908
+ max_num_tiles=max_num_tiles,
909
+ )
910
+
911
+ text, video_inputs = self._preprocess_video(
912
+ text=text,
913
+ videos=videos,
914
+ max_num_tiles=1,
915
+ )
916
+
917
+ text, audio_inputs = self._preprocess_audio(
918
+ text=text,
919
+ audios=audios,
920
+ )
921
+
922
+ text_inputs = self.tokenizer(text, add_special_tokens=False)
923
+
924
+ combined_inputs = {**text_inputs, **video_inputs, **audio_inputs}
925
+
926
+ if self.dynamic_tiler is None:
927
+ batch = BatchFeature(
928
+ {**combined_inputs, **image_inputs},
929
+ tensor_type=return_tensors,
930
+ )
931
+ else:
932
+ batch = BatchFeature(combined_inputs, tensor_type=return_tensors)
933
+ # allow images to be exempt from the BatchFeature validation:
934
+ # We will .stack() them in _parse_and_validate_image_input
935
+ batch.update(image_inputs)
936
+ return batch
937
+
938
+ def get_image_repl(
939
+ self,
940
+ feature_size: int,
941
+ num_patches: int | None,
942
+ ) -> PromptUpdateDetails[str]:
943
+ repl_features = IMG_CONTEXT * feature_size
944
+ repl_full = IMG_START + repl_features + IMG_END
945
+
946
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
947
+
948
+ def get_audio_repl(
949
+ self,
950
+ audio: npt.NDArray,
951
+ ) -> PromptUpdateDetails[str]:
952
+ assert self.audio_extractor is not None
953
+ num_tokens = self.audio_extractor.audio_token_count(len(audio))
954
+ repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
955
+ return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
956
+
957
+ @classmethod
958
+ def get_video_repl(
959
+ cls,
960
+ *,
961
+ tokens_per_frame: list[int],
962
+ frames_indices: list[int],
963
+ frame_duration_ms: int,
964
+ tokenizer: TokenizerLike,
965
+ img_start_token_ids: list[int],
966
+ img_end_token_ids: list[int],
967
+ img_context_token_ids: list[int],
968
+ ) -> PromptUpdateDetails[list[int]]:
969
+ """
970
+ Build prompt replacement for a video.
971
+ The replacement returned is not actually used to replace the placeholder
972
+ tokens - it's just used to make sure we allocate the correct number
973
+ of tokens.
974
+ Actual replacement is done in embed_multimodal of
975
+ NemotronH_Nano_VL_V2
976
+ (specifically in _process_video_input -> _create_final_video_embeddings).
977
+ There, we create the final embeddings with text embeddings for indicator tokens
978
+ and video embeddings for video tokens.
979
+ This is a single function that handles all cases - non EVS, EVS dummy, EVS real.
980
+ The differentiation is done via tokens_per_frame parameter.
981
+ - non EVS case - constant value same value across all frames
982
+ - EVS dummy - Doesn't matter how tokens are distributed between frames - just
983
+ make sure the total number of tokens is correct.
984
+ - EVS real (called from get_real_video_repl_for_evs) - different value per frame
985
+ Args:
986
+ tokens_per_frame (list[int]): number of tokens per frame
987
+ frames_indices (list[int]): frame indices
988
+ frame_duration_ms (int): duration of each frame in milliseconds
989
+ tokenizer (TokenizerLike): tokenizer to use for tokenizing frame separators
990
+ img_start_token_ids (list[int]): pre-tokenized IMG_START tokens
991
+ img_end_token_ids (list[int]): pre-tokenized IMG_END tokens
992
+ img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens
993
+ """
994
+ # TODO: Add support of frame_duration_ms to be None
995
+ # At preprocessing step we should allow absent / metadata without
996
+ # frames_indices field.
997
+ timestamps_enabled = frame_duration_ms is not None
998
+
999
+ if timestamps_enabled:
1000
+ timestamps = calculate_timestamps(frames_indices, frame_duration_ms)
1001
+
1002
+ assert len(timestamps) == len(tokens_per_frame), (
1003
+ "timestamps and tokens_per_frame must have the same length"
1004
+ )
1005
+ frame_separators = [
1006
+ f"Frame {i + 1} sampled at {timestamp:.2f} seconds: "
1007
+ for i, timestamp in enumerate(timestamps)
1008
+ ]
1009
+ else:
1010
+ frame_separators = [
1011
+ f"Frame {i + 1}: " for i, _ in enumerate(tokens_per_frame)
1012
+ ]
1013
+
1014
+ # Tokenize frame separator independently
1015
+ frame_separators_tokenized = [
1016
+ _seq2tokens(tokenizer, sep) for sep in frame_separators
1017
+ ]
1018
+
1019
+ # Tokenize each component independently to avoid tokenizer merging tokens
1020
+ # across boundaries. This ensures consistent tokenization regardless of
1021
+ # num_tokens_per_frame values.
1022
+ all_token_ids = []
1023
+ for i, num_tokens in enumerate(tokens_per_frame):
1024
+ frame_sep_token_ids = frame_separators_tokenized[i]
1025
+ all_token_ids.extend(frame_sep_token_ids)
1026
+
1027
+ # Add pre-tokenized special tokens
1028
+ all_token_ids.extend(img_start_token_ids)
1029
+ all_token_ids.extend(img_context_token_ids * num_tokens)
1030
+ all_token_ids.extend(img_end_token_ids)
1031
+
1032
+ return PromptUpdateDetails.from_seq(all_token_ids)
vllm/transformers_utils/processors/nemotron_parse.py
new
+245
@@ -0,0 +1,245 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+#
4
+# Adapted from https://github.com/amalad/vllm/blob/nemotron_parse/vllm/model_executor/models/nemotron_parse.py
5
+# that's based on https://huggingface.co/nvidia/NVIDIA-Nemotron-Parse-v1.1/blob/main/hf_nemotron_parse_modeling.py
6
+from typing import TypeVar
7
+
8
+import numpy as np
9
+import torch
10
+from PIL import Image
11
+from timm.data.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
12
+from torchvision import transforms as T
13
+from transformers import BatchFeature, PretrainedConfig, TensorType
14
+
15
+from vllm.tokenizers import TokenizerLike
16
+
17
+_T = TypeVar("_T")
18
+
19
+DEFAULT_FINAL_IMAGE_SIZE = (2048, 1648)
20
+
21
+
22
+class NemotronParseImageProcessor:
23
+ """
24
+ NemotronParse Image Processor
25
+ """
26
+
27
+ def __init__(
28
+ self,
29
+ final_size: tuple = DEFAULT_FINAL_IMAGE_SIZE,
30
+ **kwargs,
31
+ ):
32
+ # Ensure final_size is properly formatted
33
+ if isinstance(final_size, (list, tuple)) and len(final_size) >= 2:
34
+ self.final_size = (int(final_size[0]), int(final_size[1]))
35
+ elif isinstance(final_size, (int, float)):
36
+ self.final_size = (int(final_size), int(final_size))
37
+ else:
38
+ self.final_size = DEFAULT_FINAL_IMAGE_SIZE # Default fallback
39
+
40
+ self.norm_mean = torch.Tensor(OPENAI_CLIP_MEAN).reshape(1, 3, 1, 1)
41
+ self.norm_std = torch.Tensor(OPENAI_CLIP_STD).reshape(1, 3, 1, 1)
42
+
43
+ # Create transforms
44
+ self._create_transforms()
45
+
46
+ def _create_transforms(self):
47
+ """Create transform objects."""
48
+ try:
49
+ import albumentations as A
50
+ except ImportError as err:
51
+ raise ImportError(
52
+ "The package `albumentations` is required to use "
53
+ "NemotronParse model. Please install it with `pip install "
54
+ "albumentations`."
55
+ ) from err
56
+
57
+ # Ensure final_size is a tuple of integers
58
+ if isinstance(self.final_size, (list, tuple)):
59
+ self.target_height, self.target_width = (
60
+ int(self.final_size[0]),
61
+ int(self.final_size[1]),
62
+ )
63
+ else:
64
+ self.target_height = self.target_width = int(self.final_size)
65
+
66
+ import cv2
67
+
68
+ self.transform = A.Compose(
69
+ [
70
+ A.PadIfNeeded(
71
+ min_height=self.target_height,
72
+ min_width=self.target_width,
73
+ border_mode=cv2.BORDER_CONSTANT,
74
+ fill=[255, 255, 255],
75
+ p=1.0,
76
+ ),
77
+ ]
78
+ )
79
+
80
+ self.torch_transform = T.Compose(
81
+ [
82
+ T.ToTensor(),
83
+ ]
84
+ )
85
+
86
+ def _resize_with_aspect_ratio(self, image: np.ndarray) -> np.ndarray:
87
+ """Resize image maintaining aspect ratio (exact replica of original
88
+ LongestMaxSizeHW)."""
89
+ height, width = image.shape[:2]
90
+ max_size_height = self.target_height
91
+ max_size_width = self.target_width
92
+
93
+ # Original LongestMaxSizeHW algorithm from custom_augmentations.py
94
+ aspect_ratio = width / height
95
+ new_height = height
96
+ new_width = width
97
+
98
+ # If height too big then scale image down
99
+ if height > max_size_height:
100
+ new_height = max_size_height
101
+ new_width = int(new_height * aspect_ratio)
102
+
103
+ # If width too big, scale image down further
104
+ if new_width > max_size_width:
105
+ new_width = max_size_width
106
+ new_height = int(new_width / aspect_ratio)
107
+
108
+ # Use cv2.INTER_LINEAR like the original
109
+ import cv2
110
+
111
+ return cv2.resize(
112
+ image, (new_width, new_height), interpolation=cv2.INTER_LINEAR
113
+ )
114
+
115
+ def _pad_to_size(self, image: np.ndarray) -> np.ndarray:
116
+ """Pad image to target size with white padding (matches A.PadIfNeeded
117
+ behavior)."""
118
+ h, w = image.shape[:2]
119
+ min_height, min_width = self.target_height, self.target_width
120
+
121
+ # Only pad if image is smaller than target (matches A.PadIfNeeded logic)
122
+ pad_h = max(0, min_height - h)
123
+ pad_w = max(0, min_width - w)
124
+
125
+ if pad_h == 0 and pad_w == 0:
126
+ return image
127
+
128
+ # A.PadIfNeeded pads to bottom-right with constant value
129
+ if len(image.shape) == 3:
130
+ # Color image - pad bottom and right with white (255, 255, 255)
131
+ padded = np.pad(
132
+ image,
133
+ ((0, pad_h), (0, pad_w), (0, 0)),
134
+ mode="constant",
135
+ constant_values=255,
136
+ )
137
+ else:
138
+ # Grayscale image - pad with white (255)
139
+ padded = np.pad(
140
+ image, ((0, pad_h), (0, pad_w)), mode="constant", constant_values=255
141
+ )
142
+
143
+ return padded
144
+
145
+ def preprocess(
146
+ self,
147
+ images: Image.Image | list[Image.Image],
148
+ **kwargs,
149
+ ) -> dict[str, torch.Tensor]:
150
+ """
151
+ Preprocess an image or batch of images for the NemotronParse model.
152
+
153
+ Args:
154
+ images: Input image(s)
155
+ """
156
+ # Ensure images is a list
157
+ if not isinstance(images, list):
158
+ images = [images]
159
+
160
+ # Convert PIL images to numpy arrays if needed
161
+ processed_images = []
162
+ for image in images:
163
+ if isinstance(image, Image.Image):
164
+ image = np.asarray(image)
165
+ processed_images.append(image)
166
+
167
+ # Apply NemotronParse-specific transforms
168
+ pixel_values = []
169
+ for image in processed_images:
170
+ # Manual resize with aspect ratio preservation
171
+ # (replaces LongestMaxSizeHW)
172
+ processed_image = self._resize_with_aspect_ratio(image)
173
+
174
+ # Apply remaining albumentations transforms if available
175
+ if self.transform is not None:
176
+ transformed = self.transform(image=processed_image)
177
+ processed_image = transformed["image"]
178
+ else:
179
+ # Fallback: just pad to target size
180
+ processed_image = self._pad_to_size(processed_image)
181
+
182
+ # Convert to tensor
183
+ pixel_values_tensor = self.torch_transform(processed_image)
184
+
185
+ # Handle grayscale images
186
+ if pixel_values_tensor.shape[0] == 1:
187
+ pixel_values_tensor = pixel_values_tensor.expand(3, -1, -1)
188
+
189
+ pixel_values.append(pixel_values_tensor)
190
+
191
+ # Stack into batch
192
+ pixel_values = torch.stack(pixel_values)
193
+
194
+ # Normalize pixel values
195
+ normalized_values = (pixel_values - self.norm_mean) / self.norm_std
196
+ return {"pixel_values": normalized_values}
197
+
198
+ def __call__(
199
+ self, images: Image.Image | list[Image.Image], **kwargs
200
+ ) -> dict[str, torch.Tensor]:
201
+ return self.preprocess(images, **kwargs)
202
+
203
+
204
+class NemotronParseProcessor:
205
+ """
206
+ NemotronParse Processor
207
+ """
208
+
209
+ def __init__(
210
+ self,
211
+ config: PretrainedConfig,
212
+ tokenizer: TokenizerLike,
213
+ **kwargs,
214
+ ) -> None:
215
+ super().__init__()
216
+
217
+ self.config = config
218
+ self.tokenizer = tokenizer
219
+
220
+ self.image_processor = NemotronParseImageProcessor(final_size=config.image_size)
221
+
222
+ def _make_batch_input(self, input_item: _T | list[_T] | None = None) -> list[_T]:
223
+ if input_item is None:
224
+ input_item = []
225
+ if not isinstance(input_item, list):
226
+ input_item = [input_item]
227
+ return input_item
228
+
229
+ def __call__(
230
+ self,
231
+ text: str | list[str] | None = None,
232
+ images: Image.Image | list[Image.Image] | None = None,
233
+ return_tensors: str | TensorType | None = None,
234
+ **kwargs,
235
+ ) -> BatchFeature:
236
+ text = self._make_batch_input(text)
237
+ images = self._make_batch_input(images)
238
+ image_inputs = {} if len(images) == 0 else self.image_processor(images)
239
+
240
+ text_inputs = self.tokenizer(text, add_special_tokens=False, **kwargs)
241
+ combined_outputs = BatchFeature(
242
+ data={**text_inputs, **image_inputs},
243
+ tensor_type=return_tensors,
244
+ )
245
+ return combined_outputs
vllm/transformers_utils/processors/nemotron_vl.py
new
+410
@@ -0,0 +1,410 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from abc import ABC
4
+
5
+import torch
6
+import torchvision.transforms as T
7
+from PIL import Image
8
+from transformers import PretrainedConfig
9
+from transformers.image_processing_utils_fast import BaseImageProcessorFast
10
+
11
+from vllm.multimodal.image import convert_image_mode
12
+from vllm.multimodal.processing import PromptUpdateDetails
13
+from vllm.tokenizers import TokenizerLike
14
+
15
+from .internvl import InternVLProcessor
16
+
17
+# Configure PIL to handle large images without warnings
18
+# This prevents DecompressionBombWarning for legitimate large images
19
+Image.MAX_IMAGE_PIXELS = None # Disable the limit entirely
20
+# Alternative: Set a specific higher limit
21
+# Image.MAX_IMAGE_PIXELS = 300000000 # ~300M pixels
22
+
23
+
24
+def build_transform(input_size: int):
25
+ return T.Compose(
26
+ [
27
+ T.Lambda(lambda img: convert_image_mode(img, "RGB")),
28
+ T.Resize(
29
+ (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
30
+ ),
31
+ T.ToTensor(),
32
+ ]
33
+ )
34
+
35
+
36
+# adapted from https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1
37
+def find_closest_aspect_ratio(
38
+ aspect_ratio: float,
39
+ target_ratios: list[tuple[int, int]],
40
+ *,
41
+ width: int,
42
+ height: int,
43
+ image_size: int,
44
+) -> tuple[int, int]:
45
+ best_factor = float("-inf")
46
+ best_ratio = (1, 1)
47
+ area = width * height
48
+
49
+ for rw, rh in target_ratios:
50
+ target_aspect_ratio = rw / rh
51
+ size_factor = min((rw * rh * image_size * image_size) / area, 0.6)
52
+ ratio_closeness = min(
53
+ target_aspect_ratio / aspect_ratio, aspect_ratio / target_aspect_ratio
54
+ )
55
+ factor = size_factor * ratio_closeness
56
+
57
+ if factor > best_factor:
58
+ best_factor = factor
59
+ best_ratio = (rw, rh)
60
+
61
+ return best_ratio
62
+
63
+
64
+def calculate_nemotron_vl_targets(
65
+ *,
66
+ orig_width: int,
67
+ orig_height: int,
68
+ target_ratios: list[tuple[int, int]],
69
+ image_size: int,
70
+ use_thumbnail: bool,
71
+) -> tuple[int, int, int]:
72
+ aspect_ratio = orig_width / orig_height
73
+
74
+ # find the closest aspect ratio to the target
75
+ target_aspect_ratio = find_closest_aspect_ratio(
76
+ aspect_ratio,
77
+ target_ratios,
78
+ width=orig_width,
79
+ height=orig_height,
80
+ image_size=image_size,
81
+ )
82
+
83
+ # calculate the target width and height
84
+ target_width = image_size * target_aspect_ratio[0]
85
+ target_height = image_size * target_aspect_ratio[1]
86
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
87
+
88
+ # add thumbnail image if num_blocks != 1
89
+ if use_thumbnail and blocks != 1:
90
+ blocks += 1
91
+
92
+ return blocks, target_width, target_height
93
+
94
+
95
+def dynamic_preprocess_nemotron_vl(
96
+ image: Image.Image,
97
+ *,
98
+ target_ratios: list[tuple[int, int]],
99
+ image_size: int,
100
+ use_thumbnail: bool,
101
+) -> list[Image.Image]:
102
+ orig_width, orig_height = image.size
103
+
104
+ # calculate the number of blocks without thumbnail
105
+ blocks, target_width, target_height = calculate_nemotron_vl_targets(
106
+ orig_width=orig_width,
107
+ orig_height=orig_height,
108
+ target_ratios=target_ratios,
109
+ image_size=image_size,
110
+ use_thumbnail=False,
111
+ )
112
+
113
+ # resize the image
114
+ resized_img = image.resize((target_width, target_height))
115
+ processed_images = []
116
+ for i in range(blocks):
117
+ box = (
118
+ (i % (target_width // image_size)) * image_size,
119
+ (i // (target_width // image_size)) * image_size,
120
+ ((i % (target_width // image_size)) + 1) * image_size,
121
+ ((i // (target_width // image_size)) + 1) * image_size,
122
+ )
123
+ # split the image
124
+ split_img = resized_img.crop(box)
125
+ processed_images.append(split_img)
126
+
127
+ assert len(processed_images) == blocks
128
+
129
+ if use_thumbnail and len(processed_images) != 1:
130
+ thumbnail_img = image.resize((image_size, image_size))
131
+ processed_images.append(thumbnail_img)
132
+
133
+ return processed_images
134
+
135
+
136
+def get_nemotron_vl_target_ratios(
137
+ min_num: int,
138
+ max_num: int,
139
+) -> list[tuple[int, int]]:
140
+ target_ratios = {
141
+ (i, j)
142
+ for n in range(min_num, max_num + 1)
143
+ for i in range(1, n + 1)
144
+ for j in range(1, n + 1)
145
+ if min_num <= i * j <= max_num
146
+ }
147
+ return sorted(target_ratios, key=lambda x: x[0] * x[1])
148
+
149
+
150
+def image_to_pixel_values_nemotron_vl(
151
+ image: Image.Image,
152
+ *,
153
+ input_size: int,
154
+ min_num: int,
155
+ max_num: int,
156
+ use_thumbnail: bool,
157
+ transform: T.Compose | None = None,
158
+) -> torch.Tensor:
159
+ target_ratios = get_nemotron_vl_target_ratios(min_num, max_num)
160
+
161
+ if transform is None:
162
+ transform = build_transform(input_size=input_size)
163
+
164
+ images = dynamic_preprocess_nemotron_vl(
165
+ image,
166
+ target_ratios=target_ratios,
167
+ image_size=input_size,
168
+ use_thumbnail=use_thumbnail,
169
+ )
170
+
171
+ pixel_values = torch.stack([transform(image) for image in images])
172
+ return pixel_values
173
+
174
+
175
+class NemotronVLProcessor(InternVLProcessor):
176
+ IMG_START = "<img>"
177
+ IMG_END = "</img>"
178
+ IMG_CONTEXT = "<image>"
179
+
180
+ def __init__(
181
+ self,
182
+ config: PretrainedConfig,
183
+ tokenizer: TokenizerLike,
184
+ image_processor: BaseImageProcessorFast,
185
+ *,
186
+ min_dynamic_patch: int | None = None,
187
+ max_dynamic_patch: int | None = None,
188
+ dynamic_image_size: bool | None = None,
189
+ ) -> None:
190
+ ABC.__init__(self)
191
+ self.config = config
192
+ self.tokenizer = tokenizer
193
+ self.image_processor = image_processor
194
+ image_size: int = config.force_image_size
195
+ patch_size: int = config.patch_size
196
+
197
+ if min_dynamic_patch is None:
198
+ min_dynamic_patch = 1
199
+ assert isinstance(min_dynamic_patch, int)
200
+
201
+ if max_dynamic_patch is None:
202
+ max_dynamic_patch = self.image_processor.max_num_tiles
203
+ assert isinstance(max_dynamic_patch, int)
204
+
205
+ if dynamic_image_size is None:
206
+ dynamic_image_size = True
207
+ assert isinstance(dynamic_image_size, bool)
208
+
209
+ self.num_image_token = int(
210
+ (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
211
+ )
212
+ self.image_size = image_size
213
+ self.min_dynamic_patch = min_dynamic_patch
214
+ self.max_dynamic_patch = max_dynamic_patch
215
+ self.dynamic_image_size = dynamic_image_size
216
+
217
+ if image_processor is not None:
218
+ self.use_thumbnail = image_processor.use_thumbnail
219
+ else:
220
+ self.use_thumbnail = getattr(config, "use_thumbnail", True)
221
+
222
+ @property
223
+ def image_token_id(self) -> int:
224
+ return self.tokenizer.get_vocab()[self.IMG_CONTEXT]
225
+
226
+ def _get_transform(self) -> T.Compose:
227
+ return build_transform(input_size=self.image_size)
228
+
229
+ def get_num_image_tokens(
230
+ self,
231
+ *,
232
+ image_width: int,
233
+ image_height: int,
234
+ ) -> int:
235
+ target_ratios = self.resolve_target_ratios(
236
+ use_thumbnail=False, # Applied in calculate_targets
237
+ )
238
+
239
+ num_patches, _, _ = calculate_nemotron_vl_targets(
240
+ orig_width=image_width,
241
+ orig_height=image_height,
242
+ image_size=self.image_size,
243
+ target_ratios=target_ratios,
244
+ use_thumbnail=self.use_thumbnail,
245
+ )
246
+
247
+ return num_patches * self.num_image_token
248
+
249
+ def _images_to_pixel_values_lst(
250
+ self,
251
+ images: list[Image.Image],
252
+ min_dynamic_patch: int | None = None,
253
+ max_dynamic_patch: int | None = None,
254
+ dynamic_image_size: bool | None = None,
255
+ ) -> list[torch.Tensor]:
256
+ min_num, max_num = self.resolve_min_max_num(
257
+ min_dynamic_patch=min_dynamic_patch,
258
+ max_dynamic_patch=max_dynamic_patch,
259
+ dynamic_image_size=dynamic_image_size,
260
+ use_thumbnail=False, # Applied in image_to_pixel_values
261
+ )
262
+
263
+ return [
264
+ image_to_pixel_values_nemotron_vl(
265
+ image,
266
+ input_size=self.image_size,
267
+ min_num=min_num,
268
+ max_num=max_num,
269
+ use_thumbnail=self.use_thumbnail,
270
+ transform=self._get_transform(),
271
+ )
272
+ for image in images
273
+ ]
274
+
275
+ def _replace_image_tokens(
276
+ self,
277
+ text: list[str],
278
+ pixel_values_lst: list[torch.Tensor],
279
+ ) -> list[str]:
280
+ """Replace <image> placeholders with image tokens."""
281
+ for pixel_values in pixel_values_lst:
282
+ num_patches = pixel_values.shape[0]
283
+ feature_size = num_patches * self.num_image_token
284
+ image_repl = self.get_image_repl(feature_size, num_patches)
285
+ # Use temporary placeholder to avoid replacing tokens we just inserted
286
+ NVL_IMAGE_CONTEXT = image_repl.full.replace("<image>", "<NVL_IMG_CONTEXT>")
287
+ text = [t.replace("<image>", NVL_IMAGE_CONTEXT, 1) for t in text]
288
+ return [t.replace("<NVL_IMG_CONTEXT>", self.IMG_CONTEXT) for t in text]
289
+
290
+ def _preprocess_image(
291
+ self,
292
+ text: list[str],
293
+ images: list[Image.Image],
294
+ min_dynamic_patch: int | None = None,
295
+ max_dynamic_patch: int | None = None,
296
+ dynamic_image_size: bool | None = None,
297
+ ) -> tuple[list[str], dict[str, torch.Tensor]]:
298
+ if len(images) == 0:
299
+ image_inputs = {}
300
+ else:
301
+ pixel_values_lst = self._images_to_pixel_values_lst(
302
+ images,
303
+ min_dynamic_patch=min_dynamic_patch,
304
+ max_dynamic_patch=max_dynamic_patch,
305
+ dynamic_image_size=dynamic_image_size,
306
+ )
307
+ image_inputs = {
308
+ "pixel_values_flat": torch.cat(pixel_values_lst),
309
+ "image_num_patches": torch.tensor(
310
+ [len(item) for item in pixel_values_lst]
311
+ ),
312
+ }
313
+
314
+ text = self._replace_image_tokens(text, pixel_values_lst)
315
+ return text, image_inputs
316
+
317
+ def get_image_repl(
318
+ self,
319
+ feature_size: int,
320
+ num_patches: int | None,
321
+ ) -> PromptUpdateDetails[str]:
322
+ repl_features = self.IMG_CONTEXT * feature_size
323
+ repl_full = self.IMG_START + repl_features + self.IMG_END
324
+
325
+ return PromptUpdateDetails.select_text(repl_full, self.IMG_CONTEXT)
326
+
327
+
328
+# SigLIP normalization constants
329
+SIGLIP_MEAN = (0.5, 0.5, 0.5)
330
+SIGLIP_STD = (0.5, 0.5, 0.5)
331
+
332
+
333
+def build_siglip_transform(input_size: int):
334
+ """Build transform for SigLIP vision encoder with normalization.
335
+
336
+ Extends the base transform from nemotron_vl with SigLIP-specific normalization.
337
+ """
338
+ return T.Compose(
339
+ [
340
+ build_transform(input_size=input_size),
341
+ T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD),
342
+ ]
343
+ )
344
+
345
+
346
+class LlamaNemotronVLEmbedProcessor(NemotronVLProcessor):
347
+ """
348
+ Processor for LlamaNemotronVL embedding model.
349
+
350
+ Inherits from NemotronVLProcessor and specializes it for embedding tasks:
351
+ - Uses SigLIP transform with normalization instead of base transform
352
+ - Uses different image context token (<IMG_CONTEXT> vs <image>)
353
+ """
354
+
355
+ IMG_CONTEXT = "<IMG_CONTEXT>"
356
+
357
+ def __init__(
358
+ self,
359
+ config: PretrainedConfig,
360
+ tokenizer: TokenizerLike,
361
+ processor_config: dict,
362
+ *,
363
+ min_dynamic_patch: int | None = None,
364
+ max_dynamic_patch: int | None = None,
365
+ dynamic_image_size: bool | None = None,
366
+ ) -> None:
367
+ if min_dynamic_patch is None:
368
+ min_dynamic_patch = processor_config.get(
369
+ "min_input_tiles",
370
+ getattr(config, "min_dynamic_patch", 1),
371
+ )
372
+ if max_dynamic_patch is None:
373
+ max_dynamic_patch = processor_config.get(
374
+ "max_input_tiles",
375
+ getattr(config, "max_dynamic_patch", 1),
376
+ )
377
+ if dynamic_image_size is None:
378
+ dynamic_image_size = processor_config.get(
379
+ "dynamic_image_size",
380
+ getattr(config, "dynamic_image_size", True),
381
+ )
382
+ super().__init__(
383
+ config=config,
384
+ tokenizer=tokenizer,
385
+ image_processor=None,
386
+ min_dynamic_patch=min_dynamic_patch,
387
+ max_dynamic_patch=max_dynamic_patch,
388
+ dynamic_image_size=dynamic_image_size,
389
+ )
390
+
391
+ def _get_transform(self) -> T.Compose:
392
+ """Override to add SigLIP normalization."""
393
+ return build_siglip_transform(input_size=self.image_size)
394
+
395
+ def _replace_image_tokens(
396
+ self,
397
+ text: list[str],
398
+ pixel_values_lst: list[torch.Tensor],
399
+ ) -> list[str]:
400
+ """Override with simpler token replacement for embedding model.
401
+
402
+ No temporary placeholder needed because IMG_CONTEXT is <IMG_CONTEXT>,
403
+ not <image>, so there's no collision risk.
404
+ """
405
+ for pixel_values in pixel_values_lst:
406
+ num_patches = pixel_values.shape[0]
407
+ feature_size = num_patches * self.num_image_token
408
+ image_repl = self.get_image_repl(feature_size, num_patches)
409
+ text = [t.replace("<image>", image_repl.full, 1) for t in text]
410
+ return text
vllm/transformers_utils/processors/nvlm_d.py
new
+44
@@ -0,0 +1,44 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py
5
+# --------------------------------------------------------
6
+# NVLM-D
7
+# Copyright (c) 2024 NVIDIA
8
+# Licensed under Apache 2.0 License [see LICENSE for details]
9
+# --------------------------------------------------------
10
+from vllm.multimodal.processing import PromptUpdateDetails
11
+
12
+from .internvl import BaseInternVLProcessor
13
+
14
+IMG_PAD = "<|vision_pad|>"
15
+
16
+
17
+class NVLMProcessor(BaseInternVLProcessor):
18
+ @property
19
+ def image_token_id(self) -> int:
20
+ return self.tokenizer.get_vocab()[IMG_PAD]
21
+
22
+ def get_image_repl(
23
+ self,
24
+ feature_size: int,
25
+ num_patches: int | None,
26
+ ) -> PromptUpdateDetails[str]:
27
+ if num_patches is None:
28
+ raise NotImplementedError("Embedding inputs are not supported")
29
+
30
+ tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
31
+ if self.use_thumbnail:
32
+ tile_pos_identifiers += ["<tile_global_thumbnail>"]
33
+
34
+ context_size = feature_size // num_patches
35
+ features = "".join(
36
+ identifier + IMG_PAD * context_size for identifier in tile_pos_identifiers
37
+ )
38
+
39
+ # We include the start and end as well because "<Image><tile" is
40
+ # tokenized as ["<Image", "><", "tile"], resulting in assertion error
41
+ # when trying to find "<tile" as a subsequence of "<Image><tile"
42
+ repl = "<Image>" + features + "</Image>"
43
+
44
+ return PromptUpdateDetails.select_text(repl, IMG_PAD)
vllm/transformers_utils/processors/skyworkr1v.py
new
+389
@@ -0,0 +1,389 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py
5
+# --------------------------------------------------------
6
+# SkyworkR1V
7
+# Copyright (c) 2025 Skywork
8
+# Licensed under The MIT License [see LICENSE for details]
9
+# --------------------------------------------------------
10
+
11
+import torch
12
+import torchvision.transforms as T
13
+from PIL import Image
14
+from transformers import BatchFeature, PretrainedConfig, TensorType
15
+
16
+from vllm.multimodal.image import convert_image_mode
17
+from vllm.multimodal.processing import PromptUpdateDetails
18
+from vllm.tokenizers import TokenizerLike
19
+
20
+IMG_START = "<img>"
21
+IMG_END = "</img>"
22
+IMG_CONTEXT = "<IMG_CONTEXT>"
23
+
24
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
25
+IMAGENET_STD = (0.229, 0.224, 0.225)
26
+
27
+
28
+# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
29
+def build_transform(input_size: int):
30
+ MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
31
+ return T.Compose(
32
+ [
33
+ T.Lambda(lambda img: convert_image_mode(img, "RGB")),
34
+ T.Resize(
35
+ (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
36
+ ),
37
+ T.ToTensor(),
38
+ T.Normalize(mean=MEAN, std=STD),
39
+ ]
40
+ )
41
+
42
+
43
+# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
44
+def find_closest_aspect_ratio(
45
+ aspect_ratio: float,
46
+ target_ratios: list[tuple[int, int]],
47
+ *,
48
+ width: int,
49
+ height: int,
50
+ image_size: int,
51
+) -> tuple[int, int]:
52
+ best_ratio_diff = float("inf")
53
+ best_ratio = (1, 1)
54
+ area = width * height
55
+ for ratio in target_ratios:
56
+ target_aspect_ratio = ratio[0] / ratio[1]
57
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
58
+ if ratio_diff < best_ratio_diff:
59
+ best_ratio_diff = ratio_diff
60
+ best_ratio = ratio
61
+ elif ratio_diff == best_ratio_diff:
62
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
63
+ best_ratio = ratio
64
+ return best_ratio
65
+
66
+
67
+def resolve_skyworkr1v_min_max_num(
68
+ *,
69
+ min_dynamic_patch: int,
70
+ max_dynamic_patch: int,
71
+ dynamic_image_size: bool,
72
+ use_thumbnail: bool,
73
+) -> tuple[int, int]:
74
+ min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
75
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
76
+
77
+ if use_thumbnail and max_dynamic_patch != 1:
78
+ max_dynamic_patch += 1
79
+
80
+ return min_dynamic_patch, max_dynamic_patch
81
+
82
+
83
+def get_skyworkr1v_target_ratios(
84
+ min_num: int,
85
+ max_num: int,
86
+) -> list[tuple[int, int]]:
87
+ target_ratios = {
88
+ (i, j)
89
+ for n in range(min_num, max_num + 1)
90
+ for i in range(1, n + 1)
91
+ for j in range(1, n + 1)
92
+ if min_num <= i * j <= max_num
93
+ }
94
+ return sorted(target_ratios, key=lambda x: x[0] * x[1])
95
+
96
+
97
+def calculate_skyworkr1v_targets(
98
+ *,
99
+ orig_width: int,
100
+ orig_height: int,
101
+ target_ratios: list[tuple[int, int]],
102
+ image_size: int,
103
+ use_thumbnail: bool,
104
+) -> tuple[int, int, int]:
105
+ aspect_ratio = orig_width / orig_height
106
+
107
+ # find the closest aspect ratio to the target
108
+ target_aspect_ratio = find_closest_aspect_ratio(
109
+ aspect_ratio,
110
+ target_ratios,
111
+ width=orig_width,
112
+ height=orig_height,
113
+ image_size=image_size,
114
+ )
115
+
116
+ # calculate the target width and height
117
+ target_width = image_size * target_aspect_ratio[0]
118
+ target_height = image_size * target_aspect_ratio[1]
119
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
120
+
121
+ # add thumbnail image if num_blocks != 1
122
+ if use_thumbnail and blocks != 1:
123
+ blocks += 1
124
+
125
+ return blocks, target_width, target_height
126
+
127
+
128
+def dynamic_preprocess_skyworkr1v(
129
+ image: Image.Image,
130
+ *,
131
+ target_ratios: list[tuple[int, int]],
132
+ image_size: int,
133
+ use_thumbnail: bool,
134
+) -> list[Image.Image]:
135
+ orig_width, orig_height = image.size
136
+
137
+ # calculate the number of blocks without thumbnail
138
+ blocks, target_width, target_height = calculate_skyworkr1v_targets(
139
+ orig_width=orig_width,
140
+ orig_height=orig_height,
141
+ target_ratios=target_ratios,
142
+ image_size=image_size,
143
+ use_thumbnail=False,
144
+ )
145
+
146
+ # resize the image
147
+ resized_img = image.resize((target_width, target_height))
148
+ processed_images = []
149
+ for i in range(blocks):
150
+ box = (
151
+ (i % (target_width // image_size)) * image_size,
152
+ (i // (target_width // image_size)) * image_size,
153
+ ((i % (target_width // image_size)) + 1) * image_size,
154
+ ((i // (target_width // image_size)) + 1) * image_size,
155
+ )
156
+ # split the image
157
+ split_img = resized_img.crop(box)
158
+ processed_images.append(split_img)
159
+
160
+ assert len(processed_images) == blocks
161
+
162
+ if use_thumbnail and len(processed_images) != 1:
163
+ thumbnail_img = image.resize((image_size, image_size))
164
+ processed_images.append(thumbnail_img)
165
+
166
+ return processed_images
167
+
168
+
169
+# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B
170
+def image_to_pixel_values_skyworkr1v(
171
+ image: Image.Image,
172
+ *,
173
+ input_size: int,
174
+ min_num: int,
175
+ max_num: int,
176
+ use_thumbnail: bool,
177
+) -> torch.Tensor:
178
+ target_ratios = get_skyworkr1v_target_ratios(min_num, max_num)
179
+
180
+ transform = build_transform(input_size=input_size)
181
+ images = dynamic_preprocess_skyworkr1v(
182
+ image,
183
+ target_ratios=target_ratios,
184
+ image_size=input_size,
185
+ use_thumbnail=use_thumbnail,
186
+ )
187
+
188
+ pixel_values = torch.stack([transform(image) for image in images])
189
+ return pixel_values
190
+
191
+
192
+class SkyworkR1VProcessor:
193
+ """
194
+ This model doesn't define its own HF processor,
195
+ so we implement our own one here.
196
+
197
+ The code to insert image tokens is based on:
198
+ https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py#L252
199
+ """
200
+
201
+ def __init__(
202
+ self,
203
+ config: PretrainedConfig,
204
+ tokenizer: TokenizerLike,
205
+ *,
206
+ min_dynamic_patch: int | None = None,
207
+ max_dynamic_patch: int | None = None,
208
+ dynamic_image_size: bool | None = None,
209
+ ) -> None:
210
+ super().__init__()
211
+
212
+ self.config = config
213
+ self.tokenizer = tokenizer
214
+
215
+ image_size: int = config.vision_config.image_size
216
+ patch_size: int = config.vision_config.patch_size
217
+
218
+ if min_dynamic_patch is None:
219
+ min_dynamic_patch = config.min_dynamic_patch
220
+ assert isinstance(min_dynamic_patch, int)
221
+
222
+ if max_dynamic_patch is None:
223
+ max_dynamic_patch = config.max_dynamic_patch
224
+ assert isinstance(max_dynamic_patch, int)
225
+
226
+ if dynamic_image_size is None:
227
+ dynamic_image_size = config.dynamic_image_size
228
+ assert isinstance(dynamic_image_size, bool)
229
+
230
+ self.num_image_token = int(
231
+ (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
232
+ )
233
+ self.image_size = image_size
234
+ self.min_dynamic_patch = min_dynamic_patch
235
+ self.max_dynamic_patch = max_dynamic_patch
236
+ self.dynamic_image_size = dynamic_image_size
237
+ self.use_thumbnail: bool = config.use_thumbnail
238
+
239
+ @property
240
+ def image_token_id(self) -> int:
241
+ return self.tokenizer.get_vocab()[IMG_CONTEXT]
242
+
243
+ def get_image_repl(
244
+ self,
245
+ feature_size: int,
246
+ num_patches: int | None,
247
+ ) -> PromptUpdateDetails[str]:
248
+ repl_features = IMG_CONTEXT * feature_size
249
+ repl_full = IMG_START + repl_features + IMG_END
250
+
251
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
252
+
253
+ def resolve_min_max_num(
254
+ self,
255
+ *,
256
+ min_dynamic_patch: int | None = None,
257
+ max_dynamic_patch: int | None = None,
258
+ dynamic_image_size: bool | None = None,
259
+ use_thumbnail: bool | None = None,
260
+ ) -> tuple[int, int]:
261
+ min_dynamic_patch = (
262
+ self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
263
+ )
264
+ max_dynamic_patch = (
265
+ self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
266
+ )
267
+ dynamic_image_size = (
268
+ self.dynamic_image_size
269
+ if dynamic_image_size is None
270
+ else dynamic_image_size
271
+ )
272
+ use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
273
+
274
+ return resolve_skyworkr1v_min_max_num(
275
+ min_dynamic_patch=min_dynamic_patch,
276
+ max_dynamic_patch=max_dynamic_patch,
277
+ dynamic_image_size=dynamic_image_size,
278
+ use_thumbnail=use_thumbnail,
279
+ )
280
+
281
+ def resolve_target_ratios(
282
+ self,
283
+ *,
284
+ min_dynamic_patch: int | None = None,
285
+ max_dynamic_patch: int | None = None,
286
+ dynamic_image_size: bool | None = None,
287
+ use_thumbnail: bool | None = None,
288
+ ) -> list[tuple[int, int]]:
289
+ min_num, max_num = self.resolve_min_max_num(
290
+ min_dynamic_patch=min_dynamic_patch,
291
+ max_dynamic_patch=max_dynamic_patch,
292
+ dynamic_image_size=dynamic_image_size,
293
+ use_thumbnail=use_thumbnail,
294
+ )
295
+
296
+ return get_skyworkr1v_target_ratios(min_num, max_num)
297
+
298
+ def get_num_image_tokens(
299
+ self,
300
+ *,
301
+ image_width: int,
302
+ image_height: int,
303
+ ) -> int:
304
+ target_ratios = self.resolve_target_ratios(
305
+ use_thumbnail=False, # Applied in calculate_targets
306
+ )
307
+
308
+ num_patches, _, _ = calculate_skyworkr1v_targets(
309
+ orig_width=image_width,
310
+ orig_height=image_height,
311
+ image_size=self.image_size,
312
+ target_ratios=target_ratios,
313
+ use_thumbnail=self.use_thumbnail,
314
+ )
315
+
316
+ return num_patches * self.num_image_token
317
+
318
+ def _images_to_pixel_values_lst(
319
+ self,
320
+ images: list[Image.Image],
321
+ min_dynamic_patch: int | None = None,
322
+ max_dynamic_patch: int | None = None,
323
+ dynamic_image_size: bool | None = None,
324
+ ) -> list[torch.Tensor]:
325
+ min_num, max_num = self.resolve_min_max_num(
326
+ min_dynamic_patch=min_dynamic_patch,
327
+ max_dynamic_patch=max_dynamic_patch,
328
+ dynamic_image_size=dynamic_image_size,
329
+ use_thumbnail=False, # Applied in image_to_pixel_values
330
+ )
331
+
332
+ return [
333
+ image_to_pixel_values_skyworkr1v(
334
+ image,
335
+ input_size=self.image_size,
336
+ min_num=min_num,
337
+ max_num=max_num,
338
+ use_thumbnail=self.use_thumbnail,
339
+ )
340
+ for image in images
341
+ ]
342
+
343
+ def __call__(
344
+ self,
345
+ text: str | list[str] | None = None,
346
+ images: Image.Image | list[Image.Image] | None = None,
347
+ min_dynamic_patch: int | None = None,
348
+ max_dynamic_patch: int | None = None,
349
+ dynamic_image_size: bool | None = None,
350
+ return_tensors: str | TensorType | None = None,
351
+ ) -> BatchFeature:
352
+ if text is None:
353
+ text = []
354
+ if not isinstance(text, list):
355
+ text = [text]
356
+ if images is None:
357
+ images = []
358
+ if not isinstance(images, list):
359
+ images = [images]
360
+
361
+ if len(images) == 0:
362
+ image_inputs = {}
363
+ else:
364
+ pixel_values_lst = self._images_to_pixel_values_lst(
365
+ images,
366
+ min_dynamic_patch=min_dynamic_patch,
367
+ max_dynamic_patch=max_dynamic_patch,
368
+ dynamic_image_size=dynamic_image_size,
369
+ )
370
+ image_inputs = {
371
+ "pixel_values_flat": torch.cat(pixel_values_lst),
372
+ "image_num_patches": torch.tensor(
373
+ [len(item) for item in pixel_values_lst]
374
+ ),
375
+ }
376
+
377
+ for pixel_values in pixel_values_lst:
378
+ num_patches = pixel_values.shape[0]
379
+ feature_size = num_patches * self.num_image_token
380
+
381
+ image_repl = self.get_image_repl(feature_size, num_patches)
382
+
383
+ text = [t.replace("<image>", image_repl.full, 1) for t in text]
384
+
385
+ text_inputs = self.tokenizer(text)
386
+
387
+ combined_outputs = {**text_inputs, **image_inputs}
388
+
389
+ return BatchFeature(combined_outputs, tensor_type=return_tensors)