@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99267c23ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 12:00 UTC

[2/3] Refactor InternVL-based processors (#37324)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 18, 2026 at 22:22 UTC 99267c23ca51ef9b4486fecaf7d9ec25475f9894
18 files changed +762 -1146
tests/models/multimodal/generation/vlm_utils/model_utils.py
+17 -14
@@ -489,13 +489,14 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
489 self.image_size = self.vision_config.image_size
490
491 def __call__(self, text: str, images: Image | list[Image], **kwargs):
492 - from vllm.model_executor.models.h2ovl import (
493 - IMG_CONTEXT,
494 - IMG_END,
495 - IMG_START,
492 + from vllm.transformers_utils.processors.h2ovl import (
493 image_to_pixel_values_h2ovl,
494 )
495
496 + IMG_START = "<img>"
497 + IMG_END = "</img>"
498 + IMG_CONTEXT = "<IMG_CONTEXT>"
499 +
500 images = [images] if isinstance(images, Image) else images
501 pixel_values = [
502 image_to_pixel_values_h2ovl(
@@ -751,16 +752,17 @@ def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
752 self.image_size = self.vision_config.image_size
753
754 def __call__(self, text: str, images: Image | list[Image], **kwargs):
754 - from vllm.model_executor.models.skyworkr1v import (
755 - IMG_CONTEXT,
756 - IMG_END,
757 - IMG_START,
758 - image_to_pixel_values_skyworkr1v,
755 + from vllm.transformers_utils.processors.internvl import (
756 + image_to_pixel_values_internvl,
757 )
758
759 + IMG_START = "<img>"
760 + IMG_END = "</img>"
761 + IMG_CONTEXT = "<IMG_CONTEXT>"
762 +
763 images = [images] if isinstance(images, Image) else images
764 pixel_values = [
763 - image_to_pixel_values_skyworkr1v(
765 + image_to_pixel_values_internvl(
766 image,
767 input_size=self.image_size,
768 min_num=self.min_num,
@@ -815,14 +817,15 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
817 videos: npt.NDArray | list[npt.NDArray] = None,
818 **kwargs,
819 ):
818 - from vllm.model_executor.models.internvl import (
819 - IMG_CONTEXT,
820 - IMG_END,
821 - IMG_START,
820 + from vllm.transformers_utils.processors.internvl import (
821 image_to_pixel_values_internvl,
822 video_to_pixel_values_internvl,
823 )
824
825 + IMG_START = "<img>"
826 + IMG_END = "</img>"
827 + IMG_CONTEXT = "<IMG_CONTEXT>"
828 +
829 images = [images] if isinstance(images, Image) else images
830 videos = [videos] if isinstance(videos, np.ndarray) else videos
831 if images is not None:
tests/models/registry.py
+2 -1
@@ -779,7 +779,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
779 "rednote-hilab/dots.ocr", trust_remote_code=True
780 ),
781 "Eagle2_5_VLForConditionalGeneration": _HfExamplesInfo(
782 - "nvidia/Eagle2.5-8B", trust_remote_code=True, is_available_online=False
782 + "nvidia/Eagle2.5-8B",
783 + trust_remote_code=True,
784 ),
785 "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
786 "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
vllm/model_executor/models/eagle2_5_vl.py
+32 -6
@@ -16,7 +16,10 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
16 from vllm.model_executor.models.siglip import SiglipVisionModel
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.sequence import IntermediateTensors
19 -from vllm.transformers_utils.processors.eagle2_5_vl import Eagle2_5_VLProcessor
19 +from vllm.transformers_utils.processors.internvl import (
20 + InternVLImageProcessor,
21 + InternVLProcessor,
22 +)
23 from vllm.utils.tensor_schema import TensorSchema, TensorShape
24
25 from .interfaces import (
@@ -68,12 +71,35 @@ Eagle2_5_VLImageInputs: TypeAlias = (
71 class Eagle2_5_VLProcessingInfo(BaseInternVLProcessingInfo):
72 """Processing info for Eagle2.5-VL model."""
73
71 - def get_hf_processor(self, **kwargs) -> Eagle2_5_VLProcessor:
72 - return self.ctx.init_processor(
73 - Eagle2_5_VLProcessor,
74 - config=self.ctx.get_hf_config(),
74 + def get_image_processor(self, **kwargs):
75 + config = self.get_hf_config()
76 + vision_config = config.vision_config
77 +
78 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
79 + kwargs.setdefault(
80 + "image_size", config.force_image_size or vision_config.image_size
81 + )
82 + kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
83 + kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
84 + kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
85 + kwargs.setdefault("use_thumbnail", config.use_thumbnail)
86 +
87 + return InternVLImageProcessor(**kwargs)
88 +
89 + def get_hf_processor(self, **kwargs) -> InternVLProcessor:
90 + config = self.get_hf_config()
91 + vision_config = config.vision_config
92 +
93 + image_processor = self.get_image_processor(**kwargs)
94 + image_size = image_processor.image_size
95 + patch_size = vision_config.patch_size
96 + downsample_ratio = config.downsample_ratio
97 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
98 +
99 + return InternVLProcessor(
100 tokenizer=self.get_tokenizer(),
76 - **kwargs,
101 + image_processor=image_processor,
102 + image_seq_length=image_seq_length,
103 )
104
105
vllm/model_executor/models/glm4v.py
+2 -2
@@ -395,13 +395,13 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
395 vision_config = config.vision_config
396
397 image_size = vision_config["image_size"]
398 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
399 kwargs.setdefault("size", {"width": image_size, "height": image_size})
400
401 return GLM4VImageProcessorFast(**kwargs)
402
403 def get_hf_processor(self, **kwargs: object) -> GLM4VProcessor:
403 - return self.ctx.init_processor(
404 - GLM4VProcessor,
404 + return GLM4VProcessor(
405 tokenizer=self.get_tokenizer(),
406 image_processor=self.get_image_processor(**kwargs),
407 )
vllm/model_executor/models/h2ovl.py
+28 -6
@@ -28,7 +28,7 @@ from vllm.multimodal.processing.processor import (
28 PromptUpdate,
29 TimingContext,
30 )
31 -from vllm.transformers_utils.processors.h2ovl import H2OVLProcessor
31 +from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
32
33 from .intern_vit import InternVisionModel
34 from .internvl import (
@@ -40,12 +40,34 @@ from .internvl import (
40
41
42 class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
43 + def get_image_processor(self, **kwargs):
44 + config = self.get_hf_config()
45 + vision_config = config.vision_config
46 +
47 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
48 + kwargs.setdefault("image_size", vision_config.image_size)
49 + kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
50 + kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
51 + kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
52 + kwargs.setdefault("use_thumbnail", config.use_thumbnail)
53 + kwargs.setdefault("use_msac", config.use_msac)
54 +
55 + return H2OVLImageProcessor(**kwargs)
56 +
57 def get_hf_processor(self, **kwargs: object) -> H2OVLProcessor:
44 - return self.ctx.init_processor(
45 - H2OVLProcessor,
46 - config=self.get_hf_config(),
58 + config = self.get_hf_config()
59 + vision_config = config.vision_config
60 +
61 + image_processor = self.get_image_processor(**kwargs)
62 + image_size = image_processor.image_size
63 + patch_size = vision_config.patch_size
64 + downsample_ratio = config.downsample_ratio
65 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
66 +
67 + return H2OVLProcessor(
68 tokenizer=self.get_tokenizer(),
48 - **kwargs,
69 + image_processor=image_processor,
70 + image_seq_length=image_seq_length,
71 )
72
73 def get_num_image_tokens(
@@ -106,7 +128,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
128 if num_patches is not None:
129 assert isinstance(num_patches, int)
130
109 - return hf_processor.get_image_repl(feature_size, num_patches)
131 + return hf_processor.get_image_repl(num_patches, num_features=feature_size)
132
133 return [
134 PromptReplacement(
vllm/model_executor/models/internvl.py
+85 -46
@@ -9,6 +9,7 @@
9 # --------------------------------------------------------
10 from abc import abstractmethod
11 from collections.abc import Iterable, Mapping, Sequence
12 +from functools import cached_property
13 from typing import Annotated, Literal, TypeAlias, TypeVar
14
15 import torch
@@ -45,8 +46,9 @@ from vllm.multimodal.processing import (
46 )
47 from vllm.sequence import IntermediateTensors
48 from vllm.transformers_utils.processors.internvl import (
48 - BaseInternVLProcessor,
49 + InternVLImageProcessor,
50 InternVLProcessor,
51 + InternVLVideoProcessor,
52 )
53 from vllm.utils.tensor_schema import TensorSchema, TensorShape
54
@@ -123,7 +125,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
125 """Basic image-only ProcessingInfo for InternVL-style models."""
126
127 @abstractmethod
126 - def get_hf_processor(self, **kwargs: object) -> BaseInternVLProcessor:
128 + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
129 raise NotImplementedError
130
131 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -134,7 +136,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
136 *,
137 image_width: int,
138 image_height: int,
137 - processor: BaseInternVLProcessor,
139 + processor: InternVLProcessor,
140 ) -> int:
141 return processor.get_num_image_tokens(
142 image_width=image_width,
@@ -143,8 +145,9 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
145
146 def get_image_size_with_most_features(self) -> ImageSize:
147 processor = self.get_hf_processor()
148 + image_processor = processor.image_processor
149
147 - base_size = processor.image_size
150 + base_size = image_processor.image_size
151 target_ratios = processor.resolve_target_ratios()
152
153 largest_feature_size, largest_feature_pinpoint = 0, None
@@ -226,7 +229,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
229 )
230
231 hf_processor = self.info.get_hf_processor(**mm_kwargs)
229 - image_token_id = hf_processor.image_token_id
232 + image_token_id = hf_processor.ctx_image_token_id
233
234 # Since there may be extra tokens in the feature placeholders,
235 # we need to pass the image token ID to the model to select the
@@ -291,7 +294,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
294 if num_patches is not None:
295 assert isinstance(num_patches, int)
296
294 - return hf_processor.get_image_repl(feature_size, num_patches)
297 + return hf_processor.get_image_repl(num_patches, num_features=feature_size)
298
299 return [
300 PromptReplacement(
@@ -305,23 +308,73 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
308 class InternVLProcessingInfo(BaseInternVLProcessingInfo):
309 """InternVL ProcessingInfo extended for video processing"""
310
308 - @property
309 - def supports_video(self):
310 - return self.get_hf_processor().supports_video
311 + def get_image_processor(self, **kwargs):
312 + config = self.get_hf_config()
313 + vision_config = config.vision_config
314
312 - def get_supported_mm_limits(self):
313 - video_limit = {"video": None} if self.supports_video else {}
314 - return {**super().get_supported_mm_limits(), **video_limit}
315 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
316 + kwargs.setdefault("image_size", vision_config.image_size)
317 + kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
318 + kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
319 + kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
320 + kwargs.setdefault("use_thumbnail", config.use_thumbnail)
321 +
322 + return InternVLImageProcessor(**kwargs)
323 +
324 + def get_video_processor(self, **kwargs):
325 + config = self.get_hf_config()
326 + vision_config = config.vision_config
327
316 - def get_video_token(self) -> str | None:
328 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
329 + kwargs.setdefault("image_size", vision_config.image_size)
330 +
331 + return InternVLVideoProcessor(**kwargs)
332 +
333 + @cached_property
334 + def ctx_video_token(self):
335 text_model_type = self.get_hf_config().get_text_config().model_type
318 - video_token_map = {
336 + ctx_video_token_map = {
337 "qwen2": "<|video_pad|>",
338 "qwen3": "<|video_pad|>",
339 "qwen3_moe": "<|video_pad|>",
340 "gpt_oss": "<|reserved_200000|>",
341 }
324 - return video_token_map.get(text_model_type)
342 +
343 + if text_model_type not in ctx_video_token_map:
344 + return None
345 +
346 + ctx_video_token = ctx_video_token_map[text_model_type]
347 + if ctx_video_token not in self.get_tokenizer().get_vocab():
348 + return None
349 +
350 + return ctx_video_token
351 +
352 + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
353 + config = self.get_hf_config()
354 + vision_config = config.vision_config
355 +
356 + image_processor = self.get_image_processor(**kwargs)
357 + image_size = image_processor.image_size
358 + patch_size = vision_config.patch_size
359 + downsample_ratio = config.downsample_ratio
360 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
361 +
362 + ctx_video_token = self.ctx_video_token
363 + video_processor = (
364 + self.get_video_processor(**kwargs) if ctx_video_token else None
365 + )
366 +
367 + return InternVLProcessor(
368 + tokenizer=self.get_tokenizer(),
369 + image_processor=image_processor,
370 + video_processor=video_processor,
371 + image_seq_length=image_seq_length,
372 + ctx_video_token=ctx_video_token,
373 + )
374 +
375 + def get_supported_mm_limits(self):
376 + video_limit = {"video": None} if self.ctx_video_token else {}
377 + return {**super().get_supported_mm_limits(), **video_limit}
378
379 def get_num_frames_with_most_features(
380 self,
@@ -332,22 +385,14 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
385 max_videos = mm_counts.get("video", 0)
386
387 processor = self.get_hf_processor()
388 + num_image_token = processor.image_seq_length
389
390 max_image_tokens = self.get_max_image_tokens() * max_images
337 - max_total_frames = (seq_len - max_image_tokens) // processor.num_image_token
391 + max_total_frames = (seq_len - max_image_tokens) // num_image_token
392 max_frames_per_video = max_total_frames // max(max_videos, 1)
393
394 return max(max_frames_per_video, 1)
395
342 - def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
343 - return self.ctx.init_processor(
344 - InternVLProcessor,
345 - config=self.get_hf_config(),
346 - tokenizer=self.get_tokenizer(),
347 - video_token=self.get_video_token(),
348 - **kwargs,
349 - )
350 -
396
397 class InternVLDummyInputsBuilder(
398 BaseInternVLDummyInputsBuilder[InternVLProcessingInfo]
@@ -366,7 +411,7 @@ class InternVLDummyInputsBuilder(
411 mm_options: Mapping[str, BaseDummyOptions],
412 ) -> MultiModalDataDict:
413 dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
369 - if self.info.supports_video:
414 + if self.info.ctx_video_token:
415 config = self.info.get_hf_config()
416 image_size: int = config.vision_config.image_size
417 target_num_frames = self.info.get_num_frames_with_most_features(
@@ -405,11 +450,9 @@ class InternVLMultiModalProcessor(
450 )
451
452 hf_processor = self.info.get_hf_processor(**mm_kwargs)
408 - if (
409 - self.info.supports_video
410 - and (video_token_id := hf_processor.video_token_id) is not None
411 - ):
453 + if (video_token_id := hf_processor.ctx_video_token_id) is not None:
454 processed_outputs["video_token_id"] = torch.tensor(video_token_id)
455 +
456 return processed_outputs
457
458 def _get_mm_fields_config(
@@ -418,7 +461,7 @@ class InternVLMultiModalProcessor(
461 hf_processor_mm_kwargs: Mapping[str, object],
462 ) -> Mapping[str, MultiModalFieldConfig]:
463 image_fields = super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs)
421 - if self.info.supports_video:
464 + if self.info.ctx_video_token:
465 video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
466 num_videos = len(video_num_patches)
467 video_fields = dict(
@@ -444,6 +487,8 @@ class InternVLMultiModalProcessor(
487 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
488 out_mm_kwargs=out_mm_kwargs,
489 )
490 + if self.info.ctx_video_token is None:
491 + return prompt_repl
492
493 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
494
@@ -456,26 +501,20 @@ class InternVLMultiModalProcessor(
501 video_num_patches = []
502
503 def get_video_replacement_internvl(item_idx: int):
459 - feature_size = hf_processor.num_image_token
504 num_patches = video_num_patches[item_idx]
505 if num_patches is not None:
506 assert isinstance(num_patches, int)
507
464 - return hf_processor.get_video_repl(
465 - feature_size, num_patches, video_context_token=hf_processor.video_token
466 - )
467 -
468 - if self.info.supports_video:
469 - prompt_repl = [
470 - *prompt_repl,
471 - PromptReplacement(
472 - modality="video",
473 - target="<video>",
474 - replacement=get_video_replacement_internvl,
475 - ),
476 - ]
508 + return hf_processor.get_video_repl(num_patches)
509
478 - return prompt_repl
510 + return [
511 + *prompt_repl,
512 + PromptReplacement(
513 + modality="video",
514 + target="<video>",
515 + replacement=get_video_replacement_internvl,
516 + ),
517 + ]
518
519
520 @MULTIMODAL_REGISTRY.register_processor(
vllm/model_executor/models/nemotron_vl.py
+75 -29
@@ -26,8 +26,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
26 from vllm.sequence import IntermediateTensors
27 from vllm.transformers_utils.processor import cached_image_processor_from_config
28 from vllm.transformers_utils.processors.nemotron_vl import (
29 + LlamaNemotronNanoVLImageProcessor,
30 + LlamaNemotronNanoVLProcessor,
31 + LlamaNemotronVLEmbedImageProcessor,
32 LlamaNemotronVLEmbedProcessor,
30 - NemotronVLProcessor,
33 )
34 from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
35
@@ -50,19 +52,34 @@ from .utils import (
52 class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
53 """Processing info for Nemotron VL models."""
54
53 - def get_hf_processor(self, **kwargs: object) -> NemotronVLProcessor:
54 - return self.ctx.init_processor(
55 - NemotronVLProcessor,
56 - config=self.get_hf_config(),
57 - tokenizer=self.get_tokenizer(),
58 - image_processor=self.get_image_processor(),
59 - **kwargs,
55 + def get_image_processor(self, **kwargs: object):
56 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
57 + orig_processor = cached_image_processor_from_config(
58 + self.ctx.model_config, **kwargs
59 )
60
62 - def get_image_processor(self, **kwargs: object):
63 - return cached_image_processor_from_config(
64 - self.ctx.model_config,
65 - **kwargs,
61 + return LlamaNemotronNanoVLImageProcessor(
62 + image_size=orig_processor.image_size,
63 + min_dynamic_patch=1,
64 + max_dynamic_patch=orig_processor.max_num_tiles,
65 + dynamic_image_size=True,
66 + use_thumbnail=orig_processor.use_thumbnail,
67 + )
68 +
69 + def get_hf_processor(self, **kwargs: object) -> LlamaNemotronNanoVLProcessor:
70 + config = self.get_hf_config()
71 + vision_config = config.vision_config
72 +
73 + image_processor = self.get_image_processor(**kwargs)
74 + image_size = image_processor.image_size
75 + patch_size = vision_config.patch_size
76 + downsample_ratio = config.downsample_ratio
77 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
78 +
79 + return LlamaNemotronNanoVLProcessor(
80 + tokenizer=self.get_tokenizer(),
81 + image_processor=image_processor,
82 + image_seq_length=image_seq_length,
83 )
84
85
@@ -386,29 +403,58 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
403 # --------------------------------------------------------
404
405
389 -class LlamaNemotronVLEmbedProcessingInfo(NemotronVLProcessingInfo):
406 +class LlamaNemotronVLEmbedProcessingInfo(BaseInternVLProcessingInfo):
407 """Processing info for LlamaNemotronVL embedding model."""
408
392 - def get_hf_processor(self, **kwargs: object) -> LlamaNemotronVLEmbedProcessor:
393 - """Override to create embedding-specific processor without image_processor."""
409 + def get_image_processor(self, **kwargs):
410 model_config = self.ctx.model_config
395 - processor_config = {}
396 - if model_config.model is not None:
397 - processor_config = (
398 - get_hf_file_to_dict(
399 - "processor_config.json",
400 - model_config.model,
401 - model_config.revision,
402 - )
403 - or {}
411 +
412 + config = self.get_hf_config()
413 + processor_config = (
414 + get_hf_file_to_dict(
415 + "processor_config.json",
416 + model_config.model,
417 + model_config.revision,
418 )
419 + or {}
420 + )
421 +
422 + min_dynamic_patch = processor_config.get(
423 + "min_input_tiles",
424 + getattr(config, "min_dynamic_patch", 1),
425 + )
426 + max_dynamic_patch = processor_config.get(
427 + "max_input_tiles",
428 + getattr(config, "max_dynamic_patch", 1),
429 + )
430 + dynamic_image_size = processor_config.get(
431 + "dynamic_image_size",
432 + getattr(config, "dynamic_image_size", True),
433 + )
434 +
435 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
436 + kwargs.setdefault("image_size", config.force_image_size)
437 + kwargs.setdefault("min_dynamic_patch", min_dynamic_patch)
438 + kwargs.setdefault("max_dynamic_patch", max_dynamic_patch)
439 + kwargs.setdefault("dynamic_image_size", dynamic_image_size)
440 + kwargs.setdefault("use_thumbnail", True)
441 +
442 + return LlamaNemotronVLEmbedImageProcessor(**kwargs)
443 +
444 + def get_hf_processor(self, **kwargs: object) -> LlamaNemotronVLEmbedProcessor:
445 + config = self.get_hf_config()
446 + vision_config = config.vision_config
447 +
448 + image_processor = self.get_image_processor(**kwargs)
449 + image_size = image_processor.image_size
450 + patch_size = vision_config.patch_size
451 + downsample_ratio = config.downsample_ratio
452 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
453
406 - return self.ctx.init_processor(
407 - LlamaNemotronVLEmbedProcessor,
408 - config=self.get_hf_config(),
454 + return LlamaNemotronVLEmbedProcessor(
455 tokenizer=self.get_tokenizer(),
410 - processor_config=processor_config,
411 - **kwargs,
456 + image_processor=image_processor,
457 + image_seq_length=image_seq_length,
458 )
459
460
vllm/model_executor/models/nvlm_d.py
+31 -7
@@ -27,7 +27,8 @@ from vllm.multimodal.processing import (
27 PromptUpdate,
28 PromptUpdateDetails,
29 )
30 -from vllm.transformers_utils.processors.nvlm_d import IMG_PAD, NVLMProcessor
30 +from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
31 +from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
32
33 from .intern_vit import InternVisionModel
34 from .internvl import (
@@ -39,12 +40,33 @@ from .internvl import (
40
41
42 class NVLMProcessingInfo(BaseInternVLProcessingInfo):
43 + def get_image_processor(self, **kwargs):
44 + config = self.get_hf_config()
45 + vision_config = config.vision_config
46 +
47 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
48 + kwargs.setdefault("image_size", vision_config.image_size)
49 + kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
50 + kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
51 + kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
52 + kwargs.setdefault("use_thumbnail", config.use_thumbnail)
53 +
54 + return InternVLImageProcessor(**kwargs)
55 +
56 def get_hf_processor(self, **kwargs: object) -> NVLMProcessor:
43 - return self.ctx.init_processor(
44 - NVLMProcessor,
45 - config=self.get_hf_config(),
57 + config = self.get_hf_config()
58 + vision_config = config.vision_config
59 +
60 + image_processor = self.get_image_processor(**kwargs)
61 + image_size = image_processor.image_size
62 + patch_size = vision_config.patch_size
63 + downsample_ratio = config.downsample_ratio
64 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
65 +
66 + return NVLMProcessor(
67 tokenizer=self.get_tokenizer(),
47 - **kwargs,
68 + image_processor=image_processor,
69 + image_seq_length=image_seq_length,
70 )
71
72
@@ -117,9 +139,11 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
139 if num_patches is not None:
140 assert isinstance(num_patches, int)
141
120 - repl = hf_processor.get_image_repl(feature_size, num_patches)
142 + repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
143
122 - return PromptUpdateDetails.select_text(repl.full + "\n", IMG_PAD)
144 + return PromptUpdateDetails.select_text(
145 + repl.full + "\n", hf_processor.ctx_image_token
146 + )
147
148 # See note in dummy data regarding why we have the extra newline
149 return [
vllm/model_executor/models/qwen_vl.py
+2 -2
@@ -440,13 +440,13 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
440 vision_config = config.visual
441
442 image_size = vision_config["image_size"]
443 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
444 kwargs.setdefault("size", {"width": image_size, "height": image_size})
445
446 return QwenVLImageProcessorFast(**kwargs)
447
448 def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
448 - return self.ctx.init_processor(
449 - QwenVLProcessor,
449 + return QwenVLProcessor(
450 tokenizer=self.get_tokenizer(),
451 image_processor=self.get_image_processor(**kwargs),
452 )
vllm/model_executor/models/skyworkr1v.py
+35 -10
@@ -43,7 +43,10 @@ from vllm.multimodal.processing import (
43 PromptUpdate,
44 )
45 from vllm.sequence import IntermediateTensors
46 -from vllm.transformers_utils.processors.skyworkr1v import SkyworkR1VProcessor
46 +from vllm.transformers_utils.processors.internvl import (
47 + InternVLImageProcessor,
48 + InternVLProcessor,
49 +)
50 from vllm.utils.tensor_schema import TensorSchema, TensorShape
51
52 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -96,12 +99,33 @@ SkyworkR1VImageInputs: TypeAlias = (
99
100
101 class SkyworkR1VProcessingInfo(BaseProcessingInfo):
99 - def get_hf_processor(self, **kwargs: object) -> SkyworkR1VProcessor:
100 - return self.ctx.init_processor(
101 - SkyworkR1VProcessor,
102 - config=self.get_hf_config(),
102 + def get_image_processor(self, **kwargs):
103 + config = self.get_hf_config()
104 + vision_config = config.vision_config
105 +
106 + kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
107 + kwargs.setdefault("image_size", vision_config.image_size)
108 + kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
109 + kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
110 + kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
111 + kwargs.setdefault("use_thumbnail", config.use_thumbnail)
112 +
113 + return InternVLImageProcessor(**kwargs)
114 +
115 + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
116 + config = self.get_hf_config()
117 + vision_config = config.vision_config
118 +
119 + image_processor = self.get_image_processor(**kwargs)
120 + image_size = image_processor.image_size
121 + patch_size = vision_config.patch_size
122 + downsample_ratio = config.downsample_ratio
123 + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
124 +
125 + return InternVLProcessor(
126 tokenizer=self.get_tokenizer(),
104 - **kwargs,
127 + image_processor=image_processor,
128 + image_seq_length=image_seq_length,
129 )
130
131 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -112,7 +136,7 @@ class SkyworkR1VProcessingInfo(BaseProcessingInfo):
136 *,
137 image_width: int,
138 image_height: int,
115 - processor: SkyworkR1VProcessor,
139 + processor: InternVLProcessor,
140 ) -> int:
141 return processor.get_num_image_tokens(
142 image_width=image_width,
@@ -121,8 +145,9 @@ class SkyworkR1VProcessingInfo(BaseProcessingInfo):
145
146 def get_image_size_with_most_features(self) -> ImageSize:
147 processor = self.get_hf_processor()
148 + image_processor = processor.image_processor
149
125 - base_size = processor.image_size
150 + base_size = image_processor.image_size
151 target_ratios = processor.resolve_target_ratios()
152
153 largest_feature_size, largest_feature_pinpoint = 0, None
@@ -187,7 +212,7 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[SkyworkR1VProcessing
212 )
213
214 hf_processor = self.info.get_hf_processor(**mm_kwargs)
190 - image_token_id = hf_processor.image_token_id
215 + image_token_id = hf_processor.ctx_image_token_id
216
217 # Since there may be extra tokens in the feature placeholders,
218 # we need to pass the image token ID to the model to select the
@@ -252,7 +277,7 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[SkyworkR1VProcessing
277 if num_patches is not None:
278 assert isinstance(num_patches, int)
279
255 - return hf_processor.get_image_repl(feature_size, num_patches)
280 + return hf_processor.get_image_repl(num_patches, num_features=feature_size)
281
282 return [
283 PromptReplacement(
vllm/transformers_utils/processors/__init__.py
-4
@@ -14,7 +14,6 @@ __all__ = [
14 "BagelProcessor",
15 "CohereASRProcessor",
16 "DeepseekVLV2Processor",
17 - "Eagle2_5_VLProcessor",
17 "FireRedASR2Processor",
18 "FunASRProcessor",
19 "GLM4VProcessor",
@@ -34,14 +33,12 @@ __all__ = [
33 "Ovis2_5Processor",
34 "QwenVLProcessor",
35 "Qwen3ASRProcessor",
37 - "SkyworkR1VProcessor",
36 ]
37
38 _CLASS_TO_MODULE: dict[str, str] = {
39 "BagelProcessor": "vllm.transformers_utils.processors.bagel",
40 "CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
41 "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
44 - "Eagle2_5_VLProcessor": "vllm.transformers_utils.processors.eagle2_5_vl",
42 "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
43 "FunASRProcessor": "vllm.transformers_utils.processors.funasr",
44 "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
@@ -61,7 +58,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
58 "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
59 "QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
60 "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
64 - "SkyworkR1VProcessor": "vllm.transformers_utils.processors.skyworkr1v",
61 }
62
63
vllm/transformers_utils/processors/eagle2_5_vl.py deleted
-85
@@ -1,85 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -# Adapted from NVIDIA Eagle2.5-VL model
4 -# https://huggingface.co/nvidia/Eagle2.5-8B
5 -from transformers import PretrainedConfig
6 -
7 -from vllm.multimodal.processing import PromptUpdateDetails
8 -from vllm.tokenizers import TokenizerLike
9 -
10 -from .internvl import IMG_CONTEXT, IMG_END, IMG_START, BaseInternVLProcessor
11 -
12 -
13 -class Eagle2_5_VLProcessor(BaseInternVLProcessor):
14 - """
15 - Custom processor for Eagle2.5-VL model.
16 - Extends BaseInternVLProcessor with Eagle-specific token handling.
17 - """
18 -
19 - def __init__(
20 - self,
21 - config: PretrainedConfig,
22 - tokenizer: TokenizerLike,
23 - *,
24 - min_dynamic_patch: int | None = None,
25 - max_dynamic_patch: int | None = None,
26 - dynamic_image_size: bool | None = None,
27 - ) -> None:
28 - # Skip super().__init__() to avoid config manipulation
29 - # Directly initialize all required attributes
30 - self.config = config
31 - self.tokenizer = tokenizer
32 -
33 - # Image size with force_image_size override
34 - image_size: int = config.vision_config.image_size
35 - if hasattr(config, "force_image_size") and config.force_image_size:
36 - image_size = config.force_image_size
37 -
38 - patch_size: int = config.vision_config.patch_size
39 - downsample_ratio: float = getattr(config, "downsample_ratio", 0.5)
40 -
41 - # Compute num_image_token
42 - self.num_image_token = int(
43 - (image_size // patch_size) ** 2 * (downsample_ratio**2)
44 - )
45 - self.image_size = image_size
46 -
47 - # Dynamic patch settings with defaults
48 - self.min_dynamic_patch = (
49 - min_dynamic_patch
50 - if min_dynamic_patch is not None
51 - else getattr(config, "min_dynamic_patch", 1)
52 - )
53 - self.max_dynamic_patch = (
54 - max_dynamic_patch
55 - if max_dynamic_patch is not None
56 - else getattr(config, "max_dynamic_patch", 12)
57 - )
58 - self.dynamic_image_size = (
59 - dynamic_image_size
60 - if dynamic_image_size is not None
61 - else getattr(config, "dynamic_image_size", True)
62 - )
63 - self.use_thumbnail: bool = getattr(config, "use_thumbnail", True)
64 -
65 - @property
66 - def image_token_id(self) -> int:
67 - """Get the image token ID from config or tokenizer."""
68 - if hasattr(self.config, "image_token_index"):
69 - return self.config.image_token_index
70 - # Fallback to tokenizer vocab - use <IMG_CONTEXT> (ID: 151667)
71 - vocab = self.tokenizer.get_vocab()
72 - if IMG_CONTEXT in vocab:
73 - return vocab[IMG_CONTEXT]
74 - raise ValueError(f"Cannot find image token '{IMG_CONTEXT}' in vocabulary")
75 -
76 - def get_image_repl(
77 - self,
78 - feature_size: int,
79 - num_patches: int | None,
80 - ) -> PromptUpdateDetails[str]:
81 - """Get image replacement string for prompt."""
82 - repl_features = IMG_CONTEXT * feature_size
83 - repl_full = IMG_START + repl_features + IMG_END
84 -
85 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
vllm/transformers_utils/processors/h2ovl.py
+79 -82
@@ -10,16 +10,12 @@
10 # --------------------------------------------------------
11 import torch
12 from PIL import Image
13 -from transformers import PretrainedConfig
13
15 -from vllm.multimodal.processing import PromptUpdateDetails
16 -from vllm.tokenizers import TokenizerLike
14 +from vllm.tokenizers.hf import HfTokenizer
15
16 from .internvl import (
19 - IMG_CONTEXT,
20 - IMG_END,
21 - IMG_START,
22 - BaseInternVLProcessor,
17 + InternVLImageProcessor,
18 + InternVLProcessor,
19 build_transform,
20 find_closest_aspect_ratio,
21 get_internvl_target_ratios,
@@ -217,45 +213,26 @@ def image_to_pixel_values_h2ovl(
213 return pixel_values
214
215
220 -class H2OVLProcessor(BaseInternVLProcessor):
216 +class H2OVLImageProcessor(InternVLImageProcessor):
217 def __init__(
218 self,
223 - config: PretrainedConfig,
224 - tokenizer: TokenizerLike,
225 - *,
226 - min_dynamic_patch: int | None = None,
227 - max_dynamic_patch: int | None = None,
228 - dynamic_image_size: bool | None = None,
229 - use_msac: bool | None = None,
219 + image_size: int,
220 + min_dynamic_patch: int,
221 + max_dynamic_patch: int,
222 + dynamic_image_size: bool,
223 + use_thumbnail: bool,
224 + use_msac: bool,
225 ) -> None:
226 super().__init__(
232 - config,
233 - tokenizer,
227 + image_size=image_size,
228 min_dynamic_patch=min_dynamic_patch,
229 max_dynamic_patch=max_dynamic_patch,
230 dynamic_image_size=dynamic_image_size,
231 + use_thumbnail=use_thumbnail,
232 )
233
239 - if use_msac is None:
240 - use_msac = config.use_msac
241 - assert isinstance(use_msac, bool)
242 -
234 self.use_msac = use_msac
235
245 - @property
246 - def image_token_id(self) -> int:
247 - return self.tokenizer.get_vocab()[IMG_CONTEXT]
248 -
249 - def get_image_repl(
250 - self,
251 - feature_size: int,
252 - num_patches: int | None,
253 - ) -> PromptUpdateDetails[str]:
254 - repl_features = IMG_CONTEXT * feature_size
255 - repl_full = IMG_START + repl_features + IMG_END
256 -
257 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
258 -
236 def resolve_min_max_num(
237 self,
238 *,
@@ -264,18 +241,14 @@ class H2OVLProcessor(BaseInternVLProcessor):
241 dynamic_image_size: bool | None = None,
242 use_thumbnail: bool | None = None,
243 ) -> tuple[int, int]:
267 - min_dynamic_patch = (
268 - self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
269 - )
270 - max_dynamic_patch = (
271 - self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
272 - )
273 - dynamic_image_size = (
274 - self.dynamic_image_size
275 - if dynamic_image_size is None
276 - else dynamic_image_size
277 - )
278 - use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
244 + if min_dynamic_patch is None:
245 + min_dynamic_patch = self.min_dynamic_patch
246 + if max_dynamic_patch is None:
247 + max_dynamic_patch = self.max_dynamic_patch
248 + if dynamic_image_size is None:
249 + dynamic_image_size = self.dynamic_image_size
250 + if use_thumbnail is None:
251 + use_thumbnail = self.use_thumbnail
252
253 return resolve_h2ovl_min_max_num(
254 min_dynamic_patch=min_dynamic_patch,
@@ -284,6 +257,57 @@ class H2OVLProcessor(BaseInternVLProcessor):
257 use_thumbnail=use_thumbnail,
258 )
259
260 + def _images_to_pixel_values_lst(
261 + self,
262 + images: list[Image.Image],
263 + min_dynamic_patch: int | None = None,
264 + max_dynamic_patch: int | None = None,
265 + dynamic_image_size: bool | None = None,
266 + ) -> list[torch.Tensor]:
267 + use_msac = self.use_msac if len(images) == 1 else False
268 +
269 + min_num, max_num = self.resolve_min_max_num(
270 + min_dynamic_patch=min_dynamic_patch,
271 + max_dynamic_patch=max_dynamic_patch,
272 + dynamic_image_size=dynamic_image_size,
273 + use_thumbnail=False, # Applied in image_to_pixel_values
274 + )
275 +
276 + return [
277 + image_to_pixel_values_h2ovl(
278 + image,
279 + input_size=self.image_size,
280 + min_num=min_num,
281 + max_num=max_num,
282 + use_thumbnail=self.use_thumbnail,
283 + use_msac=use_msac,
284 + )
285 + for image in images
286 + ]
287 +
288 +
289 +class H2OVLProcessor(InternVLProcessor):
290 + def __init__(
291 + self,
292 + image_processor: H2OVLImageProcessor,
293 + tokenizer: HfTokenizer,
294 + *,
295 + image_seq_length: int,
296 + start_image_token: str = "<img>",
297 + end_image_token: str = "</img>",
298 + ctx_image_token: str = "<IMG_CONTEXT>",
299 + ) -> None:
300 + super().__init__(
301 + image_processor=image_processor,
302 + tokenizer=tokenizer,
303 + image_seq_length=image_seq_length,
304 + start_image_token=start_image_token,
305 + end_image_token=end_image_token,
306 + ctx_image_token=ctx_image_token,
307 + )
308 +
309 + self.image_processor: H2OVLImageProcessor
310 +
311 def resolve_target_ratios(
312 self,
313 *,
@@ -294,7 +318,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
318 prior_aspect_ratio: tuple[int, int] | None = None,
319 override_min_num: int | None = None,
320 ) -> list[tuple[int, int]]:
297 - min_num, max_num = self.resolve_min_max_num(
321 + min_num, max_num = self.image_processor.resolve_min_max_num(
322 min_dynamic_patch=min_dynamic_patch,
323 max_dynamic_patch=max_dynamic_patch,
324 dynamic_image_size=dynamic_image_size,
@@ -316,9 +340,10 @@ class H2OVLProcessor(BaseInternVLProcessor):
340 image_height: int,
341 use_msac: bool | None = None,
342 ) -> int:
319 - use_msac = self.use_msac if use_msac is None else use_msac
343 + image_processor = self.image_processor
344 + use_msac = image_processor.use_msac if use_msac is None else use_msac
345
321 - use_thumbnail = self.use_thumbnail
346 + use_thumbnail = image_processor.use_thumbnail
347
348 if use_msac:
349 target_ratios_1 = self.resolve_target_ratios(
@@ -328,7 +353,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
353 num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
354 orig_width=image_width,
355 orig_height=image_height,
331 - image_size=self.image_size,
356 + image_size=image_processor.image_size,
357 target_ratios=target_ratios_1,
358 use_thumbnail=True,
359 )
@@ -341,7 +366,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
366 num_patches_2, _, _, _ = calculate_h2ovl_targets(
367 orig_width=image_width,
368 orig_height=image_height,
344 - image_size=self.image_size,
369 + image_size=image_processor.image_size,
370 target_ratios=target_ratios_2,
371 use_thumbnail=True,
372 )
@@ -354,37 +379,9 @@ class H2OVLProcessor(BaseInternVLProcessor):
379 num_patches, _, _, _ = calculate_h2ovl_targets(
380 orig_width=image_width,
381 orig_height=image_height,
357 - image_size=self.image_size,
382 + image_size=image_processor.image_size,
383 target_ratios=target_ratios,
384 use_thumbnail=use_thumbnail,
385 )
386
362 - return num_patches * self.num_image_token
363 -
364 - def _images_to_pixel_values_lst(
365 - self,
366 - images: list[Image.Image],
367 - min_dynamic_patch: int | None = None,
368 - max_dynamic_patch: int | None = None,
369 - dynamic_image_size: bool | None = None,
370 - ) -> list[torch.Tensor]:
371 - use_msac = self.use_msac if len(images) == 1 else False
372 -
373 - min_num, max_num = self.resolve_min_max_num(
374 - min_dynamic_patch=min_dynamic_patch,
375 - max_dynamic_patch=max_dynamic_patch,
376 - dynamic_image_size=dynamic_image_size,
377 - use_thumbnail=False, # Applied in image_to_pixel_values
378 - )
379 -
380 - return [
381 - image_to_pixel_values_h2ovl(
382 - image,
383 - input_size=self.image_size,
384 - min_num=min_num,
385 - max_num=max_num,
386 - use_thumbnail=self.use_thumbnail,
387 - use_msac=use_msac,
388 - )
389 - for image in images
390 - ]
387 + return num_patches * self.image_seq_length
vllm/transformers_utils/processors/internvl.py
+233 -273
@@ -7,24 +7,17 @@
7 # Copyright (c) 2023 OpenGVLab
8 # Licensed under The MIT License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from abc import ABC, abstractmethod
11 -from typing import Any, TypeVar
10
11 import numpy.typing as npt
12 import torch
13 import torchvision.transforms as T
14 from PIL import Image
17 -from transformers import BatchFeature, PretrainedConfig, TensorType
15 +from transformers import BatchFeature, TensorType
16 +from transformers.processing_utils import ProcessorMixin
17
18 from vllm.multimodal.image import convert_image_mode
19 from vllm.multimodal.processing import PromptUpdateDetails
21 -from vllm.tokenizers import TokenizerLike
22 -
23 -_T = TypeVar("_T")
24 -
25 -IMG_START = "<img>"
26 -IMG_END = "</img>"
27 -IMG_CONTEXT = "<IMG_CONTEXT>"
20 +from vllm.tokenizers.hf import HfTokenizer
21
22 IMAGENET_MEAN = (0.485, 0.456, 0.406)
23 IMAGENET_STD = (0.229, 0.224, 0.225)
@@ -33,7 +26,7 @@ IMAGENET_STD = (0.229, 0.224, 0.225)
26 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
27 def build_transform(input_size: int):
28 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
36 - transform = T.Compose(
29 + return T.Compose(
30 [
31 T.Lambda(lambda img: convert_image_mode(img, "RGB")),
32 T.Resize(
@@ -43,7 +36,6 @@ def build_transform(input_size: int):
36 T.Normalize(mean=MEAN, std=STD),
37 ]
38 )
46 - return transform
39
40
41 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
@@ -223,65 +215,20 @@ def video_to_pixel_values_internvl(
215 return pixel_values
216
217
226 -class BaseInternVLProcessor(ABC):
227 - """
228 - This model doesn't define its own HF processor,
229 - so we implement our own one here.
230 -
231 - The code to insert image tokens is based on:
232 - https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
233 - """
234 -
218 +class InternVLImageProcessor:
219 def __init__(
220 self,
237 - config: PretrainedConfig,
238 - tokenizer: TokenizerLike,
239 - *,
240 - min_dynamic_patch: int | None = None,
241 - max_dynamic_patch: int | None = None,
242 - dynamic_image_size: bool | None = None,
221 + image_size: int,
222 + min_dynamic_patch: int,
223 + max_dynamic_patch: int,
224 + dynamic_image_size: bool,
225 + use_thumbnail: bool,
226 ) -> None:
244 - super().__init__()
245 -
246 - self.config = config
247 - self.tokenizer = tokenizer
248 -
249 - image_size: int = config.vision_config.image_size
250 - patch_size: int = config.vision_config.patch_size
251 -
252 - if min_dynamic_patch is None:
253 - min_dynamic_patch = config.min_dynamic_patch
254 - assert isinstance(min_dynamic_patch, int)
255 -
256 - if max_dynamic_patch is None:
257 - max_dynamic_patch = config.max_dynamic_patch
258 - assert isinstance(max_dynamic_patch, int)
259 -
260 - if dynamic_image_size is None:
261 - dynamic_image_size = config.dynamic_image_size
262 - assert isinstance(dynamic_image_size, bool)
263 -
264 - self.num_image_token = int(
265 - (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
266 - )
227 self.image_size = image_size
228 self.min_dynamic_patch = min_dynamic_patch
229 self.max_dynamic_patch = max_dynamic_patch
230 self.dynamic_image_size = dynamic_image_size
271 - self.use_thumbnail: bool = config.use_thumbnail
272 -
273 - @property
274 - @abstractmethod
275 - def image_token_id(self) -> int:
276 - raise NotImplementedError
277 -
278 - @abstractmethod
279 - def get_image_repl(
280 - self,
281 - feature_size: int,
282 - num_patches: int | None,
283 - ) -> PromptUpdateDetails[str]:
284 - raise NotImplementedError
231 + self.use_thumbnail = use_thumbnail
232
233 def resolve_min_max_num(
234 self,
@@ -291,18 +238,14 @@ class BaseInternVLProcessor(ABC):
238 dynamic_image_size: bool | None = None,
239 use_thumbnail: bool | None = None,
240 ) -> tuple[int, int]:
294 - min_dynamic_patch = (
295 - self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
296 - )
297 - max_dynamic_patch = (
298 - self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
299 - )
300 - dynamic_image_size = (
301 - self.dynamic_image_size
302 - if dynamic_image_size is None
303 - else dynamic_image_size
304 - )
305 - use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
241 + if min_dynamic_patch is None:
242 + min_dynamic_patch = self.min_dynamic_patch
243 + if max_dynamic_patch is None:
244 + max_dynamic_patch = self.max_dynamic_patch
245 + if dynamic_image_size is None:
246 + dynamic_image_size = self.dynamic_image_size
247 + if use_thumbnail is None:
248 + use_thumbnail = self.use_thumbnail
249
250 return resolve_internvl_min_max_num(
251 min_dynamic_patch=min_dynamic_patch,
@@ -311,43 +254,6 @@ class BaseInternVLProcessor(ABC):
254 use_thumbnail=use_thumbnail,
255 )
256
314 - def resolve_target_ratios(
315 - self,
316 - *,
317 - min_dynamic_patch: int | None = None,
318 - max_dynamic_patch: int | None = None,
319 - dynamic_image_size: bool | None = None,
320 - use_thumbnail: bool | None = None,
321 - ) -> list[tuple[int, int]]:
322 - min_num, max_num = self.resolve_min_max_num(
323 - min_dynamic_patch=min_dynamic_patch,
324 - max_dynamic_patch=max_dynamic_patch,
325 - dynamic_image_size=dynamic_image_size,
326 - use_thumbnail=use_thumbnail,
327 - )
328 -
329 - return get_internvl_target_ratios(min_num, max_num)
330 -
331 - def get_num_image_tokens(
332 - self,
333 - *,
334 - image_width: int,
335 - image_height: int,
336 - ) -> int:
337 - target_ratios = self.resolve_target_ratios(
338 - use_thumbnail=False, # Applied in calculate_targets
339 - )
340 -
341 - num_patches, _, _ = calculate_internvl_targets(
342 - orig_width=image_width,
343 - orig_height=image_height,
344 - image_size=self.image_size,
345 - target_ratios=target_ratios,
346 - use_thumbnail=self.use_thumbnail,
347 - )
348 -
349 - return num_patches * self.num_image_token
350 -
257 def _images_to_pixel_values_lst(
258 self,
259 images: list[Image.Image],
@@ -355,7 +261,14 @@ class BaseInternVLProcessor(ABC):
261 max_dynamic_patch: int | None = None,
262 dynamic_image_size: bool | None = None,
263 ) -> list[torch.Tensor]:
358 - min_num, max_num = self.resolve_min_max_num(
264 + if min_dynamic_patch is None:
265 + min_dynamic_patch = self.min_dynamic_patch
266 + if max_dynamic_patch is None:
267 + max_dynamic_patch = self.max_dynamic_patch
268 + if dynamic_image_size is None:
269 + dynamic_image_size = self.dynamic_image_size
270 +
271 + min_num, max_num = resolve_internvl_min_max_num(
272 min_dynamic_patch=min_dynamic_patch,
273 max_dynamic_patch=max_dynamic_patch,
274 dynamic_image_size=dynamic_image_size,
@@ -373,49 +286,9 @@ class BaseInternVLProcessor(ABC):
286 for image in images
287 ]
288
376 - def _preprocess_image(
377 - self,
378 - text: list[str],
379 - images: list[Image.Image],
380 - min_dynamic_patch: int | None = None,
381 - max_dynamic_patch: int | None = None,
382 - dynamic_image_size: bool | None = None,
383 - ) -> tuple[list[str], dict[str, torch.Tensor]]:
384 - if len(images) == 0:
385 - image_inputs = {}
386 - else:
387 - pixel_values_lst = self._images_to_pixel_values_lst(
388 - images,
389 - min_dynamic_patch=min_dynamic_patch,
390 - max_dynamic_patch=max_dynamic_patch,
391 - dynamic_image_size=dynamic_image_size,
392 - )
393 - image_inputs = {
394 - "pixel_values_flat": torch.cat(pixel_values_lst),
395 - "image_num_patches": torch.tensor(
396 - [len(item) for item in pixel_values_lst]
397 - ),
398 - }
399 -
400 - for pixel_values in pixel_values_lst:
401 - num_patches = pixel_values.shape[0]
402 - feature_size = num_patches * self.num_image_token
403 -
404 - image_repl = self.get_image_repl(feature_size, num_patches)
405 - text = [t.replace("<image>", image_repl.full, 1) for t in text]
406 - return text, image_inputs
407 -
408 - def _make_batch_input(self, input_item: _T | list[_T] | None = None) -> list[_T]:
409 - if input_item is None:
410 - input_item = []
411 - if not isinstance(input_item, list):
412 - input_item = [input_item]
413 - return input_item
414 -
289 def __call__(
290 self,
417 - text: str | list[str] | None = None,
418 - images: Image.Image | list[Image.Image] | None = None,
291 + images: Image.Image | list[Image.Image],
292 *,
293 min_dynamic_patch: int | None = None,
294 max_dynamic_patch: int | None = None,
@@ -423,120 +296,173 @@ class BaseInternVLProcessor(ABC):
296 return_tensors: str | TensorType | None = None,
297 **kwargs,
298 ) -> BatchFeature:
426 - text = self._make_batch_input(text)
427 - images = self._make_batch_input(images)
299 + images_lst = [images] if not isinstance(images, list) else images
300
429 - text, image_inputs = self._preprocess_image(
430 - text=text,
431 - images=images,
301 + pixel_values_lst = self._images_to_pixel_values_lst(
302 + images_lst,
303 min_dynamic_patch=min_dynamic_patch,
304 max_dynamic_patch=max_dynamic_patch,
305 dynamic_image_size=dynamic_image_size,
306 )
307
437 - text_inputs = self.tokenizer(text)
308 + image_inputs = {
309 + "pixel_values_flat": torch.cat(pixel_values_lst),
310 + "image_num_patches": torch.tensor([len(item) for item in pixel_values_lst]),
311 + }
312 + return BatchFeature(image_inputs, tensor_type=return_tensors)
313
439 - combined_outputs = {**text_inputs, **image_inputs}
314
441 - return BatchFeature(combined_outputs, tensor_type=return_tensors)
315 +class InternVLVideoProcessor:
316 + def __init__(
317 + self,
318 + image_size: int,
319 + ) -> None:
320 + self.image_size = image_size
321 +
322 + def _videos_to_pixel_values_lst(
323 + self,
324 + videos: list[npt.NDArray],
325 + ) -> list[torch.Tensor]:
326 + return [
327 + video_to_pixel_values_internvl(
328 + video,
329 + input_size=self.image_size,
330 + min_num=1,
331 + max_num=1,
332 + use_thumbnail=False,
333 + )
334 + for video in videos
335 + ]
336 +
337 + def __call__(
338 + self,
339 + videos: npt.NDArray | list[npt.NDArray],
340 + *,
341 + return_tensors: str | TensorType | None = None,
342 + **kwargs,
343 + ) -> BatchFeature:
344 + videos_lst = [videos] if not isinstance(videos, list) else videos
345
346 + pixel_values_lst = self._videos_to_pixel_values_lst(videos_lst)
347 +
348 + image_inputs = {
349 + "pixel_values_flat_video": torch.cat(pixel_values_lst),
350 + "video_num_patches": torch.tensor([len(item) for item in pixel_values_lst]),
351 + }
352 + return BatchFeature(image_inputs, tensor_type=return_tensors)
353
444 -class InternVLProcessor(BaseInternVLProcessor):
354 +
355 +class InternVLProcessor(ProcessorMixin):
356 """
446 - HF Processor for InternVLChatModel with extended video processing logic.
357 + This model doesn't define its own HF processor,
358 + so we implement our own one here.
359 +
360 + The code to insert image tokens is based on:
361 + https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
362
363 Code for video processing is adapted from video example:
364 https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
365 """
366
367 + attributes = ["image_processor", "tokenizer", "video_processor"]
368 +
369 def __init__(
370 self,
454 - config: PretrainedConfig,
455 - tokenizer: TokenizerLike,
371 + image_processor: InternVLImageProcessor,
372 + tokenizer: HfTokenizer,
373 + video_processor: InternVLVideoProcessor | None = None,
374 + *,
375 + image_seq_length: int,
376 + start_image_token: str = "<img>",
377 + end_image_token: str = "</img>",
378 + ctx_image_token: str = "<IMG_CONTEXT>",
379 + ctx_video_token: str | None = None,
380 + ) -> None:
381 + self.image_processor = image_processor
382 + self.tokenizer = tokenizer
383 + self.video_processor = video_processor
384 +
385 + self.image_seq_length = image_seq_length
386 + self.start_image_token = start_image_token
387 + self.end_image_token = end_image_token
388 + self.ctx_image_token = ctx_image_token
389 + self.ctx_video_token = ctx_video_token
390 +
391 + self.start_image_token_id = tokenizer.convert_tokens_to_ids(start_image_token)
392 + self.end_image_token_id = tokenizer.convert_tokens_to_ids(end_image_token)
393 + self.ctx_image_token_id = tokenizer.convert_tokens_to_ids(ctx_image_token)
394 + self.ctx_video_token_id = (
395 + None
396 + if ctx_video_token is None
397 + else tokenizer.convert_tokens_to_ids(ctx_video_token)
398 + )
399 +
400 + def resolve_target_ratios(
401 + self,
402 *,
403 min_dynamic_patch: int | None = None,
404 max_dynamic_patch: int | None = None,
405 dynamic_image_size: bool | None = None,
460 - video_token: str | None = None,
461 - ) -> None:
462 - super().__init__(
463 - config=config,
464 - tokenizer=tokenizer,
406 + use_thumbnail: bool | None = None,
407 + ) -> list[tuple[int, int]]:
408 + min_num, max_num = self.image_processor.resolve_min_max_num(
409 min_dynamic_patch=min_dynamic_patch,
410 max_dynamic_patch=max_dynamic_patch,
411 dynamic_image_size=dynamic_image_size,
412 + use_thumbnail=use_thumbnail,
413 )
469 - # add extra video token for video processing
470 - self.video_token = video_token
471 -
472 - @property
473 - def image_token_id(self) -> int:
474 - return self.tokenizer.get_vocab()[IMG_CONTEXT]
414
476 - @property
477 - def video_token_id(self) -> int | None:
478 - if self.video_token is None:
479 - return None
480 - return self.tokenizer.get_vocab().get(self.video_token, None)
481 -
482 - @property
483 - def supports_video(self) -> bool:
484 - return self.video_token_id is not None
415 + return get_internvl_target_ratios(min_num, max_num)
416
486 - def _videos_to_pixel_values_lst(
417 + def get_num_image_tokens(
418 self,
488 - videos: list[npt.NDArray],
489 - dynamic_image_size: bool | None = None,
490 - ) -> list[torch.Tensor]:
491 - min_num, max_num = self.resolve_min_max_num(
492 - min_dynamic_patch=1,
493 - max_dynamic_patch=1,
494 - dynamic_image_size=dynamic_image_size,
495 - use_thumbnail=False, # Applied in image_to_pixel_values
419 + *,
420 + image_width: int,
421 + image_height: int,
422 + ) -> int:
423 + image_processor = self.image_processor
424 + target_ratios = self.resolve_target_ratios(
425 + use_thumbnail=False, # Applied in calculate_targets
426 )
427
498 - return [
499 - video_to_pixel_values_internvl(
500 - video,
501 - input_size=self.image_size,
502 - min_num=min_num,
503 - max_num=max_num,
504 - use_thumbnail=False,
505 - )
506 - for video in videos
507 - ]
428 + num_patches, _, _ = calculate_internvl_targets(
429 + orig_width=image_width,
430 + orig_height=image_height,
431 + image_size=image_processor.image_size,
432 + target_ratios=target_ratios,
433 + use_thumbnail=image_processor.use_thumbnail,
434 + )
435
509 - def _preprocess_video(
436 + return num_patches * self.image_seq_length
437 +
438 + def get_image_repl(
439 self,
511 - text: list[str],
512 - videos: list[npt.NDArray],
513 - dynamic_image_size: bool | None = None,
514 - ) -> tuple[list[str], dict[str, Any]]:
515 - if len(videos) == 0 or not self.supports_video:
516 - return text, {}
440 + num_patches: int | None,
441 + num_features: int | None = None,
442 + ) -> PromptUpdateDetails[str]:
443 + if num_patches is None:
444 + assert num_features is not None
445 + else:
446 + num_features = num_patches * self.image_seq_length
447
518 - video_token = self.video_token
519 - assert video_token is not None
448 + repl_features = self.ctx_image_token * num_features
449 + repl_full = self.start_image_token + repl_features + self.end_image_token
450
521 - pixel_values_lst_video = self._videos_to_pixel_values_lst(
522 - videos,
523 - dynamic_image_size=dynamic_image_size,
524 - )
525 - video_inputs = {
526 - "pixel_values_flat_video": torch.cat(pixel_values_lst_video),
527 - "video_num_patches": torch.tensor(
528 - [len(item) for item in pixel_values_lst_video]
529 - ),
530 - }
451 + return PromptUpdateDetails.select_text(repl_full, self.ctx_image_token)
452
532 - for pixel_values in pixel_values_lst_video:
533 - num_patches = pixel_values.shape[0]
453 + def get_video_repl(self, num_patches: int) -> PromptUpdateDetails[str]:
454 + assert self.ctx_video_token is not None
455
535 - video_repl = self.get_video_repl(
536 - self.num_image_token, num_patches, video_token
537 - )
538 - text = [t.replace("<video>", video_repl.full, 1) for t in text]
539 - return text, video_inputs
456 + repl_features = self.ctx_video_token * self.image_seq_length
457 + repl_features_with_sep = (
458 + self.start_image_token + repl_features + self.end_image_token
459 + )
460 + # num_patches is equal to num_frames
461 + repl_full = "".join(
462 + [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
463 + )
464 +
465 + return PromptUpdateDetails.select_text(repl_full, self.ctx_video_token)
466
467 def __call__(
468 self,
@@ -550,54 +476,88 @@ class InternVLProcessor(BaseInternVLProcessor):
476 return_tensors: str | TensorType | None = None,
477 **kwargs,
478 ) -> BatchFeature:
553 - text = self._make_batch_input(text)
554 - images = self._make_batch_input(images)
555 - videos = self._make_batch_input(videos)
479 + if images is not None:
480 + image_inputs = self.image_processor(
481 + images=images,
482 + min_dynamic_patch=min_dynamic_patch,
483 + max_dynamic_patch=max_dynamic_patch,
484 + dynamic_image_size=dynamic_image_size,
485 + return_tensors=return_tensors,
486 + )
487 + image_num_patches = image_inputs["image_num_patches"]
488 + else:
489 + image_inputs = {}
490 + image_num_patches = []
491
557 - text, image_inputs = self._preprocess_image(
558 - text=text,
559 - images=images,
560 - min_dynamic_patch=min_dynamic_patch,
561 - max_dynamic_patch=max_dynamic_patch,
562 - dynamic_image_size=dynamic_image_size,
563 - )
492 + if videos is not None:
493 + if self.video_processor is None:
494 + raise ValueError("This model does not support video inputs")
495
565 - text, video_inputs = self._preprocess_video(
566 - text=text,
567 - videos=videos,
568 - dynamic_image_size=dynamic_image_size,
569 - )
496 + video_inputs = self.video_processor(
497 + videos=videos,
498 + return_tensors=return_tensors,
499 + )
500 + video_num_patches = video_inputs["video_num_patches"]
501 + else:
502 + video_inputs = {}
503 + video_num_patches = []
504
571 - text_inputs = self.tokenizer(text)
505 + if text is not None:
506 + if not isinstance(text, list):
507 + text = [text]
508
573 - combined_outputs = {**text_inputs, **image_inputs, **video_inputs}
509 + if image_inputs:
510 + image_token = "<image>"
511 + image_index = 0
512 + processed_text = list[str]()
513 + replace_strings = list[str]()
514
575 - return BatchFeature(combined_outputs, tensor_type=return_tensors)
515 + for prompt in text:
516 + new_prompt = prompt
517
577 - def get_image_repl(
578 - self,
579 - feature_size: int,
580 - num_patches: int | None,
581 - ) -> PromptUpdateDetails[str]:
582 - repl_features = IMG_CONTEXT * feature_size
583 - repl_full = IMG_START + repl_features + IMG_END
518 + while image_token in new_prompt:
519 + new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
520 + image_repl = self.get_image_repl(image_num_patches[image_index])
521 + replace_strings.append(image_repl.full)
522 + image_index += 1
523
585 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
524 + while "<placeholder>" in new_prompt:
525 + replace_str = replace_strings.pop(0)
526 + new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
527
587 - def get_video_repl(
588 - self,
589 - feature_size: int,
590 - num_patches: int | None,
591 - video_context_token: str = IMG_CONTEXT,
592 - ) -> PromptUpdateDetails[str]:
593 - if num_patches is None:
594 - raise NotImplementedError("Embedding inputs are not supported")
528 + processed_text.append(new_prompt)
529
596 - repl_features = video_context_token * self.num_image_token
597 - repl_features_with_sep = IMG_START + repl_features + IMG_END
598 - # num_patches is equal to num_frames
599 - repl_full = "".join(
600 - [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
601 - )
530 + text = processed_text
531 +
532 + if video_inputs:
533 + video_token = "<video>"
534 + video_index = 0
535 + processed_text = list[str]()
536 + replace_strings = list[str]()
537 +
538 + assert video_token is not None
539 +
540 + for prompt in text:
541 + new_prompt = prompt
542 +
543 + while video_token in new_prompt:
544 + new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
545 + video_repl = self.get_video_repl(video_num_patches[video_index])
546 + replace_strings.append(video_repl.full)
547 + video_index += 1
548 +
549 + while "<placeholder>" in new_prompt:
550 + replace_str = replace_strings.pop(0)
551 + new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
552
603 - return PromptUpdateDetails.select_text(repl_full, video_context_token)
553 + processed_text.append(new_prompt)
554 +
555 + text = processed_text
556 +
557 + text_inputs = self.tokenizer(text, return_tensors=return_tensors)
558 + else:
559 + text_inputs = {}
560 +
561 + combined_outputs = {**text_inputs, **image_inputs, **video_inputs}
562 +
563 + return BatchFeature(combined_outputs, tensor_type=return_tensors)
vllm/transformers_utils/processors/nano_nemotron_vl.py
+5 -5
@@ -25,7 +25,7 @@ from vllm.model_executor.models.parakeet import ParakeetExtractor
25 from vllm.multimodal.evs import compute_retained_tokens_count
26 from vllm.multimodal.inputs import AudioItem
27 from vllm.multimodal.processing.processor import PromptUpdateDetails, _seq2tokens
28 -from vllm.tokenizers import TokenizerLike
28 +from vllm.tokenizers.hf import HfTokenizer
29
30 from .internvl import calculate_internvl_targets, get_internvl_target_ratios
31
@@ -508,7 +508,7 @@ class BaseNanoNemotronVLProcessor(ABC):
508 def __init__(
509 self,
510 config: PretrainedConfig,
511 - tokenizer: TokenizerLike,
511 + tokenizer: HfTokenizer,
512 *args,
513 max_model_len: int,
514 max_num_tiles: int | None = None,
@@ -689,7 +689,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
689 def __init__(
690 self,
691 config: PretrainedConfig,
692 - tokenizer: TokenizerLike,
692 + tokenizer: HfTokenizer,
693 *,
694 max_model_len: int,
695 max_num_tiles: int | None = None,
@@ -961,7 +961,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
961 tokens_per_frame: list[int],
962 frames_indices: list[int],
963 frame_duration_ms: int,
964 - tokenizer: TokenizerLike,
964 + tokenizer: HfTokenizer,
965 img_start_token_ids: list[int],
966 img_end_token_ids: list[int],
967 img_context_token_ids: list[int],
@@ -986,7 +986,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
986 tokens_per_frame (list[int]): number of tokens per frame
987 frames_indices (list[int]): frame indices
988 frame_duration_ms (int): duration of each frame in milliseconds
989 - tokenizer (TokenizerLike): tokenizer to use for tokenizing frame separators
989 + tokenizer (HfTokenizer): tokenizer to use for tokenizing frame separators
990 img_start_token_ids (list[int]): pre-tokenized IMG_START tokens
991 img_end_token_ids (list[int]): pre-tokenized IMG_END tokens
992 img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens
vllm/transformers_utils/processors/nemotron_vl.py
+106 -172
@@ -1,18 +1,14 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from abc import ABC
3
4 import torch
5 import torchvision.transforms as T
6 from PIL import Image
8 -from transformers import PretrainedConfig
9 -from transformers.image_processing_utils_fast import BaseImageProcessorFast
7
8 from vllm.multimodal.image import convert_image_mode
12 -from vllm.multimodal.processing import PromptUpdateDetails
13 -from vllm.tokenizers import TokenizerLike
9 +from vllm.tokenizers.hf import HfTokenizer
10
15 -from .internvl import InternVLProcessor
11 +from .internvl import InternVLImageProcessor, InternVLProcessor
12
13 # Configure PIL to handle large images without warnings
14 # This prevents DecompressionBombWarning for legitimate large images
@@ -172,59 +168,61 @@ def image_to_pixel_values_nemotron_vl(
168 return pixel_values
169
170
175 -class NemotronVLProcessor(InternVLProcessor):
176 - IMG_START = "<img>"
177 - IMG_END = "</img>"
178 - IMG_CONTEXT = "<image>"
179 -
180 - def __init__(
171 +class LlamaNemotronNanoVLImageProcessor(InternVLImageProcessor):
172 + def _images_to_pixel_values_lst(
173 self,
182 - config: PretrainedConfig,
183 - tokenizer: TokenizerLike,
184 - image_processor: BaseImageProcessorFast,
185 - *,
174 + images: list[Image.Image],
175 min_dynamic_patch: int | None = None,
176 max_dynamic_patch: int | None = None,
177 dynamic_image_size: bool | None = None,
189 - ) -> None:
190 - ABC.__init__(self)
191 - self.config = config
192 - self.tokenizer = tokenizer
193 - self.image_processor = image_processor
194 - image_size: int = config.force_image_size
195 - patch_size: int = config.patch_size
196 -
197 - if min_dynamic_patch is None:
198 - min_dynamic_patch = 1
199 - assert isinstance(min_dynamic_patch, int)
200 -
201 - if max_dynamic_patch is None:
202 - max_dynamic_patch = self.image_processor.max_num_tiles
203 - assert isinstance(max_dynamic_patch, int)
204 -
205 - if dynamic_image_size is None:
206 - dynamic_image_size = True
207 - assert isinstance(dynamic_image_size, bool)
208 -
209 - self.num_image_token = int(
210 - (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
178 + ) -> list[torch.Tensor]:
179 + min_num, max_num = self.resolve_min_max_num(
180 + min_dynamic_patch=min_dynamic_patch,
181 + max_dynamic_patch=max_dynamic_patch,
182 + dynamic_image_size=dynamic_image_size,
183 + use_thumbnail=False, # Applied in image_to_pixel_values
184 )
212 - self.image_size = image_size
213 - self.min_dynamic_patch = min_dynamic_patch
214 - self.max_dynamic_patch = max_dynamic_patch
215 - self.dynamic_image_size = dynamic_image_size
185
217 - if image_processor is not None:
218 - self.use_thumbnail = image_processor.use_thumbnail
219 - else:
220 - self.use_thumbnail = getattr(config, "use_thumbnail", True)
186 + return [
187 + image_to_pixel_values_nemotron_vl(
188 + image,
189 + input_size=self.image_size,
190 + min_num=min_num,
191 + max_num=max_num,
192 + use_thumbnail=self.use_thumbnail,
193 + transform=build_transform(self.image_size),
194 + )
195 + for image in images
196 + ]
197 +
198 +
199 +class LlamaNemotronNanoVLProcessor(InternVLProcessor):
200 + """
201 + This model doesn't define its own HF processor,
202 + so we implement our own one here.
203
222 - @property
223 - def image_token_id(self) -> int:
224 - return self.tokenizer.get_vocab()[self.IMG_CONTEXT]
204 + The image processor is given by:
205 + https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1/blob/main/image_processing.py
206 + """
207
226 - def _get_transform(self) -> T.Compose:
227 - return build_transform(input_size=self.image_size)
208 + def __init__(
209 + self,
210 + image_processor: LlamaNemotronNanoVLImageProcessor,
211 + tokenizer: HfTokenizer,
212 + *,
213 + image_seq_length: int,
214 + start_image_token: str = "<img>",
215 + end_image_token: str = "</img>",
216 + ctx_image_token: str = "<image>",
217 + ) -> None:
218 + super().__init__(
219 + image_processor=image_processor,
220 + tokenizer=tokenizer,
221 + image_seq_length=image_seq_length,
222 + start_image_token=start_image_token,
223 + end_image_token=end_image_token,
224 + ctx_image_token=ctx_image_token,
225 + )
226
227 def get_num_image_tokens(
228 self,
@@ -232,6 +230,7 @@ class NemotronVLProcessor(InternVLProcessor):
230 image_width: int,
231 image_height: int,
232 ) -> int:
233 + image_processor = self.image_processor
234 target_ratios = self.resolve_target_ratios(
235 use_thumbnail=False, # Applied in calculate_targets
236 )
@@ -239,13 +238,33 @@ class NemotronVLProcessor(InternVLProcessor):
238 num_patches, _, _ = calculate_nemotron_vl_targets(
239 orig_width=image_width,
240 orig_height=image_height,
242 - image_size=self.image_size,
241 + image_size=image_processor.image_size,
242 target_ratios=target_ratios,
244 - use_thumbnail=self.use_thumbnail,
243 + use_thumbnail=image_processor.use_thumbnail,
244 )
245
247 - return num_patches * self.num_image_token
246 + return num_patches * self.image_seq_length
247 +
248
249 +# SigLIP normalization constants
250 +SIGLIP_MEAN = (0.5, 0.5, 0.5)
251 +SIGLIP_STD = (0.5, 0.5, 0.5)
252 +
253 +
254 +def build_siglip_transform(input_size: int):
255 + """Build transform for SigLIP vision encoder with normalization.
256 +
257 + Extends the base transform from nemotron_vl with SigLIP-specific normalization.
258 + """
259 + return T.Compose(
260 + [
261 + build_transform(input_size=input_size),
262 + T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD),
263 + ]
264 + )
265 +
266 +
267 +class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor):
268 def _images_to_pixel_values_lst(
269 self,
270 images: list[Image.Image],
@@ -267,83 +286,13 @@ class NemotronVLProcessor(InternVLProcessor):
286 min_num=min_num,
287 max_num=max_num,
288 use_thumbnail=self.use_thumbnail,
270 - transform=self._get_transform(),
289 + transform=build_siglip_transform(self.image_size),
290 )
291 for image in images
292 ]
293
275 - def _replace_image_tokens(
276 - self,
277 - text: list[str],
278 - pixel_values_lst: list[torch.Tensor],
279 - ) -> list[str]:
280 - """Replace <image> placeholders with image tokens."""
281 - for pixel_values in pixel_values_lst:
282 - num_patches = pixel_values.shape[0]
283 - feature_size = num_patches * self.num_image_token
284 - image_repl = self.get_image_repl(feature_size, num_patches)
285 - # Use temporary placeholder to avoid replacing tokens we just inserted
286 - NVL_IMAGE_CONTEXT = image_repl.full.replace("<image>", "<NVL_IMG_CONTEXT>")
287 - text = [t.replace("<image>", NVL_IMAGE_CONTEXT, 1) for t in text]
288 - return [t.replace("<NVL_IMG_CONTEXT>", self.IMG_CONTEXT) for t in text]
289 -
290 - def _preprocess_image(
291 - self,
292 - text: list[str],
293 - images: list[Image.Image],
294 - min_dynamic_patch: int | None = None,
295 - max_dynamic_patch: int | None = None,
296 - dynamic_image_size: bool | None = None,
297 - ) -> tuple[list[str], dict[str, torch.Tensor]]:
298 - if len(images) == 0:
299 - image_inputs = {}
300 - else:
301 - pixel_values_lst = self._images_to_pixel_values_lst(
302 - images,
303 - min_dynamic_patch=min_dynamic_patch,
304 - max_dynamic_patch=max_dynamic_patch,
305 - dynamic_image_size=dynamic_image_size,
306 - )
307 - image_inputs = {
308 - "pixel_values_flat": torch.cat(pixel_values_lst),
309 - "image_num_patches": torch.tensor(
310 - [len(item) for item in pixel_values_lst]
311 - ),
312 - }
313 -
314 - text = self._replace_image_tokens(text, pixel_values_lst)
315 - return text, image_inputs
316 -
317 - def get_image_repl(
318 - self,
319 - feature_size: int,
320 - num_patches: int | None,
321 - ) -> PromptUpdateDetails[str]:
322 - repl_features = self.IMG_CONTEXT * feature_size
323 - repl_full = self.IMG_START + repl_features + self.IMG_END
324 -
325 - return PromptUpdateDetails.select_text(repl_full, self.IMG_CONTEXT)
326 -
327 -
328 -# SigLIP normalization constants
329 -SIGLIP_MEAN = (0.5, 0.5, 0.5)
330 -SIGLIP_STD = (0.5, 0.5, 0.5)
294
332 -
333 -def build_siglip_transform(input_size: int):
334 - """Build transform for SigLIP vision encoder with normalization.
335 -
336 - Extends the base transform from nemotron_vl with SigLIP-specific normalization.
337 - """
338 - return T.Compose(
339 - [
340 - build_transform(input_size=input_size),
341 - T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD),
342 - ]
343 - )
344 -
345 -
346 -class LlamaNemotronVLEmbedProcessor(NemotronVLProcessor):
295 +class LlamaNemotronVLEmbedProcessor(InternVLProcessor):
296 """
297 Processor for LlamaNemotronVL embedding model.
298
@@ -352,59 +301,44 @@ class LlamaNemotronVLEmbedProcessor(NemotronVLProcessor):
301 - Uses different image context token (<IMG_CONTEXT> vs <image>)
302 """
303
355 - IMG_CONTEXT = "<IMG_CONTEXT>"
356 -
304 def __init__(
305 self,
359 - config: PretrainedConfig,
360 - tokenizer: TokenizerLike,
361 - processor_config: dict,
306 + image_processor: LlamaNemotronVLEmbedImageProcessor,
307 + tokenizer: HfTokenizer,
308 *,
363 - min_dynamic_patch: int | None = None,
364 - max_dynamic_patch: int | None = None,
365 - dynamic_image_size: bool | None = None,
309 + image_seq_length: int,
310 + start_image_token: str = "<img>",
311 + end_image_token: str = "</img>",
312 + ctx_image_token: str = "<IMG_CONTEXT>",
313 ) -> None:
367 - if min_dynamic_patch is None:
368 - min_dynamic_patch = processor_config.get(
369 - "min_input_tiles",
370 - getattr(config, "min_dynamic_patch", 1),
371 - )
372 - if max_dynamic_patch is None:
373 - max_dynamic_patch = processor_config.get(
374 - "max_input_tiles",
375 - getattr(config, "max_dynamic_patch", 1),
376 - )
377 - if dynamic_image_size is None:
378 - dynamic_image_size = processor_config.get(
379 - "dynamic_image_size",
380 - getattr(config, "dynamic_image_size", True),
381 - )
314 super().__init__(
383 - config=config,
315 + image_processor=image_processor,
316 tokenizer=tokenizer,
385 - image_processor=None,
386 - min_dynamic_patch=min_dynamic_patch,
387 - max_dynamic_patch=max_dynamic_patch,
388 - dynamic_image_size=dynamic_image_size,
317 + image_seq_length=image_seq_length,
318 + start_image_token=start_image_token,
319 + end_image_token=end_image_token,
320 + ctx_image_token=ctx_image_token,
321 )
322
391 - def _get_transform(self) -> T.Compose:
392 - """Override to add SigLIP normalization."""
393 - return build_siglip_transform(input_size=self.image_size)
323 + self.image_processor: LlamaNemotronVLEmbedImageProcessor
324
395 - def _replace_image_tokens(
325 + def get_num_image_tokens(
326 self,
397 - text: list[str],
398 - pixel_values_lst: list[torch.Tensor],
399 - ) -> list[str]:
400 - """Override with simpler token replacement for embedding model.
401 -
402 - No temporary placeholder needed because IMG_CONTEXT is <IMG_CONTEXT>,
403 - not <image>, so there's no collision risk.
404 - """
405 - for pixel_values in pixel_values_lst:
406 - num_patches = pixel_values.shape[0]
407 - feature_size = num_patches * self.num_image_token
408 - image_repl = self.get_image_repl(feature_size, num_patches)
409 - text = [t.replace("<image>", image_repl.full, 1) for t in text]
410 - return text
327 + *,
328 + image_width: int,
329 + image_height: int,
330 + ) -> int:
331 + image_processor = self.image_processor
332 + target_ratios = self.resolve_target_ratios(
333 + use_thumbnail=False, # Applied in calculate_targets
334 + )
335 +
336 + num_patches, _, _ = calculate_nemotron_vl_targets(
337 + orig_width=image_width,
338 + orig_height=image_height,
339 + image_size=image_processor.image_size,
340 + target_ratios=target_ratios,
341 + use_thumbnail=image_processor.use_thumbnail,
342 + )
343 +
344 + return num_patches * self.image_seq_length
vllm/transformers_utils/processors/nvlm_d.py
+30 -13
@@ -8,37 +8,54 @@
8 # Licensed under Apache 2.0 License [see LICENSE for details]
9 # --------------------------------------------------------
10 from vllm.multimodal.processing import PromptUpdateDetails
11 +from vllm.tokenizers.hf import HfTokenizer
12
12 -from .internvl import BaseInternVLProcessor
13 +from .internvl import InternVLImageProcessor, InternVLProcessor
14
14 -IMG_PAD = "<|vision_pad|>"
15
16 -
17 -class NVLMProcessor(BaseInternVLProcessor):
18 - @property
19 - def image_token_id(self) -> int:
20 - return self.tokenizer.get_vocab()[IMG_PAD]
16 +class NVLMProcessor(InternVLProcessor):
17 + def __init__(
18 + self,
19 + image_processor: InternVLImageProcessor,
20 + tokenizer: HfTokenizer,
21 + *,
22 + image_seq_length: int,
23 + start_image_token: str = "<Image>",
24 + end_image_token: str = "</Image>",
25 + ctx_image_token: str = "<|vision_pad|>",
26 + ) -> None:
27 + super().__init__(
28 + image_processor=image_processor,
29 + tokenizer=tokenizer,
30 + image_seq_length=image_seq_length,
31 + start_image_token=start_image_token,
32 + end_image_token=end_image_token,
33 + ctx_image_token=ctx_image_token,
34 + )
35
36 def get_image_repl(
37 self,
24 - feature_size: int,
38 num_patches: int | None,
39 + num_features: int | None = None,
40 ) -> PromptUpdateDetails[str]:
41 if num_patches is None:
42 raise NotImplementedError("Embedding inputs are not supported")
43
44 + num_features = num_patches * self.image_seq_length
45 +
46 tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
31 - if self.use_thumbnail:
47 + if self.image_processor.use_thumbnail:
48 tile_pos_identifiers += ["<tile_global_thumbnail>"]
49
34 - context_size = feature_size // num_patches
50 + context_size = num_features // num_patches
51 features = "".join(
36 - identifier + IMG_PAD * context_size for identifier in tile_pos_identifiers
52 + (identifier + self.ctx_image_token * context_size)
53 + for identifier in tile_pos_identifiers
54 )
55
56 # We include the start and end as well because "<Image><tile" is
57 # tokenized as ["<Image", "><", "tile"], resulting in assertion error
58 # when trying to find "<tile" as a subsequence of "<Image><tile"
42 - repl = "<Image>" + features + "</Image>"
59 + repl = self.start_image_token + features + self.end_image_token
60
44 - return PromptUpdateDetails.select_text(repl, IMG_PAD)
61 + return PromptUpdateDetails.select_text(repl, self.ctx_image_token)
vllm/transformers_utils/processors/skyworkr1v.py deleted
-389
@@ -1,389 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py
5 -# --------------------------------------------------------
6 -# SkyworkR1V
7 -# Copyright (c) 2025 Skywork
8 -# Licensed under The MIT License [see LICENSE for details]
9 -# --------------------------------------------------------
10 -
11 -import torch
12 -import torchvision.transforms as T
13 -from PIL import Image
14 -from transformers import BatchFeature, PretrainedConfig, TensorType
15 -
16 -from vllm.multimodal.image import convert_image_mode
17 -from vllm.multimodal.processing import PromptUpdateDetails
18 -from vllm.tokenizers import TokenizerLike
19 -
20 -IMG_START = "<img>"
21 -IMG_END = "</img>"
22 -IMG_CONTEXT = "<IMG_CONTEXT>"
23 -
24 -IMAGENET_MEAN = (0.485, 0.456, 0.406)
25 -IMAGENET_STD = (0.229, 0.224, 0.225)
26 -
27 -
28 -# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
29 -def build_transform(input_size: int):
30 - MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
31 - return T.Compose(
32 - [
33 - T.Lambda(lambda img: convert_image_mode(img, "RGB")),
34 - T.Resize(
35 - (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
36 - ),
37 - T.ToTensor(),
38 - T.Normalize(mean=MEAN, std=STD),
39 - ]
40 - )
41 -
42 -
43 -# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
44 -def find_closest_aspect_ratio(
45 - aspect_ratio: float,
46 - target_ratios: list[tuple[int, int]],
47 - *,
48 - width: int,
49 - height: int,
50 - image_size: int,
51 -) -> tuple[int, int]:
52 - best_ratio_diff = float("inf")
53 - best_ratio = (1, 1)
54 - area = width * height
55 - for ratio in target_ratios:
56 - target_aspect_ratio = ratio[0] / ratio[1]
57 - ratio_diff = abs(aspect_ratio - target_aspect_ratio)
58 - if ratio_diff < best_ratio_diff:
59 - best_ratio_diff = ratio_diff
60 - best_ratio = ratio
61 - elif ratio_diff == best_ratio_diff:
62 - if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
63 - best_ratio = ratio
64 - return best_ratio
65 -
66 -
67 -def resolve_skyworkr1v_min_max_num(
68 - *,
69 - min_dynamic_patch: int,
70 - max_dynamic_patch: int,
71 - dynamic_image_size: bool,
72 - use_thumbnail: bool,
73 -) -> tuple[int, int]:
74 - min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
75 - max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
76 -
77 - if use_thumbnail and max_dynamic_patch != 1:
78 - max_dynamic_patch += 1
79 -
80 - return min_dynamic_patch, max_dynamic_patch
81 -
82 -
83 -def get_skyworkr1v_target_ratios(
84 - min_num: int,
85 - max_num: int,
86 -) -> list[tuple[int, int]]:
87 - target_ratios = {
88 - (i, j)
89 - for n in range(min_num, max_num + 1)
90 - for i in range(1, n + 1)
91 - for j in range(1, n + 1)
92 - if min_num <= i * j <= max_num
93 - }
94 - return sorted(target_ratios, key=lambda x: x[0] * x[1])
95 -
96 -
97 -def calculate_skyworkr1v_targets(
98 - *,
99 - orig_width: int,
100 - orig_height: int,
101 - target_ratios: list[tuple[int, int]],
102 - image_size: int,
103 - use_thumbnail: bool,
104 -) -> tuple[int, int, int]:
105 - aspect_ratio = orig_width / orig_height
106 -
107 - # find the closest aspect ratio to the target
108 - target_aspect_ratio = find_closest_aspect_ratio(
109 - aspect_ratio,
110 - target_ratios,
111 - width=orig_width,
112 - height=orig_height,
113 - image_size=image_size,
114 - )
115 -
116 - # calculate the target width and height
117 - target_width = image_size * target_aspect_ratio[0]
118 - target_height = image_size * target_aspect_ratio[1]
119 - blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
120 -
121 - # add thumbnail image if num_blocks != 1
122 - if use_thumbnail and blocks != 1:
123 - blocks += 1
124 -
125 - return blocks, target_width, target_height
126 -
127 -
128 -def dynamic_preprocess_skyworkr1v(
129 - image: Image.Image,
130 - *,
131 - target_ratios: list[tuple[int, int]],
132 - image_size: int,
133 - use_thumbnail: bool,
134 -) -> list[Image.Image]:
135 - orig_width, orig_height = image.size
136 -
137 - # calculate the number of blocks without thumbnail
138 - blocks, target_width, target_height = calculate_skyworkr1v_targets(
139 - orig_width=orig_width,
140 - orig_height=orig_height,
141 - target_ratios=target_ratios,
142 - image_size=image_size,
143 - use_thumbnail=False,
144 - )
145 -
146 - # resize the image
147 - resized_img = image.resize((target_width, target_height))
148 - processed_images = []
149 - for i in range(blocks):
150 - box = (
151 - (i % (target_width // image_size)) * image_size,
152 - (i // (target_width // image_size)) * image_size,
153 - ((i % (target_width // image_size)) + 1) * image_size,
154 - ((i // (target_width // image_size)) + 1) * image_size,
155 - )
156 - # split the image
157 - split_img = resized_img.crop(box)
158 - processed_images.append(split_img)
159 -
160 - assert len(processed_images) == blocks
161 -
162 - if use_thumbnail and len(processed_images) != 1:
163 - thumbnail_img = image.resize((image_size, image_size))
164 - processed_images.append(thumbnail_img)
165 -
166 - return processed_images
167 -
168 -
169 -# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B
170 -def image_to_pixel_values_skyworkr1v(
171 - image: Image.Image,
172 - *,
173 - input_size: int,
174 - min_num: int,
175 - max_num: int,
176 - use_thumbnail: bool,
177 -) -> torch.Tensor:
178 - target_ratios = get_skyworkr1v_target_ratios(min_num, max_num)
179 -
180 - transform = build_transform(input_size=input_size)
181 - images = dynamic_preprocess_skyworkr1v(
182 - image,
183 - target_ratios=target_ratios,
184 - image_size=input_size,
185 - use_thumbnail=use_thumbnail,
186 - )
187 -
188 - pixel_values = torch.stack([transform(image) for image in images])
189 - return pixel_values
190 -
191 -
192 -class SkyworkR1VProcessor:
193 - """
194 - This model doesn't define its own HF processor,
195 - so we implement our own one here.
196 -
197 - The code to insert image tokens is based on:
198 - https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py#L252
199 - """
200 -
201 - def __init__(
202 - self,
203 - config: PretrainedConfig,
204 - tokenizer: TokenizerLike,
205 - *,
206 - min_dynamic_patch: int | None = None,
207 - max_dynamic_patch: int | None = None,
208 - dynamic_image_size: bool | None = None,
209 - ) -> None:
210 - super().__init__()
211 -
212 - self.config = config
213 - self.tokenizer = tokenizer
214 -
215 - image_size: int = config.vision_config.image_size
216 - patch_size: int = config.vision_config.patch_size
217 -
218 - if min_dynamic_patch is None:
219 - min_dynamic_patch = config.min_dynamic_patch
220 - assert isinstance(min_dynamic_patch, int)
221 -
222 - if max_dynamic_patch is None:
223 - max_dynamic_patch = config.max_dynamic_patch
224 - assert isinstance(max_dynamic_patch, int)
225 -
226 - if dynamic_image_size is None:
227 - dynamic_image_size = config.dynamic_image_size
228 - assert isinstance(dynamic_image_size, bool)
229 -
230 - self.num_image_token = int(
231 - (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
232 - )
233 - self.image_size = image_size
234 - self.min_dynamic_patch = min_dynamic_patch
235 - self.max_dynamic_patch = max_dynamic_patch
236 - self.dynamic_image_size = dynamic_image_size
237 - self.use_thumbnail: bool = config.use_thumbnail
238 -
239 - @property
240 - def image_token_id(self) -> int:
241 - return self.tokenizer.get_vocab()[IMG_CONTEXT]
242 -
243 - def get_image_repl(
244 - self,
245 - feature_size: int,
246 - num_patches: int | None,
247 - ) -> PromptUpdateDetails[str]:
248 - repl_features = IMG_CONTEXT * feature_size
249 - repl_full = IMG_START + repl_features + IMG_END
250 -
251 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
252 -
253 - def resolve_min_max_num(
254 - self,
255 - *,
256 - min_dynamic_patch: int | None = None,
257 - max_dynamic_patch: int | None = None,
258 - dynamic_image_size: bool | None = None,
259 - use_thumbnail: bool | None = None,
260 - ) -> tuple[int, int]:
261 - min_dynamic_patch = (
262 - self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
263 - )
264 - max_dynamic_patch = (
265 - self.max_dynamic_patch if max_dynamic_patch is None else max_dynamic_patch
266 - )
267 - dynamic_image_size = (
268 - self.dynamic_image_size
269 - if dynamic_image_size is None
270 - else dynamic_image_size
271 - )
272 - use_thumbnail = self.use_thumbnail if use_thumbnail is None else use_thumbnail
273 -
274 - return resolve_skyworkr1v_min_max_num(
275 - min_dynamic_patch=min_dynamic_patch,
276 - max_dynamic_patch=max_dynamic_patch,
277 - dynamic_image_size=dynamic_image_size,
278 - use_thumbnail=use_thumbnail,
279 - )
280 -
281 - def resolve_target_ratios(
282 - self,
283 - *,
284 - min_dynamic_patch: int | None = None,
285 - max_dynamic_patch: int | None = None,
286 - dynamic_image_size: bool | None = None,
287 - use_thumbnail: bool | None = None,
288 - ) -> list[tuple[int, int]]:
289 - min_num, max_num = self.resolve_min_max_num(
290 - min_dynamic_patch=min_dynamic_patch,
291 - max_dynamic_patch=max_dynamic_patch,
292 - dynamic_image_size=dynamic_image_size,
293 - use_thumbnail=use_thumbnail,
294 - )
295 -
296 - return get_skyworkr1v_target_ratios(min_num, max_num)
297 -
298 - def get_num_image_tokens(
299 - self,
300 - *,
301 - image_width: int,
302 - image_height: int,
303 - ) -> int:
304 - target_ratios = self.resolve_target_ratios(
305 - use_thumbnail=False, # Applied in calculate_targets
306 - )
307 -
308 - num_patches, _, _ = calculate_skyworkr1v_targets(
309 - orig_width=image_width,
310 - orig_height=image_height,
311 - image_size=self.image_size,
312 - target_ratios=target_ratios,
313 - use_thumbnail=self.use_thumbnail,
314 - )
315 -
316 - return num_patches * self.num_image_token
317 -
318 - def _images_to_pixel_values_lst(
319 - self,
320 - images: list[Image.Image],
321 - min_dynamic_patch: int | None = None,
322 - max_dynamic_patch: int | None = None,
323 - dynamic_image_size: bool | None = None,
324 - ) -> list[torch.Tensor]:
325 - min_num, max_num = self.resolve_min_max_num(
326 - min_dynamic_patch=min_dynamic_patch,
327 - max_dynamic_patch=max_dynamic_patch,
328 - dynamic_image_size=dynamic_image_size,
329 - use_thumbnail=False, # Applied in image_to_pixel_values
330 - )
331 -
332 - return [
333 - image_to_pixel_values_skyworkr1v(
334 - image,
335 - input_size=self.image_size,
336 - min_num=min_num,
337 - max_num=max_num,
338 - use_thumbnail=self.use_thumbnail,
339 - )
340 - for image in images
341 - ]
342 -
343 - def __call__(
344 - self,
345 - text: str | list[str] | None = None,
346 - images: Image.Image | list[Image.Image] | None = None,
347 - min_dynamic_patch: int | None = None,
348 - max_dynamic_patch: int | None = None,
349 - dynamic_image_size: bool | None = None,
350 - return_tensors: str | TensorType | None = None,
351 - ) -> BatchFeature:
352 - if text is None:
353 - text = []
354 - if not isinstance(text, list):
355 - text = [text]
356 - if images is None:
357 - images = []
358 - if not isinstance(images, list):
359 - images = [images]
360 -
361 - if len(images) == 0:
362 - image_inputs = {}
363 - else:
364 - pixel_values_lst = self._images_to_pixel_values_lst(
365 - images,
366 - min_dynamic_patch=min_dynamic_patch,
367 - max_dynamic_patch=max_dynamic_patch,
368 - dynamic_image_size=dynamic_image_size,
369 - )
370 - image_inputs = {
371 - "pixel_values_flat": torch.cat(pixel_values_lst),
372 - "image_num_patches": torch.tensor(
373 - [len(item) for item in pixel_values_lst]
374 - ),
375 - }
376 -
377 - for pixel_values in pixel_values_lst:
378 - num_patches = pixel_values.shape[0]
379 - feature_size = num_patches * self.num_image_token
380 -
381 - image_repl = self.get_image_repl(feature_size, num_patches)
382 -
383 - text = [t.replace("<image>", image_repl.full, 1) for t in text]
384 -
385 - text_inputs = self.tokenizer(text)
386 -
387 - combined_outputs = {**text_inputs, **image_inputs}
388 -
389 - return BatchFeature(combined_outputs, tensor_type=return_tensors)