A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
[Misc] Cleanup more configs and processors (#37560)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 19, 2026 at 23:45 UTC
657855ab417988834a4d0ff99de27fd66c6a6b3c
27 files changed
+1062
-1141
vllm/config/speculative.py
+3
-1
@@ -520,8 +520,10 @@ class SpeculativeConfig:
520
521
# Replace hf_config for EAGLE draft_model
522
if self.method in ("eagle", "eagle3"):
523
- from vllm.transformers_utils.configs import SpeculatorsConfig
523
from vllm.transformers_utils.configs.eagle import EAGLEConfig
524
+ from vllm.transformers_utils.configs.speculators import (
525
+ SpeculatorsConfig,
526
+ )
527
528
if isinstance(
529
self.draft_model_config.hf_config,
vllm/model_executor/models/chatglm.py
+1
-1
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
)
33
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.sequence import IntermediateTensors
35
-from vllm.transformers_utils.configs import ChatGLMConfig
35
+from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
36
37
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
38
from .utils import (
vllm/model_executor/models/falcon.py
+1
-1
@@ -54,7 +54,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
54
)
55
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56
from vllm.sequence import IntermediateTensors
57
-from vllm.transformers_utils.configs import RWConfig
57
+from vllm.transformers_utils.configs.falcon import RWConfig
58
59
from .interfaces import SupportsPP
60
from .utils import (
vllm/model_executor/models/flex_olmo.py
+1
-1
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.fused_moe import FusedMoE
24
from vllm.model_executor.layers.layernorm import RMSNorm
25
from vllm.model_executor.layers.linear import ReplicatedLinear
26
from vllm.model_executor.models.olmoe import OlmoeAttention, OlmoeForCausalLM
27
-from vllm.transformers_utils.configs import FlexOlmoConfig
27
+from vllm.transformers_utils.configs.flex_olmo import FlexOlmoConfig
28
29
logger = init_logger(__name__)
30
vllm/model_executor/models/hyperclovax_vision.py
+2
-26
@@ -20,7 +20,6 @@ from vllm.config import VllmConfig
20
from vllm.config.multimodal import BaseDummyOptions
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.multimodal import MULTIMODAL_REGISTRY
23
-from vllm.multimodal.cache import BaseMultiModalProcessorCache
23
from vllm.multimodal.inputs import (
24
MultiModalDataDict,
25
MultiModalFieldConfig,
@@ -31,7 +30,6 @@ from vllm.multimodal.processing import (
30
BaseDummyInputsBuilder,
31
BaseMultiModalProcessor,
32
BaseProcessingInfo,
34
- InputProcessingContext,
33
PromptReplacement,
34
PromptUpdate,
35
)
@@ -336,28 +334,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
334
return fields
335
336
339
-def _build_hcxvision_hf_info(
340
- ctx: InputProcessingContext,
341
-) -> HCXVisionProcessingInfo:
342
- return HCXVisionProcessingInfo(ctx)
343
-
344
-
345
-def _build_hcxvision_hf_processor(
346
- info: HCXVisionProcessingInfo,
347
- dummy_inputs: BaseDummyInputsBuilder[HCXVisionProcessingInfo],
348
- *,
349
- cache: BaseMultiModalProcessorCache | None = None,
350
-) -> BaseMultiModalProcessor:
351
- if isinstance(info, HCXVisionProcessingInfo):
352
- return HCXVisionMultiModalProcessor(
353
- info,
354
- dummy_inputs, # type: ignore
355
- cache=cache,
356
- )
357
-
358
- raise NotImplementedError(type(info))
359
-
360
-
337
def init_vision_tower_for_hcxvision(
338
vision_config,
339
quant_config: QuantizationConfig | None,
@@ -587,8 +563,8 @@ class HCXVisionCAbstractor(nn.Module):
563
564
565
@MULTIMODAL_REGISTRY.register_processor(
590
- _build_hcxvision_hf_processor,
591
- info=_build_hcxvision_hf_info,
566
+ HCXVisionMultiModalProcessor,
567
+ info=HCXVisionProcessingInfo,
568
dummy_inputs=HCXVisionDummyInputsBuilder,
569
)
570
class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
vllm/model_executor/models/isaac.py
+23
-480
@@ -2,22 +2,17 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from __future__ import annotations
4
5
-import math
5
from collections.abc import Iterable, Iterator, Mapping, Sequence
6
from typing import Annotated, Any
7
8
import numpy as np
10
-import PIL.Image
9
import torch
10
import torch.nn as nn
11
import torch.nn.functional as F
12
from einops import rearrange
13
from transformers.image_processing_utils import BatchFeature
16
-from transformers.utils import TensorType
17
-from typing_extensions import TypedDict, Unpack
14
19
-from vllm.config import VllmConfig
20
-from vllm.config.model import ModelConfig
15
+from vllm.config import ModelConfig, VllmConfig
16
from vllm.config.multimodal import BaseDummyOptions
17
from vllm.distributed import parallel_state
18
from vllm.distributed import utils as dist_utils
@@ -64,13 +59,17 @@ from vllm.multimodal.processing import (
59
PromptUpdateDetails,
60
)
61
from vllm.sequence import IntermediateTensors
67
-from vllm.tokenizers import get_tokenizer
68
-from vllm.tokenizers.hf import get_cached_tokenizer
62
+from vllm.tokenizers import cached_tokenizer_from_config
63
from vllm.transformers_utils.config import patch_rope_parameters
70
-from vllm.transformers_utils.configs import (
64
+from vllm.transformers_utils.configs.isaac import (
65
IsaacConfig,
66
PixelShuffleSiglip2VisionConfig,
67
)
68
+from vllm.transformers_utils.processors.isaac import (
69
+ IsaacImageProcessor,
70
+ IsaacProcessor,
71
+ get_image_size_for_max_num_patches,
72
+)
73
from vllm.utils.tensor_schema import TensorSchema, TensorShape
74
75
from .vision import is_vit_use_data_parallel
@@ -307,467 +306,6 @@ def pixel_shuffle_varlen(
306
# Configuration
307
# ============================================================================
308
310
-MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px
311
-
312
-# Vision preprocessing constants
313
-VISION_MEAN = (0.5, 0.5, 0.5)
314
-VISION_STD = (0.5, 0.5, 0.5)
315
-VISION_SCALE = 1 / 255
316
-
317
-
318
-def _make_writeable(arr: np.ndarray) -> np.ndarray:
319
- """Return *arr* itself if it is already writeable, otherwise try to flip the
320
- write flag in-place and finally fall back to `arr.copy()`.
321
- This guarantees the buffer handed to `torch.from_numpy()` is always
322
- writeable, silencing the PyTorch warning about undefined behaviour.
323
- """
324
- if arr.flags.writeable:
325
- return arr
326
-
327
- # First, try the cheap path — in-place flag toggle (works for mmap'd arrays
328
- # and some shared memory buffers):
329
- try:
330
- arr.setflags(write=True)
331
- return arr # success: no data copy
332
- except ValueError:
333
- # Buffer is inherently read-only (e.g. backed by PyAV / PIL): make copy
334
- return arr.copy()
335
-
336
-
337
-def extract_image_pil(image: PIL.Image.Image) -> torch.Tensor | None:
338
- if image.width * image.height > MAX_PIXELS:
339
- raise ValueError(
340
- f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`"
341
- )
342
- img = image if image.mode == "RGB" else image.convert("RGB")
343
- arr = np.asarray(img)
344
- arr = _make_writeable(arr)
345
- return torch.from_numpy(arr)
346
-
347
-
348
-def get_image_size_for_max_num_patches(
349
- image_height: int,
350
- image_width: int,
351
- patch_size: int,
352
- max_num_patches: int,
353
- min_num_patches: int | None = None,
354
- eps: float = 1e-5,
355
- pixel_shuffle_scale: int = 1,
356
-) -> tuple[int, int]:
357
- r"""Compute a target resolution whose patch grid satisfies patching parametrization.
358
-
359
- Args:
360
- image_height (`int`):
361
- Height in pixels of the source image prior to any resizing.
362
- image_width (`int`):
363
- Width in pixels of the source image prior to any resizing.
364
- patch_size (`int`):
365
- Size of the square patch used by the vision encoder.
366
- max_num_patches (`int`):
367
- Upper bound on `(height / patch_size) * (width / patch_size)` after
368
- resizing.
369
- min_num_patches (`int`, *optional*):
370
- Lower bound on the number of patches. When provided the image will
371
- be scaled up if necessary.
372
- eps (`float`, *optional*, defaults to 1e-5):
373
- Convergence tolerance for the internal binary search to determine
374
- the target dimensions.
375
- pixel_shuffle_scale (`int`, *optional*, defaults to 1):
376
- Additional stride multiplier applied when pixel shuffle later
377
- reduces spatial resolution.
378
-
379
- Returns:
380
- `tuple[int, int]`: Height and width (in pixels) that are multiples of
381
- `patch_size * pixel_shuffle_scale` and respect both the maximum and
382
- optional minimum patch-count constraints.
383
- """
384
-
385
- def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale):
386
- scaled_size = scale * original_size
387
- divisor = patch_size * pixel_shuffle_scale
388
- scaled_size = math.ceil(scaled_size / divisor) * divisor
389
- scaled_size = max(divisor, scaled_size)
390
- return int(scaled_size)
391
-
392
- # Ensure divisibility
393
- divisor = patch_size * pixel_shuffle_scale
394
- adjusted_height = math.ceil(image_height / divisor) * divisor
395
- adjusted_height = max(divisor, adjusted_height)
396
- adjusted_width = math.ceil(image_width / divisor) * divisor
397
- adjusted_width = max(divisor, adjusted_width)
398
-
399
- num_patches = (adjusted_height / patch_size) * (adjusted_width / patch_size)
400
-
401
- if min_num_patches is not None and num_patches < min_num_patches:
402
- # Scale up
403
- scale_min, scale_max = 1.0, 100.0
404
- while (scale_max - scale_min) >= eps:
405
- scale = (scale_min + scale_max) / 2
406
- target_height = get_scaled_image_size(
407
- scale, image_height, patch_size, pixel_shuffle_scale
408
- )
409
- target_width = get_scaled_image_size(
410
- scale, image_width, patch_size, pixel_shuffle_scale
411
- )
412
- num_patches = (target_height / patch_size) * (target_width / patch_size)
413
- if num_patches >= min_num_patches:
414
- scale_max = scale
415
- else:
416
- scale_min = scale
417
- scale = scale_max
418
- target_height = get_scaled_image_size(
419
- scale, image_height, patch_size, pixel_shuffle_scale
420
- )
421
- target_width = get_scaled_image_size(
422
- scale, image_width, patch_size, pixel_shuffle_scale
423
- )
424
- return target_height, target_width
425
- elif num_patches <= max_num_patches:
426
- return adjusted_height, adjusted_width
427
- else:
428
- # Scale down
429
- scale_min, scale_max = eps / 10, 1.0
430
- while (scale_max - scale_min) >= eps:
431
- scale = (scale_min + scale_max) / 2
432
- target_height = get_scaled_image_size(
433
- scale, image_height, patch_size, pixel_shuffle_scale
434
- )
435
- target_width = get_scaled_image_size(
436
- scale, image_width, patch_size, pixel_shuffle_scale
437
- )
438
- num_patches = (target_height / patch_size) * (target_width / patch_size)
439
- if num_patches <= max_num_patches:
440
- scale_min = scale
441
- else:
442
- scale_max = scale
443
- scale = scale_min
444
- target_height = get_scaled_image_size(
445
- scale, image_height, patch_size, pixel_shuffle_scale
446
- )
447
- target_width = get_scaled_image_size(
448
- scale, image_width, patch_size, pixel_shuffle_scale
449
- )
450
- return target_height, target_width
451
-
452
-
453
-_MEAN_TENSOR = torch.tensor(VISION_MEAN, dtype=torch.float32).view(1, 1, 1, -1)
454
-_STD_TENSOR = torch.tensor(VISION_STD, dtype=torch.float32).view(1, 1, 1, -1)
455
-
456
-
457
-def _resolve_vision_token_id(model_config: ModelConfig, vision_token: str) -> int:
458
- tokenizer_name = model_config.tokenizer or model_config.model
459
- tokenizer = get_cached_tokenizer(
460
- get_tokenizer(
461
- tokenizer_name,
462
- tokenizer_mode=model_config.tokenizer_mode,
463
- trust_remote_code=model_config.trust_remote_code,
464
- revision=model_config.tokenizer_revision or model_config.revision,
465
- )
466
- )
467
- return tokenizer.encode(vision_token, add_special_tokens=False)[0]
468
-
469
-
470
-def prepare_image_tensor(
471
- image: torch.Tensor,
472
- scale: float = VISION_SCALE,
473
-) -> torch.Tensor:
474
- r"""Standardize RGB images prior to patch extraction via rescaling and whitening.
475
-
476
- Args:
477
- image (`torch.Tensor`):
478
- Tensor with shape `(..., height, width, 3)` containing RGB values.
479
- The tensor is converted to floating point if needed.
480
- scale (`float`, *optional*, defaults to `VISION_SCALE`):
481
- Scalar multiplier applied before normalization.
482
- Returns:
483
- `torch.Tensor`: Normalized tensor with the same shape as the input and
484
- dtype `torch.float32`.
485
- """
486
- if not torch.is_floating_point(image):
487
- image = image.float()
488
- rescaled = image * scale
489
-
490
- # Use precomputed tensors and move to the correct device if needed
491
- mean_tensor = _MEAN_TENSOR.to(image.device)
492
- std_tensor = _STD_TENSOR.to(image.device)
493
-
494
- normalized = (rescaled - mean_tensor) / std_tensor
495
- return normalized
496
-
497
-
498
-def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor:
499
- r"""Convert normalized images into flattened ViT-style patches.
500
-
501
- Args:
502
- image (`torch.Tensor`):
503
- Tensor of shape `(num_images, height, width, channels)`.
504
- patch_size (`int`):
505
- Edge length of the square patches
506
-
507
- Returns:
508
- `torch.Tensor`:
509
- Patch tensor where each position stores the flattened pixels
510
- belonging to that patch.
511
-
512
- Raises:
513
- ValueError: If `height` or `width` is not divisible by `patch_size`.
514
- """
515
- num_images, height, width, channels = image.shape
516
- if height % patch_size or width % patch_size:
517
- raise ValueError(
518
- "Dimensions of images "
519
- f"{image.shape} are not divisible by patch_size={patch_size}."
520
- )
521
- patches = image.reshape(
522
- num_images,
523
- height // patch_size,
524
- patch_size,
525
- width // patch_size,
526
- patch_size,
527
- channels,
528
- )
529
- patches = patches.permute(0, 1, 3, 2, 4, 5)
530
- patches = patches.reshape(
531
- num_images,
532
- height // patch_size,
533
- width // patch_size,
534
- channels * patch_size * patch_size,
535
- )
536
- return patches
537
-
538
-
539
-def process_vision_for_patches(
540
- images: torch.Tensor,
541
- patch_size: int,
542
- max_num_patches: int,
543
- min_num_patches: int | None = None,
544
- pixel_shuffle_scale: int = 1,
545
-) -> tuple[torch.Tensor, list[int]]:
546
- r"""Resize, normalize, and patchify RGB images for the vision encoder.
547
-
548
- Args:
549
- images (`torch.Tensor`):
550
- Either `(height, width, channels)` for a single image or
551
- `(num_images, height, width, channels)` for a batch. Channels are
552
- expected to be RGB.
553
- patch_size (`int`):
554
- Edge length of square patches; implicitly controls resize grid granularity.
555
- max_num_patches (`int`):
556
- Maximum number of patches allowed after resizing.
557
- min_num_patches (`int`, *optional*):
558
- Minimum number of patches. If provided, the routine upsamples images
559
- as needed to satisfy the lower bound.
560
- pixel_shuffle_scale (`int`, *optional*, defaults to 1):
561
- Pixel shuffle scale factor; influences the target grid that the
562
- function produces.
563
-
564
- Returns:
565
- `tuple[torch.Tensor, list[int]]`: A pair `(patches, dims_virtual)`
566
- where `patches` has shape `(num_images, target_h / patch_size, target_w
567
- / patch_size, channels * patch_size**2)` and `dims_virtual` encodes
568
- effective `(images, height, width)` dimensions after optional pixel
569
- shuffling.
570
- """
571
- # Add batch dim if single image
572
- if images.dim() == 3:
573
- images = images.unsqueeze(0)
574
-
575
- # Permute to channel first for resize
576
- images = images.permute(0, 3, 1, 2)
577
-
578
- # Get target dimensions
579
- _, _, orig_height, orig_width = images.shape
580
- target_height, target_width = get_image_size_for_max_num_patches(
581
- orig_height,
582
- orig_width,
583
- patch_size,
584
- max_num_patches,
585
- min_num_patches=min_num_patches,
586
- pixel_shuffle_scale=pixel_shuffle_scale,
587
- )
588
-
589
- # Resize
590
- images = F.interpolate(
591
- images,
592
- size=(target_height, target_width),
593
- mode="bilinear",
594
- align_corners=False,
595
- )
596
-
597
- # Back to channel last
598
- images = images.permute(0, 2, 3, 1)
599
-
600
- # Normalize
601
- images = prepare_image_tensor(images)
602
-
603
- # Patchify
604
- patches = patchify_vision(images, patch_size=patch_size)
605
-
606
- # Calculate dimensions for the patches
607
- n_images, h_patches, w_patches, _ = patches.shape
608
- dims_virtual = (
609
- [1, h_patches, w_patches]
610
- if pixel_shuffle_scale == 1
611
- else [1, h_patches // pixel_shuffle_scale, w_patches // pixel_shuffle_scale]
612
- )
613
-
614
- return patches, dims_virtual
615
-
616
-
617
-class IsaacImageProcessorKwargs(TypedDict, total=False):
618
- patch_size: int
619
- max_num_patches: int
620
- min_num_patches: int
621
- pixel_shuffle_scale: int
622
-
623
-
624
-class IsaacImageProcessor:
625
- patch_size = 16
626
- max_num_patches = 6144
627
- min_num_patches = 256
628
- pixel_shuffle_scale = 2
629
-
630
- valid_kwargs = IsaacImageProcessorKwargs
631
- model_input_names = ["pixel_values", "image_grid_thw"]
632
-
633
- def __init__(self, kwargs):
634
- self.patch_size = kwargs.pop("patch_size", self.patch_size)
635
- self.vision_max_num_patches = kwargs.pop(
636
- "vision_max_num_patches", self.max_num_patches
637
- )
638
- self.vision_min_num_patches = kwargs.pop(
639
- "vision_min_num_patches", self.min_num_patches
640
- )
641
- self.pixel_shuffle_scale = kwargs.pop("pixel_shuffle_scale", 2)
642
-
643
- def preprocess(
644
- self,
645
- images: list[torch.Tensor],
646
- return_tensors: str | TensorType | None,
647
- **kwargs: Unpack[IsaacImageProcessorKwargs],
648
- ) -> BatchFeature:
649
- """Preprocess images into format compatible with vLLM input processing."""
650
-
651
- all_pixel_values: list[torch.Tensor] = []
652
- all_image_grids: list[torch.Tensor] = []
653
-
654
- for image in images:
655
- image_tensor = extract_image_pil(image)
656
-
657
- patches, dims_virtual = process_vision_for_patches(
658
- image_tensor,
659
- patch_size=self.patch_size,
660
- max_num_patches=self.vision_max_num_patches,
661
- min_num_patches=self.vision_min_num_patches,
662
- pixel_shuffle_scale=self.pixel_shuffle_scale,
663
- )
664
-
665
- # Isaac packs a dummy temporal dim for images
666
- patches = patches.unsqueeze(1) # [N, T=1, Hp, Wp, D]
667
-
668
- hp, wp, dim = patches.shape[-3], patches.shape[-2], patches.shape[-1]
669
- current_num_patches = hp * wp
670
- pixel_values = patches.reshape(current_num_patches, dim) # [N_tokens, D]
671
-
672
- # Use real patch dimensions for image_grid_thw, not virtual dimensions
673
- # This ensures the vision model receives correct grid info for pixel shuffle
674
- dims_real = [1, hp, wp] # Real patch dimensions
675
- image_grid_thw = torch.tensor(dims_real).unsqueeze(0)
676
-
677
- all_pixel_values.append(pixel_values)
678
- all_image_grids.append(image_grid_thw)
679
-
680
- if all_pixel_values:
681
- final_pixel_values = torch.cat(all_pixel_values, dim=0)
682
- final_image_grids = torch.cat(all_image_grids, dim=0)
683
- else:
684
- final_pixel_values = torch.empty(0, 0)
685
- final_image_grids = torch.empty(0, 3)
686
-
687
- return BatchFeature(
688
- data={
689
- "pixel_values": final_pixel_values,
690
- "image_grid_thw": final_image_grids,
691
- },
692
- tensor_type=return_tensors,
693
- )
694
-
695
-
696
-class IsaacProcessor:
697
- """Processor wrapper (tokenizer + IsaacImageProcessor)."""
698
-
699
- def __init__(self, image_processor=None, tokenizer=None, **kwargs):
700
- self.image_token = kwargs.pop("image_token", "<image>")
701
- self.image_processor = image_processor or IsaacImageProcessor(kwargs)
702
- self.tokenizer = tokenizer
703
-
704
- def __call__(self, text=None, images=None, **kwargs) -> BatchFeature:
705
- result = {}
706
-
707
- if images is not None:
708
- image_inputs = self.image_processor.preprocess(images, **kwargs)
709
- image_grid_thw = image_inputs["image_grid_thw"]
710
- result.update(image_inputs)
711
-
712
- if text is not None:
713
- if not isinstance(text, list):
714
- text = [text]
715
-
716
- text = text.copy() # below lines change text in-place
717
- merge_length = self.image_processor.pixel_shuffle_scale**2
718
- index = 0
719
- for i in range(len(text)):
720
- while self.image_token in text[i]:
721
- num_image_tokens = image_grid_thw[index].prod() // merge_length
722
- text[i] = text[i].replace(
723
- self.image_token, "<|placeholder|>" * num_image_tokens, 1
724
- )
725
- index += 1
726
- text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
727
-
728
- if text is not None:
729
- result.update(self.tokenizer(text, **kwargs))
730
-
731
- return BatchFeature(result)
732
-
733
- def apply_chat_template(
734
- self,
735
- messages: list[dict[str, Any]],
736
- tokenize: bool = False,
737
- add_generation_prompt: bool = False,
738
- **kwargs,
739
- ) -> Any:
740
- # Convert mixed content messages to simple text format
741
- processed_messages = []
742
-
743
- for message in messages:
744
- if "content" in message and isinstance(message["content"], list):
745
- # Handle mixed content (text + image)
746
- text_parts = []
747
- for content_item in message["content"]:
748
- if content_item.get("type") == "text":
749
- text_parts.append(content_item.get("text", ""))
750
- elif content_item.get("type") == "image":
751
- # Replace image with vision token
752
- text_parts.append(self.image_token)
753
-
754
- processed_message = {
755
- "role": message.get("role", "user"),
756
- "content": "".join(text_parts),
757
- }
758
- processed_messages.append(processed_message)
759
- else:
760
- # Regular text message
761
- processed_messages.append(message)
762
-
763
- kwargs["return_dict"] = False
764
- return self.tokenizer.apply_chat_template(
765
- processed_messages,
766
- tokenize=tokenize,
767
- add_generation_prompt=add_generation_prompt,
768
- **kwargs,
769
- )
770
-
309
310
class IsaacProcessingInfo(BaseProcessingInfo):
311
def get_hf_config(self) -> IsaacConfig:
@@ -795,16 +333,18 @@ class IsaacProcessingInfo(BaseProcessingInfo):
333
)
334
return IsaacConfig()
335
336
+ def get_image_processor(self, **kwargs) -> IsaacImageProcessor:
337
+ return IsaacImageProcessor(kwargs)
338
+
339
def get_hf_processor(self, **kwargs) -> IsaacProcessor:
340
hf_config = self.get_hf_config()
800
- processor_kwargs = {
801
- "image_token": hf_config.vision_token,
802
- }
803
- processor_kwargs.update(kwargs)
804
- return self.ctx.get_hf_processor(IsaacProcessor, **processor_kwargs)
341
806
- def get_tokenizer(self):
807
- return self.ctx.tokenizer
342
+ return self.ctx.init_processor(
343
+ IsaacProcessor,
344
+ tokenizer=self.get_tokenizer(),
345
+ image_processor=self.get_image_processor(),
346
+ image_token=hf_config.vision_token,
347
+ )
348
349
def get_image_size_with_most_features(self) -> ImageSize:
350
hf_config = self.get_hf_config()
@@ -819,9 +359,6 @@ class IsaacProcessingInfo(BaseProcessingInfo):
359
)
360
return ImageSize(width=target_width, height=target_height)
361
822
- def get_image_processor(self, **kwargs) -> IsaacImageProcessor:
823
- return self.get_hf_processor(**kwargs).image_processor
824
-
362
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
363
return {"image": None}
364
@@ -1206,6 +743,12 @@ class Siglip2VisionTransformer(nn.Module):
743
return loaded_params
744
745
746
+def _resolve_vision_token_id(model_config: ModelConfig, vision_token: str) -> int:
747
+ tokenizer = cached_tokenizer_from_config(model_config)
748
+ assert tokenizer is not None
749
+ return tokenizer.encode(vision_token, add_special_tokens=False)[0]
750
+
751
+
752
class IsaacVisionEmbedding(nn.Module):
753
def __init__(
754
self,
vllm/model_executor/models/jais.py
+1
-1
@@ -49,7 +49,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
49
)
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
from vllm.sequence import IntermediateTensors
52
-from vllm.transformers_utils.configs import JAISConfig
52
+from vllm.transformers_utils.configs.jais import JAISConfig
53
54
from .interfaces import SupportsPP
55
from .utils import (
vllm/model_executor/models/kimi_k25.py
+14
-81
@@ -1,14 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# ruff: noqa: E501
3
"""
4
Kimi-K2.5 Model Implementation for vLLM.
5
7
-Kimi-K2.5 extends Kimi-K2 with vision support
8
-
9
-This module defines:
10
-- KimiK25ProcessingInfo/KimiK25MultiModalProcessor: Processing logic
11
-- KimiK25ForConditionalGeneration: Main model class
6
+Kimi-K2.5 extends Kimi-K2 with vision support.
7
"""
8
9
from collections.abc import Iterable, Mapping, Sequence
@@ -18,14 +13,13 @@ from typing import Annotated, Any, Literal
13
import torch
14
from torch import nn
15
from transformers import BatchFeature
21
-from transformers.processing_utils import ProcessorMixin
16
17
from vllm.config import VllmConfig
18
from vllm.config.multimodal import BaseDummyOptions
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.quantization import QuantizationConfig
27
-from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import (
28
- CompressedTensorsConfig,
21
+from vllm.model_executor.layers.quantization.compressed_tensors import (
22
+ compressed_tensors,
23
)
24
from vllm.model_executor.models.interfaces import (
25
SupportsEagle,
@@ -45,7 +39,6 @@ from vllm.multimodal.inputs import (
39
MultiModalFieldConfig,
40
MultiModalKwargsItems,
41
NestedTensors,
48
- VisionChunk,
42
VisionChunkImage,
43
VisionChunkVideo,
44
)
@@ -60,8 +53,9 @@ from vllm.multimodal.processing import (
53
)
54
from vllm.platforms import current_platform
55
from vllm.sequence import IntermediateTensors
63
-from vllm.transformers_utils.configs import KimiK25Config
56
+from vllm.transformers_utils.configs.kimi_k25 import KimiK25Config
57
from vllm.transformers_utils.processor import cached_get_image_processor
58
+from vllm.transformers_utils.processors.kimi_k25 import KimiK25Processor
59
from vllm.utils.tensor_schema import TensorSchema, TensorShape
60
61
from .utils import (
@@ -101,69 +95,6 @@ class KimiK25MediaPixelInputs(TensorSchema):
95
grid_thws: Annotated[torch.Tensor, TensorShape("nm", 3)]
96
97
104
-class MoonshotKimiVAutoProcessor(ProcessorMixin):
105
- attributes = ["tokenizer"]
106
- tokenizer_class = "AutoTokenizer"
107
-
108
- def __init__(
109
- self, media_processor=None, tokenizer=None, media_token_id: int | None = None
110
- ):
111
- super().__init__(tokenizer)
112
- self.media_processor = media_processor
113
- self.media_token_id = media_token_id
114
- assert self.media_token_id is not None
115
-
116
- # We do not support str input for text here
117
- def __call__(
118
- self,
119
- vision_chunks: list[VisionChunk] | None = None,
120
- *,
121
- text: list[int] | str,
122
- **kwargs,
123
- ) -> BatchFeature:
124
- """
125
- Args:
126
- vision_chunks: List of VisionChunk items to be processed.
127
- For image: VisionChunkImage with type='image', image=PIL.Image
128
- For video_chunk: VisionChunkVideo with type='video_chunk', video_chunk=list[PIL.Image]
129
- text: The token ids to be fed to a model (required).
130
- Returns:
131
- [`BatchFeature`]: A [`BatchFeature`] with the following fields:
132
-
133
- - **input_ids** -- list of token ids to be fed to a model.
134
- - **pixel_values** -- Pixel values to be fed to a model. Returned when `vision_chunks` is not `None`.
135
- - **grid_thws** -- list of image 3D grid in LLM. Returned when `vision_chunks` is not `None`.
136
- """
137
- mm_inputs = {}
138
- input_ids = self.tokenizer.encode(text) if isinstance(text, str) else text
139
- if vision_chunks is not None:
140
- assert isinstance(vision_chunks, list)
141
- mm_inputs = self.media_processor.preprocess(vision_chunks)
142
-
143
- num_tokens_per_chunk = [
144
- self.media_processor.media_tokens_calculator(chunk)
145
- for chunk in vision_chunks
146
- ]
147
-
148
- new_input_ids = []
149
- for token in input_ids:
150
- if token == self.media_token_id:
151
- new_input_ids.extend(
152
- [self.media_token_id] * num_tokens_per_chunk.pop(0)
153
- )
154
- else:
155
- new_input_ids.append(token)
156
- input_ids = new_input_ids
157
-
158
- # XXX: _apply_hf_processor_text_mm will call tolist() on input_ids
159
- return BatchFeature(
160
- data={
161
- "input_ids": torch.tensor([input_ids]),
162
- **mm_inputs,
163
- }
164
- )
165
-
166
-
98
class KimiK25ProcessingInfo(BaseProcessingInfo):
99
"""Processing information for Kimi-K2.5 model.
100
@@ -180,7 +111,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
111
trust_remote_code=self.ctx.model_config.trust_remote_code,
112
)
113
self.media_processor = media_processor
183
- self.hf_processor = MoonshotKimiVAutoProcessor(
114
+ self.hf_processor = KimiK25Processor(
115
media_processor=self.media_processor,
116
tokenizer=self.get_tokenizer(),
117
media_token_id=self.media_token_id,
@@ -263,12 +194,14 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
194
) -> Mapping[str, MultiModalFieldConfig]:
195
"""Indicates how to slice media input into multiple items.
196
266
- pixel_values: [N, 3, patch_size, patch_size], all patches collected from B medias
267
- grid_thws: [B,3], each item: [N_t, N_h ,N_w], indicates the grid size in time/height/width direction
268
- for current item.
197
+ pixel_values: [N, 3, patch_size, patch_size],
198
+ all patches collected from B medias
199
+ grid_thws: [B,3], each item: [N_t, N_h ,N_w],
200
+ indicates the grid size in time/height/width direction for current item.
201
270
- by multiplying [N_t, N_h ,N_w], we get the number of patches for each media item, thus we can slice
271
- pixel_values by pixel_values[start:start + N_t*N_h*N_w] to get patches of one item.
202
+ by multiplying [N_t, N_h ,N_w], we get the number of patches
203
+ for each media item, thus we can slice pixel_values by
204
+ pixel_values[start:start + N_t*N_h*N_w] to get patches of one item.
205
206
"""
207
grid_thws = hf_inputs.get("grid_thws", torch.empty((0, 3)))
@@ -403,7 +336,7 @@ class KimiK25ForConditionalGeneration(
336
self.media_placeholder: int = self.config.media_placeholder_token_id
337
338
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
406
- if isinstance(quant_config, CompressedTensorsConfig):
339
+ if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
340
return None
341
return quant_config
342
vllm/model_executor/models/kimi_vl.py
+1
-1
@@ -77,7 +77,7 @@ from vllm.multimodal.processing import (
77
PromptUpdate,
78
)
79
from vllm.sequence import IntermediateTensors
80
-from vllm.transformers_utils.configs import KimiVLConfig, MoonViTConfig
80
+from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig, MoonViTConfig
81
from vllm.utils.tensor_schema import TensorSchema, TensorShape
82
83
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
vllm/model_executor/models/lfm2_moe.py
+1
-1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39
)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.sequence import IntermediateTensors
42
-from vllm.transformers_utils.configs import Lfm2MoeConfig
42
+from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
43
44
from .interfaces import (
45
HasInnerState,
vllm/model_executor/models/lightonocr.py
+4
-17
@@ -16,8 +16,7 @@ from vllm.model_executor.models.mistral3 import (
16
Mistral3ForConditionalGeneration,
17
Mistral3MultiModalProjector,
18
Mistral3ProcessingInfo,
19
- _build_mistral3_info,
20
- init_vision_tower_for_llava,
19
+ init_vision_tower_for_mistral3,
20
)
21
from vllm.model_executor.models.pixtral import PixtralHFEncoderInfo
22
from vllm.model_executor.models.utils import (
@@ -27,11 +26,9 @@ from vllm.model_executor.models.utils import (
26
maybe_prefix,
27
)
28
from vllm.multimodal import MULTIMODAL_REGISTRY
30
-from vllm.multimodal.cache import BaseMultiModalProcessorCache
29
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
30
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
31
from vllm.multimodal.processing import (
34
- BaseDummyInputsBuilder,
32
BaseMultiModalProcessor,
33
PromptReplacement,
34
PromptUpdate,
@@ -128,19 +125,9 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
125
]
126
127
131
-def _build_LightOnOCR_processor(
132
- info: _I,
133
- dummy_inputs: BaseDummyInputsBuilder[_I],
134
- *,
135
- cache: BaseMultiModalProcessorCache | None = None,
136
-):
137
- assert isinstance(info, Mistral3ProcessingInfo)
138
- return LightOnOCRMultiModalProcessor(info, dummy_inputs, cache=cache)
139
-
140
-
128
@MULTIMODAL_REGISTRY.register_processor(
142
- _build_LightOnOCR_processor,
143
- info=_build_mistral3_info,
129
+ LightOnOCRMultiModalProcessor,
130
+ info=Mistral3ProcessingInfo,
131
dummy_inputs=Mistral3DummyInputsBuilder,
132
)
133
class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
@@ -164,7 +151,7 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
151
self.multimodal_config = multimodal_config
152
153
with self._mark_tower_model(vllm_config, "image"):
167
- self.vision_tower = init_vision_tower_for_llava(
154
+ self.vision_tower = init_vision_tower_for_mistral3(
155
config,
156
quant_config=quant_config,
157
require_post_norm=False,
vllm/model_executor/models/mistral3.py
+13
-63
@@ -1,18 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from abc import abstractmethod
4
from collections.abc import Iterable, Mapping, Sequence
6
-from typing import Annotated, Final, Literal, Protocol, TypeVar
5
+from typing import Annotated, Literal
6
7
import torch
8
import torch.nn as nn
10
-from transformers import (
11
- BatchFeature,
12
- Mistral3Config,
13
- PixtralVisionConfig,
14
- PretrainedConfig,
15
-)
9
+from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
10
from transformers.models.pixtral import PixtralProcessor
11
12
from vllm.config import VllmConfig
@@ -23,7 +17,6 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelL
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.models.module_mapping import MultiModelKeys
19
from vllm.multimodal import MULTIMODAL_REGISTRY
26
-from vllm.multimodal.cache import BaseMultiModalProcessorCache
20
from vllm.multimodal.inputs import (
21
MultiModalDataDict,
22
MultiModalFieldConfig,
@@ -34,7 +27,6 @@ from vllm.multimodal.processing import (
27
BaseDummyInputsBuilder,
28
BaseMultiModalProcessor,
29
BaseProcessingInfo,
37
- InputProcessingContext,
30
PromptReplacement,
31
PromptUpdate,
32
PromptUpdateDetails,
@@ -178,27 +170,15 @@ class Mistral3MultiModalProjector(nn.Module):
170
return hidden_states
171
172
181
-class LlavaLikeConfig(Protocol):
182
- vision_config: Final[PretrainedConfig]
183
- image_token_index: Final[int]
184
- vision_feature_select_strategy: Final[str]
185
- vision_feature_layer: Final[int | list[int]]
186
-
187
-
188
-class LlavaLikeProcessor(Protocol):
189
- image_token: Final[str]
190
-
191
-
192
-class BaseLlavaProcessingInfo(BaseProcessingInfo):
193
- def get_hf_config(self) -> LlavaLikeConfig:
173
+class Mistral3ProcessingInfo(BaseProcessingInfo):
174
+ def get_hf_config(self) -> Mistral3Config:
175
return self.ctx.get_hf_config(Mistral3Config)
176
177
def get_vision_encoder_info(self):
178
return get_vision_encoder_info(self.get_hf_config())
179
199
- @abstractmethod
200
- def get_hf_processor(self, **kwargs: object) -> LlavaLikeProcessor:
201
- raise NotImplementedError
180
+ def get_hf_processor(self, **kwargs: object):
181
+ return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
182
183
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
184
return {"image": None}
@@ -221,10 +201,7 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
201
return ImageSize(width=width, height=height)
202
203
224
-_I = TypeVar("_I", bound=BaseLlavaProcessingInfo)
225
-
226
-
227
-class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
204
+class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]):
205
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
206
num_images = mm_counts.get("image", 0)
207
@@ -255,11 +232,6 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
232
}
233
234
258
-class Mistral3ProcessingInfo(BaseLlavaProcessingInfo):
259
- def get_hf_processor(self, **kwargs: object):
260
- return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
261
-
262
-
235
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
236
def _call_hf_processor(
237
self,
@@ -339,29 +311,7 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo
311
]
312
313
342
-def _build_mistral3_info(
343
- ctx: InputProcessingContext,
344
-) -> BaseLlavaProcessingInfo:
345
- hf_config = ctx.get_hf_config(Mistral3Config)
346
- assert isinstance(hf_config.vision_config, PixtralVisionConfig)
347
- return Mistral3ProcessingInfo(ctx)
348
-
349
-
350
-def _build_mistral3_processor(
351
- info: _I,
352
- dummy_inputs: BaseDummyInputsBuilder[_I],
353
- *,
354
- cache: BaseMultiModalProcessorCache | None = None,
355
-) -> BaseMultiModalProcessor:
356
- assert isinstance(info, Mistral3ProcessingInfo)
357
- return Mistral3MultiModalProcessor(
358
- info,
359
- dummy_inputs, # type: ignore
360
- cache=cache,
361
- )
362
-
363
-
364
-def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
314
+def _get_num_hidden_layers(hf_config: Mistral3Config) -> int:
315
"""Determine the number of hidden layers to initialize up to in the
316
visual encoder.
317
@@ -381,8 +331,8 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
331
)
332
333
384
-def init_vision_tower_for_llava(
385
- hf_config: LlavaLikeConfig,
334
+def init_vision_tower_for_mistral3(
335
+ hf_config: Mistral3Config,
336
quant_config: QuantizationConfig | None,
337
*,
338
require_post_norm: bool | None = None,
@@ -405,8 +355,8 @@ def init_vision_tower_for_llava(
355
356
357
@MULTIMODAL_REGISTRY.register_processor(
408
- _build_mistral3_processor,
409
- info=_build_mistral3_info,
358
+ Mistral3MultiModalProcessor,
359
+ info=Mistral3ProcessingInfo,
360
dummy_inputs=Mistral3DummyInputsBuilder,
361
)
362
class Mistral3ForConditionalGeneration(
@@ -466,7 +416,7 @@ class Mistral3ForConditionalGeneration(
416
config.projector_hidden_act = "gelu"
417
418
with self._mark_tower_model(vllm_config, "image"):
469
- self.vision_tower = init_vision_tower_for_llava(
419
+ self.vision_tower = init_vision_tower_for_mistral3(
420
config,
421
quant_config=quant_config,
422
require_post_norm=False,
vllm/model_executor/models/nemotron.py
+1
-1
@@ -52,7 +52,7 @@ from vllm.model_executor.model_loader.weight_utils import (
52
maybe_remap_kv_scale_name,
53
)
54
from vllm.sequence import IntermediateTensors
55
-from vllm.transformers_utils.configs import NemotronConfig
55
+from vllm.transformers_utils.configs.nemotron import NemotronConfig
56
57
from .interfaces import SupportsLoRA, SupportsPP
58
from .utils import (
vllm/model_executor/models/nemotron_h.py
+1
-1
@@ -81,7 +81,7 @@ from vllm.model_executor.models.utils import (
81
sequence_parallel_chunk,
82
)
83
from vllm.sequence import IntermediateTensors
84
-from vllm.transformers_utils.configs import NemotronHConfig
84
+from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
85
86
87
class NemotronHMLP(nn.Module):
vllm/model_executor/models/nemotron_h_mtp.py
+1
-1
@@ -26,7 +26,7 @@ from vllm.model_executor.models.utils import (
26
maybe_prefix,
27
)
28
from vllm.sequence import IntermediateTensors
29
-from vllm.transformers_utils.configs import NemotronHConfig
29
+from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
30
31
from .interfaces import SupportsPP
32
from .nemotron_h import (
vllm/model_executor/models/olmo2.py
+1
-1
@@ -63,7 +63,7 @@ from vllm.model_executor.models.utils import (
63
maybe_prefix,
64
)
65
from vllm.sequence import IntermediateTensors
66
-from vllm.transformers_utils.configs import Olmo3Config
66
+from vllm.transformers_utils.configs.olmo3 import Olmo3Config
67
68
69
class Olmo2Attention(nn.Module):
vllm/model_executor/models/qwen3_next.py
+1
-1
@@ -80,7 +80,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk
80
from vllm.model_executor.utils import set_weight_attrs
81
from vllm.platforms import current_platform
82
from vllm.sequence import IntermediateTensors
83
-from vllm.transformers_utils.configs import Qwen3NextConfig
83
+from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
84
from vllm.triton_utils import tl, triton
85
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
86
from vllm.utils.torch_utils import (
vllm/model_executor/models/qwen3_next_mtp.py
+1
-1
@@ -25,7 +25,7 @@ from vllm.model_executor.models.qwen3_next import (
25
QwenNextMixtureOfExperts,
26
)
27
from vllm.sequence import IntermediateTensors
28
-from vllm.transformers_utils.configs import Qwen3NextConfig
28
+from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
29
30
from .utils import (
31
AutoWeightsLoader,
vllm/model_executor/models/step3_vl.py
+4
-433
@@ -2,18 +2,13 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import math
4
from collections.abc import Iterable, Mapping, Sequence
5
-from itertools import product
6
-from math import ceil, sqrt
5
+from math import sqrt
6
from typing import Annotated, Any, Literal, TypeAlias
7
9
-import numpy as np
8
import torch
9
import torch.nn as nn
10
import torch.nn.functional as F
13
-from PIL import Image
14
-from torchvision import transforms
15
-from torchvision.transforms.functional import InterpolationMode
16
-from transformers import BatchFeature, PretrainedConfig, TensorType
11
+from transformers import BatchFeature
12
13
from vllm.config import VllmConfig
14
from vllm.config.multimodal import BaseDummyOptions
@@ -43,8 +38,8 @@ from vllm.multimodal.processing import (
38
PromptUpdateDetails,
39
)
40
from vllm.sequence import IntermediateTensors
46
-from vllm.tokenizers import TokenizerLike
47
-from vllm.transformers_utils.configs import Step3VisionEncoderConfig
41
+from vllm.transformers_utils.configs.step3_vl import Step3VisionEncoderConfig
42
+from vllm.transformers_utils.processors.step3_vl import Step3VLProcessor
43
from vllm.utils.tensor_schema import TensorSchema, TensorShape
44
45
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -89,430 +84,6 @@ class Step3VLImageEmbeddingInputs(TensorSchema):
84
85
Step3VLImageInputs: TypeAlias = Step3VLImagePixelInputs | Step3VLImageEmbeddingInputs
86
92
-ImageWithPatches = tuple[Image.Image, list[Image.Image], list[bool] | None]
93
-
94
-MAX_IMAGE_SIZE: int = 3024
95
-
96
-
97
-class Step3VisionProcessor:
98
- def __init__(self, size, interpolation_mode="bicubic", patch_size=None):
99
- mean = [0.48145466, 0.4578275, 0.40821073]
100
- std = [0.26862954, 0.26130258, 0.27577711]
101
- patch_size = patch_size if patch_size is not None else size
102
-
103
- self.transform = transforms.Compose(
104
- [
105
- transforms.ToTensor(),
106
- transforms.Normalize(mean, std),
107
- transforms.Resize(
108
- (size, size),
109
- interpolation=InterpolationMode.BICUBIC
110
- if interpolation_mode == "bicubic"
111
- else InterpolationMode.BILINEAR,
112
- antialias=True,
113
- ),
114
- ]
115
- )
116
-
117
- self.patch_transform = (
118
- transforms.Compose(
119
- [
120
- transforms.ToTensor(),
121
- transforms.Normalize(mean, std),
122
- transforms.Resize(
123
- (patch_size, patch_size),
124
- interpolation=InterpolationMode.BICUBIC
125
- if interpolation_mode == "bicubic"
126
- else InterpolationMode.BILINEAR,
127
- antialias=True,
128
- ),
129
- ]
130
- )
131
- if patch_size is not None
132
- else None
133
- )
134
-
135
- def __call__(self, image, is_patch=False):
136
- if is_patch:
137
- return {"pixel_values": self.patch_transform(image).unsqueeze(0)}
138
- else:
139
- return {"pixel_values": self.transform(image).unsqueeze(0)}
140
-
141
-
142
-class ImagePatcher:
143
- def __init__(self, enable_patch: bool = True) -> None:
144
- self.enable_patch = enable_patch
145
-
146
- def determine_window_size(self, long: int, short: int) -> int:
147
- if long < 728:
148
- return short if long / short > 1.5 else 0
149
- return min(short, 504) if long / short > 4 else 504
150
-
151
- def slide_window(
152
- self,
153
- width: int,
154
- height: int,
155
- sizes: list[tuple[int, int]],
156
- steps: list[tuple[int, int]],
157
- img_rate_thr: float = 0.6,
158
- ) -> tuple[list[tuple[int, int, int, int]], tuple[int, int]]:
159
- assert 1 >= img_rate_thr >= 0, "The `in_rate_thr` should lie in 0~1"
160
- windows = []
161
- # Sliding windows.
162
- for size, step in zip(sizes, steps):
163
- size_w, size_h = size
164
- step_w, step_h = step
165
-
166
- x_num = 1 if width <= size_w else ceil((width - size_w) / step_w + 1)
167
- x_start = [step_w * i for i in range(x_num)]
168
- if len(x_start) > 1 and x_start[-1] + size_w > width:
169
- x_start[-1] = width - size_w
170
-
171
- y_num = 1 if height <= size_h else ceil((height - size_h) / step_h + 1)
172
- y_start = [step_h * i for i in range(y_num)]
173
- if len(y_start) > 1 and y_start[-1] + size_h > height:
174
- y_start[-1] = height - size_h
175
-
176
- start = np.array(list(product(y_start, x_start)), dtype=int)
177
- start[:, [0, 1]] = start[:, [1, 0]]
178
- windows.append(np.concatenate([start, start + size], axis=1))
179
- windows = np.concatenate(windows, axis=0)
180
-
181
- return [
182
- (int(box[0]), int(box[1]), int(box[2] - box[0]), int(box[3] - box[1]))
183
- for box in windows
184
- ], (x_num, y_num)
185
-
186
- def square_pad(self, img: Image.Image) -> Image.Image:
187
- w, h = img.size
188
- if w == h:
189
- return img
190
- size = max(w, h)
191
- padded = Image.new(img.mode, (size, size), 0)
192
- padded.paste(img, (0, 0))
193
- return padded
194
-
195
- def get_image_size_for_padding(
196
- self, img_width: int, img_height: int
197
- ) -> tuple[int, int]:
198
- ratio = img_width / img_height
199
- if min(img_height, img_width) < 32 and (ratio > 4 or ratio < 1 / 4):
200
- new_size = max(img_height, img_width)
201
- return new_size, new_size
202
- return img_width, img_height
203
-
204
- def get_image_size_for_preprocess(
205
- self, img_width: int, img_height: int
206
- ) -> tuple[int, int]:
207
- if max(img_height, img_width) > MAX_IMAGE_SIZE:
208
- scale_factor = MAX_IMAGE_SIZE / max(img_height, img_width)
209
- img_width = int(img_width * scale_factor)
210
- img_height = int(img_height * scale_factor)
211
- return img_width, img_height
212
-
213
- def get_image_size_for_crop(
214
- self, img_width: int, img_height: int, window_size: int
215
- ):
216
- w_ratio = img_width / window_size
217
- h_ratio = img_height / window_size
218
-
219
- if w_ratio < 1:
220
- width_new = img_width
221
- else:
222
- decimal_w = w_ratio - img_width // window_size
223
- w_ratio = int(w_ratio) + 1 if decimal_w > 0.2 else int(w_ratio)
224
- width_new = window_size * w_ratio
225
- if h_ratio < 1:
226
- height_new = img_height
227
- else:
228
- decimal_h = h_ratio - img_height // window_size
229
- h_ratio = int(h_ratio) + 1 if decimal_h > 0.2 else int(h_ratio)
230
- height_new = window_size * h_ratio
231
- return int(width_new), int(height_new)
232
-
233
- def patch_crop(self, img: Image.Image, i: int, j: int, th: int, tw: int):
234
- target = img.crop((j, i, j + tw, i + th))
235
- return target
236
-
237
- def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]:
238
- img_width, img_height = self.get_image_size_for_padding(img_width, img_height)
239
- img_width, img_height = self.get_image_size_for_preprocess(
240
- img_width, img_height
241
- )
242
- window_size = self.determine_window_size(
243
- max(img_height, img_width), min(img_height, img_width)
244
- )
245
- if window_size == 0 or not self.enable_patch:
246
- return 0, 0
247
- else:
248
- img_width, img_height = self.get_image_size_for_crop(
249
- img_width, img_height, window_size
250
- )
251
- center_list, (x_num, y_num) = self.slide_window(
252
- img_width,
253
- img_height,
254
- [(window_size, window_size)],
255
- [(window_size, window_size)],
256
- )
257
- full_rows = (len(center_list) - 1) // x_num + 1
258
- if len(center_list) > 0 and len(center_list) % x_num == 0:
259
- full_rows -= 1
260
- return len(center_list), full_rows
261
-
262
- def __call__(
263
- self, img: Image.Image
264
- ) -> tuple[Image.Image, list[Image.Image], list[bool] | None]:
265
- img_width, img_height = img.size
266
- new_img_width, new_img_height = self.get_image_size_for_padding(
267
- img_width, img_height
268
- )
269
- if new_img_width != img_width or new_img_height != img_height:
270
- img = self.square_pad(img)
271
- img_width, img_height = img.size
272
-
273
- new_img_width, new_img_height = self.get_image_size_for_preprocess(
274
- img_width, img_height
275
- )
276
- img = img.resize((new_img_width, new_img_height), Image.Resampling.BILINEAR)
277
- window_size = self.determine_window_size(
278
- max(new_img_height, new_img_width), min(new_img_height, new_img_width)
279
- )
280
-
281
- if window_size == 0 or not self.enable_patch:
282
- return img, [], None
283
- else:
284
- new_img_width, new_img_height = self.get_image_size_for_crop(
285
- new_img_width, new_img_height, window_size
286
- )
287
- if (new_img_width, new_img_height) != (img_width, img_height):
288
- img_for_crop = img.resize(
289
- (new_img_width, new_img_height), Image.Resampling.BILINEAR
290
- )
291
- else:
292
- img_for_crop = img
293
-
294
- patches = []
295
- newlines = []
296
- center_list, (x_num, y_num) = self.slide_window(
297
- new_img_width,
298
- new_img_height,
299
- [(window_size, window_size)],
300
- [(window_size, window_size)],
301
- )
302
- for patch_id, center_lf_point in enumerate(center_list):
303
- x, y, patch_w, patch_h = center_lf_point
304
- big_patch = self.patch_crop(img_for_crop, y, x, patch_h, patch_w)
305
- patches.append(big_patch)
306
- if (patch_id + 1) % x_num == 0:
307
- newlines.append(patch_id)
308
-
309
- if newlines and newlines[-1] == len(patches) - 1:
310
- newlines.pop()
311
-
312
- return (
313
- img,
314
- patches,
315
- [i in newlines for i in range(len(patches))]
316
- if len(patches) > 0
317
- else None,
318
- )
319
-
320
-
321
-class Step3VLProcessor:
322
- def __init__(
323
- self,
324
- config: PretrainedConfig,
325
- tokenizer: TokenizerLike,
326
- ) -> None:
327
- super().__init__()
328
-
329
- self.config = config
330
- self.tokenizer = tokenizer
331
- self.image_size = 728
332
- self.patch_size = 504
333
- self.image_preprocessor = Step3VisionProcessor(
334
- self.image_size, "bilinear", self.patch_size
335
- )
336
-
337
- self.num_image_feature_size = 169
338
- self.num_patch_feature_size = 81
339
- self.image_token = "<im_patch>"
340
- self.image_feature_placeholder = self.image_token * self.num_image_feature_size
341
- self.patch_feature_placeholder = self.image_token * self.num_patch_feature_size
342
-
343
- # Respect vision config switch to enable/disable patch extraction.
344
- # For video understanding, it's preferable to disable patch.
345
- enable_patch = getattr(self.config.vision_config, "enable_patch", True)
346
- self.patcher = ImagePatcher(enable_patch=enable_patch)
347
-
348
- @property
349
- def image_token_id(self) -> int:
350
- return self.tokenizer.get_vocab()[self.image_token]
351
-
352
- def get_num_image_tokens(self, img_width: int, img_height: int) -> int:
353
- num_patches, num_newlines = self.patcher.get_num_patches(img_width, img_height)
354
-
355
- return (
356
- num_patches * (self.num_patch_feature_size + 2)
357
- + self.num_image_feature_size
358
- + 2
359
- + num_newlines
360
- )
361
-
362
- def _split_images(self, images: list[Image.Image]) -> list[ImageWithPatches]:
363
- result = []
364
- for img in images:
365
- result.append(self.patcher(img))
366
- return result
367
-
368
- def _convert_images_to_pixel_values(
369
- self,
370
- images: list[Image.Image],
371
- is_patch: bool = False,
372
- ) -> list[torch.Tensor]:
373
- return [
374
- self.image_preprocessor(img, is_patch=is_patch)["pixel_values"]
375
- for img in images
376
- ]
377
-
378
- def _get_patch_repl(
379
- self,
380
- num_patches: int,
381
- patch_newline_mask: list[bool] | None,
382
- ) -> tuple[str, list[int]]:
383
- text = ""
384
- token_ids = []
385
- for i in range(num_patches):
386
- assert len(patch_newline_mask) == num_patches
387
- text += f"<patch_start>{self.patch_feature_placeholder}<patch_end>"
388
- token_ids.extend(
389
- [self.tokenizer.convert_tokens_to_ids("<patch_start>")]
390
- + [self.image_token_id] * self.num_patch_feature_size
391
- + [self.tokenizer.convert_tokens_to_ids("<patch_end>")]
392
- )
393
- if patch_newline_mask and patch_newline_mask[i]:
394
- text += "<patch_newline>"
395
- token_ids.append(
396
- self.tokenizer.convert_tokens_to_ids("<patch_newline>")
397
- )
398
- return text, token_ids
399
-
400
- def _get_image_repl(
401
- self,
402
- num_images: int,
403
- ) -> tuple[str, list[int]]:
404
- text = f"<im_start>{self.image_feature_placeholder}<im_end>"
405
- token_ids = (
406
- [self.tokenizer.convert_tokens_to_ids("<im_start>")]
407
- + [self.image_token_id] * self.num_image_feature_size
408
- + [self.tokenizer.convert_tokens_to_ids("<im_end>")]
409
- )
410
- return text * num_images, token_ids * num_images
411
-
412
- def _get_image_repl_features(
413
- self,
414
- num_images: int,
415
- num_patches: int,
416
- patch_new_line_idx: list[bool] | None,
417
- ) -> tuple[str, list[int]]:
418
- if num_patches > 0:
419
- patch_repl, patch_repl_ids = self._get_patch_repl(
420
- num_patches, patch_new_line_idx
421
- )
422
- else:
423
- patch_repl = ""
424
- patch_repl_ids = []
425
- image_repl, image_repl_ids = self._get_image_repl(num_images)
426
- return patch_repl + image_repl, patch_repl_ids + image_repl_ids
427
-
428
- def replace_placeholder(self, text: str, placeholder: str, repls: list[str]) -> str:
429
- parts = text.split(placeholder)
430
-
431
- if len(parts) - 1 != len(repls):
432
- raise ValueError(
433
- "The number of placeholders does not match the number of replacements."
434
- )
435
-
436
- result = [parts[0]]
437
- for i, repl in enumerate(repls):
438
- result.append(repl)
439
- result.append(parts[i + 1])
440
-
441
- return "".join(result)
442
-
443
- def __call__(
444
- self,
445
- text: str | list[str] | None = None,
446
- images: Image.Image | list[Image.Image] | None = None,
447
- return_tensors: str | TensorType | None = None,
448
- ) -> BatchFeature:
449
- if text is None:
450
- text = []
451
- if not isinstance(text, list):
452
- text = [text]
453
- if images is None:
454
- images = []
455
- if not isinstance(images, list):
456
- images = [images]
457
-
458
- if len(images) == 0:
459
- image_inputs = {}
460
- text_inputs = self.tokenizer(text)
461
- else:
462
- split_images_data = self._split_images(images)
463
- pixel_values_lst = []
464
- patch_pixel_values_lst = []
465
- patch_newline_mask_lst = []
466
- image_repl_str_lst = []
467
- image_repl_ids_lst = []
468
- num_patches = []
469
- for raw_img, img_patches, patch_newline_mask in split_images_data:
470
- pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
471
-
472
- if len(img_patches) > 0:
473
- patch_pixel_values_lst.extend(
474
- self._convert_images_to_pixel_values(img_patches, is_patch=True)
475
- )
476
- num_patches.append(len(img_patches))
477
-
478
- image_repl_str, image_repl_ids = self._get_image_repl_features(
479
- 1, len(img_patches), patch_newline_mask
480
- )
481
- image_repl_str_lst.append(image_repl_str)
482
- image_repl_ids_lst.extend(image_repl_ids)
483
-
484
- if patch_newline_mask is not None:
485
- patch_newline_mask_lst.extend(patch_newline_mask)
486
-
487
- pixel_values = torch.cat(pixel_values_lst)
488
- patch_size = self.patch_size
489
- image_inputs = {
490
- "pixel_values": pixel_values,
491
- "num_patches": num_patches,
492
- "patch_pixel_values": (
493
- torch.cat(patch_pixel_values_lst)
494
- if patch_pixel_values_lst
495
- else pixel_values.new_empty((0, 3, patch_size, patch_size))
496
- ),
497
- "patch_newline_mask": torch.tensor(
498
- patch_newline_mask_lst, dtype=torch.bool
499
- ),
500
- }
501
-
502
- text = [
503
- self.replace_placeholder(t, self.image_token, image_repl_str_lst)
504
- for t in text
505
- ]
506
- text_inputs = self.tokenizer(text)
507
-
508
- return BatchFeature(
509
- {
510
- **text_inputs,
511
- **image_inputs,
512
- },
513
- tensor_type=return_tensors,
514
- )
515
-
87
88
class Step3VLProcessingInfo(BaseProcessingInfo):
89
def get_hf_processor(self) -> Step3VLProcessor:
vllm/model_executor/models/tarsier.py
+2
-24
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelL
25
from vllm.model_executor.layers.quantization import QuantizationConfig
26
from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
27
from vllm.multimodal import MULTIMODAL_REGISTRY
28
-from vllm.multimodal.cache import BaseMultiModalProcessorCache
28
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
29
from vllm.multimodal.parse import (
30
ImageEmbeddingItems,
@@ -34,10 +33,8 @@ from vllm.multimodal.parse import (
33
MultiModalDataItems,
34
)
35
from vllm.multimodal.processing import (
37
- BaseDummyInputsBuilder,
36
BaseMultiModalProcessor,
37
BaseProcessingInfo,
40
- InputProcessingContext,
38
PromptReplacement,
39
PromptUpdate,
40
)
@@ -329,25 +326,6 @@ class TarsierMultiModalProcessor(BaseMultiModalProcessor[_I_Tarsier]):
326
]
327
328
332
-def _build_tarsier_hf_info(ctx: InputProcessingContext) -> TarsierProcessingInfo:
333
- return TarsierProcessingInfo(ctx)
334
-
335
-
336
-def _build_tarsier_hf_processor(
337
- info: _I_Tarsier,
338
- dummy_inputs: BaseDummyInputsBuilder[_I_Tarsier],
339
- *,
340
- cache: BaseMultiModalProcessorCache | None = None,
341
-) -> BaseMultiModalProcessor:
342
- if isinstance(info, TarsierProcessingInfo):
343
- return TarsierMultiModalProcessor(
344
- info,
345
- dummy_inputs,
346
- cache=cache,
347
- )
348
- raise NotImplementedError(type(info))
349
-
350
-
329
def init_vision_tower_for_tarsier(
330
hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol
331
quant_config: QuantizationConfig | None,
@@ -395,8 +373,8 @@ def init_vision_tower_for_tarsier(
373
374
375
@MULTIMODAL_REGISTRY.register_processor(
398
- _build_tarsier_hf_processor,
399
- info=_build_tarsier_hf_info,
376
+ TarsierMultiModalProcessor,
377
+ info=TarsierProcessingInfo,
378
dummy_inputs=TarsierDummyInputsBuilder,
379
)
380
class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
vllm/transformers_utils/configs/__init__.py
+1
-1
@@ -55,7 +55,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
55
"OvisConfig": "vllm.transformers_utils.configs.ovis",
56
"PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac",
57
"RadioConfig": "vllm.transformers_utils.configs.radio",
58
- "SpeculatorsConfig": "vllm.transformers_utils.configs.speculators.base",
58
+ "SpeculatorsConfig": "vllm.transformers_utils.configs.speculators",
59
"UltravoxConfig": "vllm.transformers_utils.configs.ultravox",
60
"Step3VLConfig": "vllm.transformers_utils.configs.step3_vl",
61
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
vllm/transformers_utils/configs/speculators/__init__.py
+3
@@ -1,2 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from .base import SpeculatorsConfig
4
+
5
+__all__ = ["SpeculatorsConfig"]
vllm/transformers_utils/configs/speculators/base.py
-3
@@ -8,9 +8,6 @@ from transformers import PretrainedConfig
8
from vllm.transformers_utils.configs.speculators.algos import (
9
SUPPORTED_SPECULATORS_TYPES,
10
)
11
-
12
-__all__ = ["SpeculatorsConfig"]
13
-
11
from vllm.transformers_utils.utils import without_trust_remote_code
12
13
vllm/transformers_utils/processors/__init__.py
+6
@@ -21,7 +21,9 @@ __all__ = [
21
"HunYuanVLProcessor",
22
"HunYuanVLImageProcessor",
23
"InternVLProcessor",
24
+ "IsaacProcessor",
25
"KimiAudioProcessor",
26
+ "KimiK25Processor",
27
"MistralCommonPixtralProcessor",
28
"MistralCommonVoxtralProcessor",
29
"NanoNemotronVLProcessor",
@@ -32,6 +34,7 @@ __all__ = [
34
"Ovis2_5Processor",
35
"QwenVLProcessor",
36
"Qwen3ASRProcessor",
37
+ "Step3VLProcessor",
38
]
39
40
_CLASS_TO_MODULE: dict[str, str] = {
@@ -45,7 +48,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
48
"HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
49
"HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
50
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
51
+ "IsaacProcessor": "vllm.transformers_utils.processors.isaac",
52
"KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
53
+ "KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25",
54
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
55
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
56
"NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
@@ -56,6 +61,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
61
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
62
"QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
63
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
64
+ "Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
65
}
66
67
vllm/transformers_utils/processors/isaac.py
new
+461
@@ -0,0 +1,461 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from __future__ import annotations
4
+
5
+import math
6
+from typing import Any
7
+
8
+import numpy as np
9
+import PIL.Image
10
+import torch
11
+import torch.nn.functional as F
12
+from transformers import BatchFeature, ProcessorMixin, TensorType
13
+from typing_extensions import TypedDict, Unpack
14
+
15
+MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px
16
+
17
+# Vision preprocessing constants
18
+VISION_MEAN = (0.5, 0.5, 0.5)
19
+VISION_STD = (0.5, 0.5, 0.5)
20
+VISION_SCALE = 1 / 255
21
+
22
+
23
+def _make_writeable(arr: np.ndarray) -> np.ndarray:
24
+ """Return *arr* itself if it is already writeable, otherwise try to flip the
25
+ write flag in-place and finally fall back to `arr.copy()`.
26
+ This guarantees the buffer handed to `torch.from_numpy()` is always
27
+ writeable, silencing the PyTorch warning about undefined behaviour.
28
+ """
29
+ if arr.flags.writeable:
30
+ return arr
31
+
32
+ # First, try the cheap path — in-place flag toggle (works for mmap'd arrays
33
+ # and some shared memory buffers):
34
+ try:
35
+ arr.setflags(write=True)
36
+ return arr # success: no data copy
37
+ except ValueError:
38
+ # Buffer is inherently read-only (e.g. backed by PyAV / PIL): make copy
39
+ return arr.copy()
40
+
41
+
42
+def extract_image_pil(image: PIL.Image.Image) -> torch.Tensor | None:
43
+ if image.width * image.height > MAX_PIXELS:
44
+ raise ValueError(
45
+ f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`"
46
+ )
47
+ img = image if image.mode == "RGB" else image.convert("RGB")
48
+ arr = np.asarray(img)
49
+ arr = _make_writeable(arr)
50
+ return torch.from_numpy(arr)
51
+
52
+
53
+def get_image_size_for_max_num_patches(
54
+ image_height: int,
55
+ image_width: int,
56
+ patch_size: int,
57
+ max_num_patches: int,
58
+ min_num_patches: int | None = None,
59
+ eps: float = 1e-5,
60
+ pixel_shuffle_scale: int = 1,
61
+) -> tuple[int, int]:
62
+ r"""Compute a target resolution whose patch grid satisfies patching parametrization.
63
+
64
+ Args:
65
+ image_height (`int`):
66
+ Height in pixels of the source image prior to any resizing.
67
+ image_width (`int`):
68
+ Width in pixels of the source image prior to any resizing.
69
+ patch_size (`int`):
70
+ Size of the square patch used by the vision encoder.
71
+ max_num_patches (`int`):
72
+ Upper bound on `(height / patch_size) * (width / patch_size)` after
73
+ resizing.
74
+ min_num_patches (`int`, *optional*):
75
+ Lower bound on the number of patches. When provided the image will
76
+ be scaled up if necessary.
77
+ eps (`float`, *optional*, defaults to 1e-5):
78
+ Convergence tolerance for the internal binary search to determine
79
+ the target dimensions.
80
+ pixel_shuffle_scale (`int`, *optional*, defaults to 1):
81
+ Additional stride multiplier applied when pixel shuffle later
82
+ reduces spatial resolution.
83
+
84
+ Returns:
85
+ `tuple[int, int]`: Height and width (in pixels) that are multiples of
86
+ `patch_size * pixel_shuffle_scale` and respect both the maximum and
87
+ optional minimum patch-count constraints.
88
+ """
89
+
90
+ def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale):
91
+ scaled_size = scale * original_size
92
+ divisor = patch_size * pixel_shuffle_scale
93
+ scaled_size = math.ceil(scaled_size / divisor) * divisor
94
+ scaled_size = max(divisor, scaled_size)
95
+ return int(scaled_size)
96
+
97
+ # Ensure divisibility
98
+ divisor = patch_size * pixel_shuffle_scale
99
+ adjusted_height = math.ceil(image_height / divisor) * divisor
100
+ adjusted_height = max(divisor, adjusted_height)
101
+ adjusted_width = math.ceil(image_width / divisor) * divisor
102
+ adjusted_width = max(divisor, adjusted_width)
103
+
104
+ num_patches = (adjusted_height / patch_size) * (adjusted_width / patch_size)
105
+
106
+ if min_num_patches is not None and num_patches < min_num_patches:
107
+ # Scale up
108
+ scale_min, scale_max = 1.0, 100.0
109
+ while (scale_max - scale_min) >= eps:
110
+ scale = (scale_min + scale_max) / 2
111
+ target_height = get_scaled_image_size(
112
+ scale, image_height, patch_size, pixel_shuffle_scale
113
+ )
114
+ target_width = get_scaled_image_size(
115
+ scale, image_width, patch_size, pixel_shuffle_scale
116
+ )
117
+ num_patches = (target_height / patch_size) * (target_width / patch_size)
118
+ if num_patches >= min_num_patches:
119
+ scale_max = scale
120
+ else:
121
+ scale_min = scale
122
+ scale = scale_max
123
+ target_height = get_scaled_image_size(
124
+ scale, image_height, patch_size, pixel_shuffle_scale
125
+ )
126
+ target_width = get_scaled_image_size(
127
+ scale, image_width, patch_size, pixel_shuffle_scale
128
+ )
129
+ return target_height, target_width
130
+ elif num_patches <= max_num_patches:
131
+ return adjusted_height, adjusted_width
132
+ else:
133
+ # Scale down
134
+ scale_min, scale_max = eps / 10, 1.0
135
+ while (scale_max - scale_min) >= eps:
136
+ scale = (scale_min + scale_max) / 2
137
+ target_height = get_scaled_image_size(
138
+ scale, image_height, patch_size, pixel_shuffle_scale
139
+ )
140
+ target_width = get_scaled_image_size(
141
+ scale, image_width, patch_size, pixel_shuffle_scale
142
+ )
143
+ num_patches = (target_height / patch_size) * (target_width / patch_size)
144
+ if num_patches <= max_num_patches:
145
+ scale_min = scale
146
+ else:
147
+ scale_max = scale
148
+ scale = scale_min
149
+ target_height = get_scaled_image_size(
150
+ scale, image_height, patch_size, pixel_shuffle_scale
151
+ )
152
+ target_width = get_scaled_image_size(
153
+ scale, image_width, patch_size, pixel_shuffle_scale
154
+ )
155
+ return target_height, target_width
156
+
157
+
158
+_MEAN_TENSOR = torch.tensor(VISION_MEAN, dtype=torch.float32).view(1, 1, 1, -1)
159
+_STD_TENSOR = torch.tensor(VISION_STD, dtype=torch.float32).view(1, 1, 1, -1)
160
+
161
+
162
+def prepare_image_tensor(
163
+ image: torch.Tensor,
164
+ scale: float = VISION_SCALE,
165
+) -> torch.Tensor:
166
+ r"""Standardize RGB images prior to patch extraction via rescaling and whitening.
167
+
168
+ Args:
169
+ image (`torch.Tensor`):
170
+ Tensor with shape `(..., height, width, 3)` containing RGB values.
171
+ The tensor is converted to floating point if needed.
172
+ scale (`float`, *optional*, defaults to `VISION_SCALE`):
173
+ Scalar multiplier applied before normalization.
174
+ Returns:
175
+ `torch.Tensor`: Normalized tensor with the same shape as the input and
176
+ dtype `torch.float32`.
177
+ """
178
+ if not torch.is_floating_point(image):
179
+ image = image.float()
180
+ rescaled = image * scale
181
+
182
+ # Use precomputed tensors and move to the correct device if needed
183
+ mean_tensor = _MEAN_TENSOR.to(image.device)
184
+ std_tensor = _STD_TENSOR.to(image.device)
185
+
186
+ normalized = (rescaled - mean_tensor) / std_tensor
187
+ return normalized
188
+
189
+
190
+def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor:
191
+ r"""Convert normalized images into flattened ViT-style patches.
192
+
193
+ Args:
194
+ image (`torch.Tensor`):
195
+ Tensor of shape `(num_images, height, width, channels)`.
196
+ patch_size (`int`):
197
+ Edge length of the square patches
198
+
199
+ Returns:
200
+ `torch.Tensor`:
201
+ Patch tensor where each position stores the flattened pixels
202
+ belonging to that patch.
203
+
204
+ Raises:
205
+ ValueError: If `height` or `width` is not divisible by `patch_size`.
206
+ """
207
+ num_images, height, width, channels = image.shape
208
+ if height % patch_size or width % patch_size:
209
+ raise ValueError(
210
+ "Dimensions of images "
211
+ f"{image.shape} are not divisible by patch_size={patch_size}."
212
+ )
213
+ patches = image.reshape(
214
+ num_images,
215
+ height // patch_size,
216
+ patch_size,
217
+ width // patch_size,
218
+ patch_size,
219
+ channels,
220
+ )
221
+ patches = patches.permute(0, 1, 3, 2, 4, 5)
222
+ patches = patches.reshape(
223
+ num_images,
224
+ height // patch_size,
225
+ width // patch_size,
226
+ channels * patch_size * patch_size,
227
+ )
228
+ return patches
229
+
230
+
231
+def process_vision_for_patches(
232
+ images: torch.Tensor,
233
+ patch_size: int,
234
+ max_num_patches: int,
235
+ min_num_patches: int | None = None,
236
+ pixel_shuffle_scale: int = 1,
237
+) -> tuple[torch.Tensor, list[int]]:
238
+ r"""Resize, normalize, and patchify RGB images for the vision encoder.
239
+
240
+ Args:
241
+ images (`torch.Tensor`):
242
+ Either `(height, width, channels)` for a single image or
243
+ `(num_images, height, width, channels)` for a batch. Channels are
244
+ expected to be RGB.
245
+ patch_size (`int`):
246
+ Edge length of square patches; implicitly controls resize grid granularity.
247
+ max_num_patches (`int`):
248
+ Maximum number of patches allowed after resizing.
249
+ min_num_patches (`int`, *optional*):
250
+ Minimum number of patches. If provided, the routine upsamples images
251
+ as needed to satisfy the lower bound.
252
+ pixel_shuffle_scale (`int`, *optional*, defaults to 1):
253
+ Pixel shuffle scale factor; influences the target grid that the
254
+ function produces.
255
+
256
+ Returns:
257
+ `tuple[torch.Tensor, list[int]]`: A pair `(patches, dims_virtual)`
258
+ where `patches` has shape `(num_images, target_h / patch_size, target_w
259
+ / patch_size, channels * patch_size**2)` and `dims_virtual` encodes
260
+ effective `(images, height, width)` dimensions after optional pixel
261
+ shuffling.
262
+ """
263
+ # Add batch dim if single image
264
+ if images.dim() == 3:
265
+ images = images.unsqueeze(0)
266
+
267
+ # Permute to channel first for resize
268
+ images = images.permute(0, 3, 1, 2)
269
+
270
+ # Get target dimensions
271
+ _, _, orig_height, orig_width = images.shape
272
+ target_height, target_width = get_image_size_for_max_num_patches(
273
+ orig_height,
274
+ orig_width,
275
+ patch_size,
276
+ max_num_patches,
277
+ min_num_patches=min_num_patches,
278
+ pixel_shuffle_scale=pixel_shuffle_scale,
279
+ )
280
+
281
+ # Resize
282
+ images = F.interpolate(
283
+ images,
284
+ size=(target_height, target_width),
285
+ mode="bilinear",
286
+ align_corners=False,
287
+ )
288
+
289
+ # Back to channel last
290
+ images = images.permute(0, 2, 3, 1)
291
+
292
+ # Normalize
293
+ images = prepare_image_tensor(images)
294
+
295
+ # Patchify
296
+ patches = patchify_vision(images, patch_size=patch_size)
297
+
298
+ # Calculate dimensions for the patches
299
+ n_images, h_patches, w_patches, _ = patches.shape
300
+ dims_virtual = (
301
+ [1, h_patches, w_patches]
302
+ if pixel_shuffle_scale == 1
303
+ else [1, h_patches // pixel_shuffle_scale, w_patches // pixel_shuffle_scale]
304
+ )
305
+
306
+ return patches, dims_virtual
307
+
308
+
309
+class IsaacImageProcessorKwargs(TypedDict, total=False):
310
+ patch_size: int
311
+ max_num_patches: int
312
+ min_num_patches: int
313
+ pixel_shuffle_scale: int
314
+
315
+
316
+class IsaacImageProcessor:
317
+ patch_size = 16
318
+ max_num_patches = 6144
319
+ min_num_patches = 256
320
+ pixel_shuffle_scale = 2
321
+
322
+ valid_kwargs = IsaacImageProcessorKwargs
323
+ model_input_names = ["pixel_values", "image_grid_thw"]
324
+
325
+ def __init__(self, kwargs):
326
+ self.patch_size = kwargs.pop("patch_size", self.patch_size)
327
+ self.vision_max_num_patches = kwargs.pop(
328
+ "vision_max_num_patches", self.max_num_patches
329
+ )
330
+ self.vision_min_num_patches = kwargs.pop(
331
+ "vision_min_num_patches", self.min_num_patches
332
+ )
333
+ self.pixel_shuffle_scale = kwargs.pop("pixel_shuffle_scale", 2)
334
+
335
+ def preprocess(
336
+ self,
337
+ images: list[torch.Tensor],
338
+ return_tensors: str | TensorType | None,
339
+ **kwargs: Unpack[IsaacImageProcessorKwargs],
340
+ ) -> BatchFeature:
341
+ """Preprocess images into format compatible with vLLM input processing."""
342
+
343
+ all_pixel_values: list[torch.Tensor] = []
344
+ all_image_grids: list[torch.Tensor] = []
345
+
346
+ for image in images:
347
+ image_tensor = extract_image_pil(image)
348
+
349
+ patches, dims_virtual = process_vision_for_patches(
350
+ image_tensor,
351
+ patch_size=self.patch_size,
352
+ max_num_patches=self.vision_max_num_patches,
353
+ min_num_patches=self.vision_min_num_patches,
354
+ pixel_shuffle_scale=self.pixel_shuffle_scale,
355
+ )
356
+
357
+ # Isaac packs a dummy temporal dim for images
358
+ patches = patches.unsqueeze(1) # [N, T=1, Hp, Wp, D]
359
+
360
+ hp, wp, dim = patches.shape[-3], patches.shape[-2], patches.shape[-1]
361
+ current_num_patches = hp * wp
362
+ pixel_values = patches.reshape(current_num_patches, dim) # [N_tokens, D]
363
+
364
+ # Use real patch dimensions for image_grid_thw, not virtual dimensions
365
+ # This ensures the vision model receives correct grid info for pixel shuffle
366
+ dims_real = [1, hp, wp] # Real patch dimensions
367
+ image_grid_thw = torch.tensor(dims_real).unsqueeze(0)
368
+
369
+ all_pixel_values.append(pixel_values)
370
+ all_image_grids.append(image_grid_thw)
371
+
372
+ if all_pixel_values:
373
+ final_pixel_values = torch.cat(all_pixel_values, dim=0)
374
+ final_image_grids = torch.cat(all_image_grids, dim=0)
375
+ else:
376
+ final_pixel_values = torch.empty(0, 0)
377
+ final_image_grids = torch.empty(0, 3)
378
+
379
+ return BatchFeature(
380
+ data={
381
+ "pixel_values": final_pixel_values,
382
+ "image_grid_thw": final_image_grids,
383
+ },
384
+ tensor_type=return_tensors,
385
+ )
386
+
387
+
388
+class IsaacProcessor(ProcessorMixin):
389
+ attributes = ["image_processor", "tokenizer"]
390
+
391
+ def __init__(self, image_processor=None, tokenizer=None, **kwargs):
392
+ self.image_token = kwargs.pop("image_token", "<image>")
393
+ self.image_processor = image_processor
394
+ self.tokenizer = tokenizer
395
+
396
+ def __call__(self, text=None, images=None, **kwargs) -> BatchFeature:
397
+ result = {}
398
+
399
+ if images is not None:
400
+ image_inputs = self.image_processor.preprocess(images, **kwargs)
401
+ image_grid_thw = image_inputs["image_grid_thw"]
402
+ result.update(image_inputs)
403
+
404
+ if text is not None:
405
+ if not isinstance(text, list):
406
+ text = [text]
407
+
408
+ text = text.copy() # below lines change text in-place
409
+ merge_length = self.image_processor.pixel_shuffle_scale**2
410
+ index = 0
411
+ for i in range(len(text)):
412
+ while self.image_token in text[i]:
413
+ num_image_tokens = image_grid_thw[index].prod() // merge_length
414
+ text[i] = text[i].replace(
415
+ self.image_token, "<|placeholder|>" * num_image_tokens, 1
416
+ )
417
+ index += 1
418
+ text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
419
+
420
+ if text is not None:
421
+ result.update(self.tokenizer(text, **kwargs))
422
+
423
+ return BatchFeature(result)
424
+
425
+ def apply_chat_template(
426
+ self,
427
+ messages: list[dict[str, Any]],
428
+ tokenize: bool = False,
429
+ add_generation_prompt: bool = False,
430
+ **kwargs,
431
+ ) -> Any:
432
+ # Convert mixed content messages to simple text format
433
+ processed_messages = []
434
+
435
+ for message in messages:
436
+ if "content" in message and isinstance(message["content"], list):
437
+ # Handle mixed content (text + image)
438
+ text_parts = []
439
+ for content_item in message["content"]:
440
+ if content_item.get("type") == "text":
441
+ text_parts.append(content_item.get("text", ""))
442
+ elif content_item.get("type") == "image":
443
+ # Replace image with vision token
444
+ text_parts.append(self.image_token)
445
+
446
+ processed_message = {
447
+ "role": message.get("role", "user"),
448
+ "content": "".join(text_parts),
449
+ }
450
+ processed_messages.append(processed_message)
451
+ else:
452
+ # Regular text message
453
+ processed_messages.append(message)
454
+
455
+ kwargs["return_dict"] = False
456
+ return self.tokenizer.apply_chat_template(
457
+ processed_messages,
458
+ tokenize=tokenize,
459
+ add_generation_prompt=add_generation_prompt,
460
+ **kwargs,
461
+ )
vllm/transformers_utils/processors/kimi_k25.py
new
+73
@@ -0,0 +1,73 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import torch
4
+from transformers import BatchFeature
5
+from transformers.processing_utils import ProcessorMixin
6
+
7
+from vllm.multimodal.inputs import VisionChunk
8
+
9
+
10
+class KimiK25Processor(ProcessorMixin):
11
+ attributes = ["tokenizer"]
12
+ tokenizer_class = "AutoTokenizer"
13
+
14
+ def __init__(
15
+ self, media_processor=None, tokenizer=None, media_token_id: int | None = None
16
+ ):
17
+ super().__init__(tokenizer)
18
+ self.media_processor = media_processor
19
+ self.media_token_id = media_token_id
20
+ assert self.media_token_id is not None
21
+
22
+ # We do not support str input for text here
23
+ def __call__(
24
+ self,
25
+ vision_chunks: list[VisionChunk] | None = None,
26
+ *,
27
+ text: list[int] | str,
28
+ **kwargs,
29
+ ) -> BatchFeature:
30
+ """
31
+ Args:
32
+ vision_chunks: List of VisionChunk items to be processed.
33
+ For image: VisionChunkImage with type='image', image=PIL.Image
34
+ For video_chunk: VisionChunkVideo with type='video_chunk',
35
+ video_chunk=list[PIL.Image]
36
+ text: The token ids to be fed to a model (required).
37
+ Returns:
38
+ [`BatchFeature`]: A [`BatchFeature`] with the following fields:
39
+
40
+ - **input_ids** -- list of token ids to be fed to a model.
41
+ - **pixel_values** -- Pixel values to be fed to a model.
42
+ Returned when `vision_chunks` is not `None`.
43
+ - **grid_thws** -- list of image 3D grid in LLM.
44
+ Returned when `vision_chunks` is not `None`.
45
+ """
46
+ mm_inputs = {}
47
+ input_ids = self.tokenizer.encode(text) if isinstance(text, str) else text
48
+ if vision_chunks is not None:
49
+ assert isinstance(vision_chunks, list)
50
+ mm_inputs = self.media_processor.preprocess(vision_chunks)
51
+
52
+ num_tokens_per_chunk = [
53
+ self.media_processor.media_tokens_calculator(chunk)
54
+ for chunk in vision_chunks
55
+ ]
56
+
57
+ new_input_ids = []
58
+ for token in input_ids:
59
+ if token == self.media_token_id:
60
+ new_input_ids.extend(
61
+ [self.media_token_id] * num_tokens_per_chunk.pop(0)
62
+ )
63
+ else:
64
+ new_input_ids.append(token)
65
+ input_ids = new_input_ids
66
+
67
+ # XXX: _apply_hf_processor_text_mm will call tolist() on input_ids
68
+ return BatchFeature(
69
+ data={
70
+ "input_ids": torch.tensor([input_ids]),
71
+ **mm_inputs,
72
+ }
73
+ )
vllm/transformers_utils/processors/step3_vl.py
new
+441
@@ -0,0 +1,441 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from itertools import product
4
+from math import ceil
5
+
6
+import numpy as np
7
+import torch
8
+from PIL import Image
9
+from torchvision import transforms
10
+from torchvision.transforms.functional import InterpolationMode
11
+from transformers import BatchFeature, PretrainedConfig, TensorType
12
+
13
+from vllm.tokenizers import TokenizerLike
14
+
15
+MAX_IMAGE_SIZE: int = 3024
16
+
17
+ImageWithPatches = tuple[Image.Image, list[Image.Image], list[bool] | None]
18
+
19
+
20
+class Step3VisionProcessor:
21
+ def __init__(self, size, interpolation_mode="bicubic", patch_size=None):
22
+ mean = [0.48145466, 0.4578275, 0.40821073]
23
+ std = [0.26862954, 0.26130258, 0.27577711]
24
+ patch_size = patch_size if patch_size is not None else size
25
+
26
+ self.transform = transforms.Compose(
27
+ [
28
+ transforms.ToTensor(),
29
+ transforms.Normalize(mean, std),
30
+ transforms.Resize(
31
+ (size, size),
32
+ interpolation=InterpolationMode.BICUBIC
33
+ if interpolation_mode == "bicubic"
34
+ else InterpolationMode.BILINEAR,
35
+ antialias=True,
36
+ ),
37
+ ]
38
+ )
39
+
40
+ self.patch_transform = (
41
+ transforms.Compose(
42
+ [
43
+ transforms.ToTensor(),
44
+ transforms.Normalize(mean, std),
45
+ transforms.Resize(
46
+ (patch_size, patch_size),
47
+ interpolation=InterpolationMode.BICUBIC
48
+ if interpolation_mode == "bicubic"
49
+ else InterpolationMode.BILINEAR,
50
+ antialias=True,
51
+ ),
52
+ ]
53
+ )
54
+ if patch_size is not None
55
+ else None
56
+ )
57
+
58
+ def __call__(self, image, is_patch=False):
59
+ if is_patch:
60
+ assert self.patch_transform is not None
61
+ return {"pixel_values": self.patch_transform(image).unsqueeze(0)}
62
+
63
+ return {"pixel_values": self.transform(image).unsqueeze(0)}
64
+
65
+
66
+class ImagePatcher:
67
+ def __init__(self, enable_patch: bool = True) -> None:
68
+ self.enable_patch = enable_patch
69
+
70
+ def determine_window_size(self, long: int, short: int) -> int:
71
+ if long < 728:
72
+ return short if long / short > 1.5 else 0
73
+ return min(short, 504) if long / short > 4 else 504
74
+
75
+ def slide_window(
76
+ self,
77
+ width: int,
78
+ height: int,
79
+ sizes: list[tuple[int, int]],
80
+ steps: list[tuple[int, int]],
81
+ img_rate_thr: float = 0.6,
82
+ ) -> tuple[list[tuple[int, int, int, int]], tuple[int, int]]:
83
+ assert 1 >= img_rate_thr >= 0, "The `in_rate_thr` should lie in 0~1"
84
+ windows = []
85
+ # Sliding windows.
86
+ for size, step in zip(sizes, steps):
87
+ size_w, size_h = size
88
+ step_w, step_h = step
89
+
90
+ x_num = 1 if width <= size_w else ceil((width - size_w) / step_w + 1)
91
+ x_start = [step_w * i for i in range(x_num)]
92
+ if len(x_start) > 1 and x_start[-1] + size_w > width:
93
+ x_start[-1] = width - size_w
94
+
95
+ y_num = 1 if height <= size_h else ceil((height - size_h) / step_h + 1)
96
+ y_start = [step_h * i for i in range(y_num)]
97
+ if len(y_start) > 1 and y_start[-1] + size_h > height:
98
+ y_start[-1] = height - size_h
99
+
100
+ start = np.array(list(product(y_start, x_start)), dtype=int)
101
+ start[:, [0, 1]] = start[:, [1, 0]]
102
+ windows.append(np.concatenate([start, start + size], axis=1))
103
+ windows = np.concatenate(windows, axis=0)
104
+
105
+ return [
106
+ (int(box[0]), int(box[1]), int(box[2] - box[0]), int(box[3] - box[1]))
107
+ for box in windows
108
+ ], (x_num, y_num)
109
+
110
+ def square_pad(self, img: Image.Image) -> Image.Image:
111
+ w, h = img.size
112
+ if w == h:
113
+ return img
114
+ size = max(w, h)
115
+ padded = Image.new(img.mode, (size, size), 0)
116
+ padded.paste(img, (0, 0))
117
+ return padded
118
+
119
+ def get_image_size_for_padding(
120
+ self, img_width: int, img_height: int
121
+ ) -> tuple[int, int]:
122
+ ratio = img_width / img_height
123
+ if min(img_height, img_width) < 32 and (ratio > 4 or ratio < 1 / 4):
124
+ new_size = max(img_height, img_width)
125
+ return new_size, new_size
126
+ return img_width, img_height
127
+
128
+ def get_image_size_for_preprocess(
129
+ self, img_width: int, img_height: int
130
+ ) -> tuple[int, int]:
131
+ if max(img_height, img_width) > MAX_IMAGE_SIZE:
132
+ scale_factor = MAX_IMAGE_SIZE / max(img_height, img_width)
133
+ img_width = int(img_width * scale_factor)
134
+ img_height = int(img_height * scale_factor)
135
+ return img_width, img_height
136
+
137
+ def get_image_size_for_crop(
138
+ self, img_width: int, img_height: int, window_size: int
139
+ ):
140
+ w_ratio = img_width / window_size
141
+ h_ratio = img_height / window_size
142
+
143
+ if w_ratio < 1:
144
+ width_new = img_width
145
+ else:
146
+ decimal_w = w_ratio - img_width // window_size
147
+ w_ratio = int(w_ratio) + 1 if decimal_w > 0.2 else int(w_ratio)
148
+ width_new = window_size * w_ratio
149
+ if h_ratio < 1:
150
+ height_new = img_height
151
+ else:
152
+ decimal_h = h_ratio - img_height // window_size
153
+ h_ratio = int(h_ratio) + 1 if decimal_h > 0.2 else int(h_ratio)
154
+ height_new = window_size * h_ratio
155
+ return int(width_new), int(height_new)
156
+
157
+ def patch_crop(self, img: Image.Image, i: int, j: int, th: int, tw: int):
158
+ target = img.crop((j, i, j + tw, i + th))
159
+ return target
160
+
161
+ def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]:
162
+ img_width, img_height = self.get_image_size_for_padding(img_width, img_height)
163
+ img_width, img_height = self.get_image_size_for_preprocess(
164
+ img_width, img_height
165
+ )
166
+ window_size = self.determine_window_size(
167
+ max(img_height, img_width), min(img_height, img_width)
168
+ )
169
+ if window_size == 0 or not self.enable_patch:
170
+ return 0, 0
171
+ else:
172
+ img_width, img_height = self.get_image_size_for_crop(
173
+ img_width, img_height, window_size
174
+ )
175
+ center_list, (x_num, y_num) = self.slide_window(
176
+ img_width,
177
+ img_height,
178
+ [(window_size, window_size)],
179
+ [(window_size, window_size)],
180
+ )
181
+ full_rows = (len(center_list) - 1) // x_num + 1
182
+ if len(center_list) > 0 and len(center_list) % x_num == 0:
183
+ full_rows -= 1
184
+ return len(center_list), full_rows
185
+
186
+ def __call__(
187
+ self, img: Image.Image
188
+ ) -> tuple[Image.Image, list[Image.Image], list[bool] | None]:
189
+ img_width, img_height = img.size
190
+ new_img_width, new_img_height = self.get_image_size_for_padding(
191
+ img_width, img_height
192
+ )
193
+ if new_img_width != img_width or new_img_height != img_height:
194
+ img = self.square_pad(img)
195
+ img_width, img_height = img.size
196
+
197
+ new_img_width, new_img_height = self.get_image_size_for_preprocess(
198
+ img_width, img_height
199
+ )
200
+ img = img.resize((new_img_width, new_img_height), Image.Resampling.BILINEAR)
201
+ window_size = self.determine_window_size(
202
+ max(new_img_height, new_img_width), min(new_img_height, new_img_width)
203
+ )
204
+
205
+ if window_size == 0 or not self.enable_patch:
206
+ return img, [], None
207
+ else:
208
+ new_img_width, new_img_height = self.get_image_size_for_crop(
209
+ new_img_width, new_img_height, window_size
210
+ )
211
+ if (new_img_width, new_img_height) != (img_width, img_height):
212
+ img_for_crop = img.resize(
213
+ (new_img_width, new_img_height), Image.Resampling.BILINEAR
214
+ )
215
+ else:
216
+ img_for_crop = img
217
+
218
+ patches = []
219
+ newlines = []
220
+ center_list, (x_num, y_num) = self.slide_window(
221
+ new_img_width,
222
+ new_img_height,
223
+ [(window_size, window_size)],
224
+ [(window_size, window_size)],
225
+ )
226
+ for patch_id, center_lf_point in enumerate(center_list):
227
+ x, y, patch_w, patch_h = center_lf_point
228
+ big_patch = self.patch_crop(img_for_crop, y, x, patch_h, patch_w)
229
+ patches.append(big_patch)
230
+ if (patch_id + 1) % x_num == 0:
231
+ newlines.append(patch_id)
232
+
233
+ if newlines and newlines[-1] == len(patches) - 1:
234
+ newlines.pop()
235
+
236
+ return (
237
+ img,
238
+ patches,
239
+ [i in newlines for i in range(len(patches))]
240
+ if len(patches) > 0
241
+ else None,
242
+ )
243
+
244
+
245
+class Step3VLProcessor:
246
+ def __init__(
247
+ self,
248
+ config: PretrainedConfig,
249
+ tokenizer: TokenizerLike,
250
+ ) -> None:
251
+ super().__init__()
252
+
253
+ self.config = config
254
+ self.tokenizer = tokenizer
255
+ self.image_size = 728
256
+ self.patch_size = 504
257
+ self.image_preprocessor = Step3VisionProcessor(
258
+ self.image_size, "bilinear", self.patch_size
259
+ )
260
+
261
+ self.num_image_feature_size = 169
262
+ self.num_patch_feature_size = 81
263
+ self.image_token = "<im_patch>"
264
+ self.image_feature_placeholder = self.image_token * self.num_image_feature_size
265
+ self.patch_feature_placeholder = self.image_token * self.num_patch_feature_size
266
+
267
+ # Respect vision config switch to enable/disable patch extraction.
268
+ # For video understanding, it's preferable to disable patch.
269
+ enable_patch = getattr(self.config.vision_config, "enable_patch", True)
270
+ self.patcher = ImagePatcher(enable_patch=enable_patch)
271
+
272
+ @property
273
+ def image_token_id(self) -> int:
274
+ return self.tokenizer.get_vocab()[self.image_token]
275
+
276
+ def get_num_image_tokens(self, img_width: int, img_height: int) -> int:
277
+ num_patches, num_newlines = self.patcher.get_num_patches(img_width, img_height)
278
+
279
+ return (
280
+ num_patches * (self.num_patch_feature_size + 2)
281
+ + self.num_image_feature_size
282
+ + 2
283
+ + num_newlines
284
+ )
285
+
286
+ def _split_images(self, images: list[Image.Image]) -> list[ImageWithPatches]:
287
+ result = []
288
+ for img in images:
289
+ result.append(self.patcher(img))
290
+ return result
291
+
292
+ def _convert_images_to_pixel_values(
293
+ self,
294
+ images: list[Image.Image],
295
+ is_patch: bool = False,
296
+ ) -> list[torch.Tensor]:
297
+ return [
298
+ self.image_preprocessor(img, is_patch=is_patch)["pixel_values"]
299
+ for img in images
300
+ ]
301
+
302
+ def _get_patch_repl(
303
+ self,
304
+ num_patches: int,
305
+ patch_newline_mask: list[bool] | None,
306
+ ) -> tuple[str, list[int]]:
307
+ text = ""
308
+ token_ids = []
309
+ for i in range(num_patches):
310
+ assert (
311
+ patch_newline_mask is not None
312
+ and len(patch_newline_mask) == num_patches
313
+ )
314
+ text += f"<patch_start>{self.patch_feature_placeholder}<patch_end>"
315
+ token_ids.extend(
316
+ [self.tokenizer.convert_tokens_to_ids("<patch_start>")]
317
+ + [self.image_token_id] * self.num_patch_feature_size
318
+ + [self.tokenizer.convert_tokens_to_ids("<patch_end>")]
319
+ )
320
+ if patch_newline_mask and patch_newline_mask[i]:
321
+ text += "<patch_newline>"
322
+ token_ids.append(
323
+ self.tokenizer.convert_tokens_to_ids("<patch_newline>")
324
+ )
325
+ return text, token_ids
326
+
327
+ def _get_image_repl(
328
+ self,
329
+ num_images: int,
330
+ ) -> tuple[str, list[int]]:
331
+ text = f"<im_start>{self.image_feature_placeholder}<im_end>"
332
+ token_ids = (
333
+ [self.tokenizer.convert_tokens_to_ids("<im_start>")]
334
+ + [self.image_token_id] * self.num_image_feature_size
335
+ + [self.tokenizer.convert_tokens_to_ids("<im_end>")]
336
+ )
337
+ return text * num_images, token_ids * num_images
338
+
339
+ def _get_image_repl_features(
340
+ self,
341
+ num_images: int,
342
+ num_patches: int,
343
+ patch_new_line_idx: list[bool] | None,
344
+ ) -> tuple[str, list[int]]:
345
+ if num_patches > 0:
346
+ patch_repl, patch_repl_ids = self._get_patch_repl(
347
+ num_patches, patch_new_line_idx
348
+ )
349
+ else:
350
+ patch_repl = ""
351
+ patch_repl_ids = []
352
+ image_repl, image_repl_ids = self._get_image_repl(num_images)
353
+ return patch_repl + image_repl, patch_repl_ids + image_repl_ids
354
+
355
+ def replace_placeholder(self, text: str, placeholder: str, repls: list[str]) -> str:
356
+ parts = text.split(placeholder)
357
+
358
+ if len(parts) - 1 != len(repls):
359
+ raise ValueError(
360
+ "The number of placeholders does not match the number of replacements."
361
+ )
362
+
363
+ result = [parts[0]]
364
+ for i, repl in enumerate(repls):
365
+ result.append(repl)
366
+ result.append(parts[i + 1])
367
+
368
+ return "".join(result)
369
+
370
+ def __call__(
371
+ self,
372
+ text: str | list[str] | None = None,
373
+ images: Image.Image | list[Image.Image] | None = None,
374
+ return_tensors: str | TensorType | None = None,
375
+ ) -> BatchFeature:
376
+ if text is None:
377
+ text = []
378
+ if not isinstance(text, list):
379
+ text = [text]
380
+ if images is None:
381
+ images = []
382
+ if not isinstance(images, list):
383
+ images = [images]
384
+
385
+ if len(images) == 0:
386
+ image_inputs = {}
387
+ text_inputs = self.tokenizer(text)
388
+ else:
389
+ split_images_data = self._split_images(images)
390
+ pixel_values_lst = []
391
+ patch_pixel_values_lst = []
392
+ patch_newline_mask_lst = []
393
+ image_repl_str_lst = []
394
+ image_repl_ids_lst = []
395
+ num_patches = []
396
+ for raw_img, img_patches, patch_newline_mask in split_images_data:
397
+ pixel_values_lst.extend(self._convert_images_to_pixel_values([raw_img]))
398
+
399
+ if len(img_patches) > 0:
400
+ patch_pixel_values_lst.extend(
401
+ self._convert_images_to_pixel_values(img_patches, is_patch=True)
402
+ )
403
+ num_patches.append(len(img_patches))
404
+
405
+ image_repl_str, image_repl_ids = self._get_image_repl_features(
406
+ 1, len(img_patches), patch_newline_mask
407
+ )
408
+ image_repl_str_lst.append(image_repl_str)
409
+ image_repl_ids_lst.extend(image_repl_ids)
410
+
411
+ if patch_newline_mask is not None:
412
+ patch_newline_mask_lst.extend(patch_newline_mask)
413
+
414
+ pixel_values = torch.cat(pixel_values_lst)
415
+ patch_size = self.patch_size
416
+ image_inputs = {
417
+ "pixel_values": pixel_values,
418
+ "num_patches": num_patches,
419
+ "patch_pixel_values": (
420
+ torch.cat(patch_pixel_values_lst)
421
+ if patch_pixel_values_lst
422
+ else pixel_values.new_empty((0, 3, patch_size, patch_size))
423
+ ),
424
+ "patch_newline_mask": torch.tensor(
425
+ patch_newline_mask_lst, dtype=torch.bool
426
+ ),
427
+ }
428
+
429
+ text = [
430
+ self.replace_placeholder(t, self.image_token, image_repl_str_lst)
431
+ for t in text
432
+ ]
433
+ text_inputs = self.tokenizer(text)
434
+
435
+ return BatchFeature(
436
+ {
437
+ **text_inputs,
438
+ **image_inputs,
439
+ },
440
+ tensor_type=return_tensors,
441
+ )