@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 2fa490470d

[Model] Fused mm preprocess normalisation on the Device (#50411)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

wang.yuqi committed Aug 6, 2026 at 18:11 UTC 2fa490470ddf0fe578f7b70cd3aa8f6c1e06b2c6

13 files changed +359 -11

docs/design/mm_processing.md

+42

@@ -61,3 +61,45 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](https://github

61 When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.

63 Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other.

64 +

65 +## Speeding Up Multi‑Modal Data Processing

66 +

67 +### Fused Normalisation on the Device

68 +

69 +To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation, and rescaling), we offload the heavy numerical preprocessing from the CPU to the GPU and optimise data movement.

70 +

71 +#### Fusing Normalisation and Rescaling on the GPU

72 +

73 +Traditionally, the CPU would divide pixel values by 255, then subtract the mean and divide by the standard deviation. We fuse these steps into one operation and run it entirely on the GPU.

74 +

75 +- **How it works**: We use a dedicated `FusedInputNorm` module — essentially a frozen `BatchNorm1d` with the rescale factor baked into its statistics — and bake the rescale factor (typically 1/255) directly into the effective mean and standard deviation:

76 + - Effective mean = `image_mean * (1/rescale_factor)`

77 + - Effective std = `image_std * (1/rescale_factor)`

78 +- **At runtime**: The `FusedInputNorm` layer takes raw `uint8` pixel values (0–255) and performs the full normalised mapping in a single GPU kernel—no CPU involvement.

79 +

80 +#### Optimized Data Path for Fused Normalisation

81 +

82 +Performing fused normalisation directly on the device allows us to keep the entire transfer path—from **Entrypoint** through **Engine Core** to **GPU memory**—in **`uint8`**. This halves PCIe bandwidth and reduces CPU memory footprint.

83 +

84 +Only after data reaches GPU memory do we cast to `fp32` for the `FusedInputNorm` layer (to ensure numerical accuracy), then cast to `bf16` for subsequent layers—all within the GPU, avoiding any host‑side conversions.

85 +

86 +Overall path: **`Entrypoint (uint8) → Engine Core (uint8) → GPU Memory (uint8)`** → GPU‑local `fp32` `FusedInputNorm` → `bf16` output.

87 +

88 +#### Toggle: `mm_device_do_normalize`

89 +

90 +This GPU‑side fusion is controlled by a config flag called **`mm_device_do_normalize`**.

91 +

92 +- When `True`, normalisation and rescaling are done on the GPU using the `FusedInputNorm` layer; when `False`, we fall back to the old CPU‑side path.

93 +- The flag is **enabled by default** for all models that support it.

94 +- Currently, it’s on by default for these architectures:

95 +

96 +| name | Architecture | Example HF Models |

97 +|--------------|--------------------------------------|-------------------------------------|

98 +| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. |

99 +| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. |

100 +

101 +#### What We Gain Overall

102 +

103 +- **CPU offload**: The arithmetic for normalisation and rescaling is completely gone from the CPU.

104 +- **PCIe savings**: Sending `uint8` (1 byte) instead of `bf16` (2 bytes) slashes data transfer volume by **50%** .

105 +- **GPU overhead**: The fused kernel is very lightweight and can often be merged with subsequent CUDA operations, so it hardly adds any extra cost.

tests/models/multimodal/generation_ppl_test/ppl_utils.py

@@ -72,6 +72,7 @@ def vqa_ppl_test(

72 # and avoid batch different requests together.

74 model_config = vllm_model.llm.llm_engine.model_config

75 + mm_device_do_normalize = model_config.multimodal_config.mm_device_do_normalize

77 # Confirm whether vllm is using the correct architecture

78 if model_info.architecture:

@@ -147,6 +148,7 @@ def vqa_ppl_test(

148

149 differ = (vllm_ppl - hf_ppl) / hf_ppl

150 print("Model:", model_info.name)

151 + print("mm_device_do_normalize:", mm_device_do_normalize)

152 print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_ppl)

153 print("Transformers:", hf_dtype, hf_ppl)

154 print("Difference (%):", differ * 100)

tests/models/multimodal/generation_ppl_test/test_qwen.py

+11 -4

@@ -8,8 +8,8 @@ from tests.models.utils import GenerateModelInfo

8 from .ppl_utils import vqa_ppl_test

10 MODELS = [

11 - GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct"),

12 - GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct"),

11 + GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),

12 + GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),

13 ]

@@ -20,7 +20,14 @@ mm_processor_kwargs = {

22 @pytest.mark.parametrize("model_info", MODELS)

23 -def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):

23 +@pytest.mark.parametrize("mm_device_do_normalize", [True, False])

24 +def test_ppl(

25 + hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool

26 +):

27 vqa_ppl_test(

25 - hf_runner, vllm_runner, model_info, mm_processor_kwargs=mm_processor_kwargs

28 + hf_runner,

29 + vllm_runner,

30 + model_info,

31 + vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},

32 + mm_processor_kwargs=mm_processor_kwargs,

33 )

tests/models/multimodal/processing/test_qwen2_vl.py

+51

@@ -2,9 +2,11 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import pytest

5 +import torch

6 from packaging.version import Version

7 from transformers import __version__ as TRANSFORMERS_VERSION

9 +from vllm.model_executor.models.vision import FusedInputNorm

10 from vllm.multimodal import MULTIMODAL_REGISTRY

12 from ....conftest import ImageTestAssets

@@ -128,3 +130,52 @@ def test_get_image_size_with_most_features(

130 t, h, w = grid_thw[0]

131 tokens = (t * h * w) // (merge_size**2)

132 assert tokens < max_tokens

133 +

134 +

135 +@pytest.mark.parametrize(

136 + "model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]

137 +)

138 +@pytest.mark.parametrize("num_imgs", [1, 2])

139 +def test_mm_device_do_normalize(

140 + image_assets: ImageTestAssets,

141 + model_id: str,

142 + num_imgs: int,

143 +):

144 + """Ensure that enable mm_device_do_normalize yields the correct result."""

145 +

146 + ctx = build_model_context(

147 + model_id,

148 + limit_mm_per_prompt={"image": num_imgs},

149 + )

150 + ctx.model_config.multimodal_config.mm_device_do_normalize = False

151 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

152 +

153 + # Build the image str / prompt based on the number of images we pass

155 + mm_data = {"image": [image_assets[0].pil_image] * num_imgs}

156 +

157 + processed_inputs_with_normalize = processor(

158 + prompt,

159 + mm_items=processor.info.parse_mm_data(mm_data),

160 + )

161 + pixel_values_with_normalize = processed_inputs_with_normalize[

162 + "mm_kwargs"

163 + ].get_data()["pixel_values"]

164 + dtype = pixel_values_with_normalize.dtype

165 +

166 + processed_inputs_without_normalize = processor(

167 + prompt,

168 + mm_items=processor.info.parse_mm_data(mm_data),

169 + hf_processor_mm_kwargs={"do_normalize": False, "do_rescale": False},

170 + )

171 + pixel_values_without_normalize = processed_inputs_without_normalize[

172 + "mm_kwargs"

173 + ].get_data()["pixel_values"]

174 +

175 + ctx.model_config.multimodal_config.mm_device_do_normalize = True

176 + input_norm = FusedInputNorm.from_model_config(ctx.model_config)

177 + pixel_values_do_input_norm = input_norm(

178 + pixel_values_without_normalize.to(dtype), dtype

179 + )

180 +

181 + torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm)

vllm/config/model.py

+50

@@ -396,6 +396,7 @@ class ModelConfig:

396 video_pruning_method: InitVar[str | None] = None

397 mm_tensor_ipc: InitVar[MMTensorIPC] = None

398 mm_ipc_gpu_memory_gb: InitVar[float | None] = None

399 + mm_device_do_normalize: InitVar[bool | None] = None

400 mm_processor_device: InitVar[MMProcessorDevice | None] = None

401

402 def compute_hash(self) -> str:

@@ -527,6 +528,7 @@ class ModelConfig:

528 video_pruning_method: str | None,

529 mm_tensor_ipc: MMTensorIPC,

530 mm_ipc_gpu_memory_gb: float | None,

531 + mm_device_do_normalize: bool | None,

532 mm_processor_device: MMProcessorDevice | None,

533 ) -> None:

534 # Keep set served_model_name before maybe_model_redirect(self.model)

@@ -793,6 +795,9 @@ class ModelConfig:

795 video_pruning_method=video_pruning_method,

796 mm_tensor_ipc=mm_tensor_ipc,

797 mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,

798 + mm_device_do_normalize=self._resolve_mm_device_do_normalize(

799 + mm_device_do_normalize

800 + ),

801 )

802

803 mm_config_kwargs = {

@@ -927,6 +932,51 @@ class ModelConfig:

932 )

933 return self

934

935 + def _resolve_mm_device_do_normalize(

936 + self, mm_device_do_normalize: bool | None

937 + ) -> bool:

938 + if mm_device_do_normalize is None:

939 + if envs.VLLM_USE_RUST_FRONTEND:

940 + logger.debug(

941 + "VLLM_USE_RUST_FRONTEND is set. "

942 + "Rust frontend does not currently support mm_device_do_normalize, "

943 + "forcing mm_device_do_normalize = False."

944 + )

945 + mm_device_do_normalize = False

946 + else:

947 + mm_device_do_normalize = (

948 + self._model_info.supports_mm_device_do_normalize

949 + )

950 + logger.debug(

951 + "mm_device_do_normalize is %s by default.",

952 + "enabled" if mm_device_do_normalize else "disabled",

953 + )

954 + else:

955 + if mm_device_do_normalize and envs.VLLM_USE_RUST_FRONTEND:

956 + logger.warning(

957 + "VLLM_USE_RUST_FRONTEND is set. "

958 + "Rust frontend does not currently support mm_device_do_normalize, "

959 + "forcing mm_device_do_normalize = False."

960 + )

961 + mm_device_do_normalize = False

962 +

963 + if (

964 + mm_device_do_normalize

965 + and not self._model_info.supports_mm_device_do_normalize

966 + ):

967 + logger.warning(

968 + "Model does not support mm_device_do_normalize, "

969 + "forcing mm_device_do_normalize = False."

970 + )

971 + mm_device_do_normalize = False

972 +

973 + logger.debug(

974 + "mm_device_do_normalize is %s.",

975 + "enabled" if mm_device_do_normalize else "disabled",

976 + )

977 +

978 + return mm_device_do_normalize

979 +

980 def _get_transformers_backend_cls(self) -> str:

981 """Determine which Transformers modeling backend class will be used if

982 `model_impl` is set to `transformers` or `auto`."""

vllm/config/multimodal.py

@@ -144,6 +144,11 @@ class MultiModalConfig:

144

145 For example, for Phi-3-Vision:

146 `{"num_crops": 4}`."""

147 + mm_device_do_normalize: bool | None = True

148 + """

149 + Move the do_normalize computation in the mm preprocessing to before the ViT,

150 + and let the device do it, so that CPU computation can be saved.

151 + """

152 mm_processor_cache_gb: float = Field(default=4, ge=0)

153 """The size (in GiB) of the multi-modal processor cache, which is used to

154 avoid re-processing past multi-modal inputs.

@@ -474,6 +479,7 @@ class MultiModalConfig:

479 self.mm_encoder_tp_mode,

480 self.mm_encoder_attn_dtype,

481 self.mm_encoder_fp8_scale_path,

482 + self.mm_device_do_normalize,

483 ]

484 hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()

485 return hash_str

@@ -503,6 +509,9 @@ class MultiModalConfig:

509 according to the extra arguments passed during inference.

510 """

511 kwargs = self.mm_processor_kwargs or {}

512 + if self.mm_device_do_normalize:

513 + kwargs["do_normalize"] = False

514 + kwargs["do_rescale"] = False

515 return kwargs | dict(inference_kwargs)

516

517 def use_gpu_video_backend(self) -> bool:

vllm/engine/arg_utils.py

@@ -599,6 +599,7 @@ class EngineArgs:

599 mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc

600 mm_processor_device: MMProcessorDevice = "auto"

601 mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb

602 + mm_device_do_normalize: bool | None = MultiModalConfig.mm_device_do_normalize

603 # LoRA fields

604 enable_lora: bool = False

605 max_loras: int = LoRAConfig.max_loras

@@ -1383,6 +1384,13 @@ class EngineArgs:

1384 "--mm-ipc-gpu-memory-gb",

1385 **multimodal_kwargs["mm_ipc_gpu_memory_gb"],

1386 )

1387 + multimodal_group.add_argument(

1388 + "--mm-device-do-normalize",

1389 + **{

1390 + **multimodal_kwargs["mm_device_do_normalize"],

1391 + "default": None,

1392 + },

1393 + )

1394

1395 # LoRA related configs

1396 lora_kwargs = get_kwargs(LoRAConfig)

@@ -1769,6 +1777,7 @@ class EngineArgs:

1777 video_pruning_method=self.video_pruning_method,

1778 mm_tensor_ipc=self.mm_tensor_ipc,

1779 mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,

1780 + mm_device_do_normalize=self.mm_device_do_normalize,

1781 mm_processor_device=self.mm_processor_device,

1782 io_processor_plugin=self.io_processor_plugin,

1783 renderer_num_workers=self.renderer_num_workers,

vllm/model_executor/models/interfaces.py

@@ -149,6 +149,12 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):

149 `multimodal_config.mm_encoder_tp_mode="data"`.

150 """

151

152 + supports_mm_device_do_normalize: ClassVar[bool] = False

153 + """

154 + A flag that indicates whether this model supports

155 + `multimodal_config.mm_device_do_normalize`.

156 + """

157 +

158 requires_raw_input_tokens: ClassVar[bool] = False

159 """

160 A flag that indicates this model processes input id tokens

vllm/model_executor/models/qwen2_5_vl.py

@@ -112,6 +112,7 @@ from .utils import (

112 maybe_prefix,

113 )

114 from .vision import (

115 + FusedInputNorm,

116 get_fp8_padded_hidden_size,

117 get_vit_attn_backend,

118 is_vit_use_data_parallel,

@@ -1254,6 +1255,7 @@ class Qwen2_5_VLForConditionalGeneration(

1255 )

1256

1257 supports_encoder_tp_data = True

1258 + supports_mm_device_do_normalize = True

1259

1260 def iter_mm_grid_thw(

1261 self, mm_features: list[MultiModalFeatureSpec]

@@ -1363,6 +1365,7 @@ class Qwen2_5_VLForConditionalGeneration(

1365 quant_config=self.quant_config,

1366 prefix=maybe_prefix(prefix, "visual"),

1367 )

1368 + self.input_norm = FusedInputNorm.from_model_config(self.model_config)

1369

1370 with self._mark_language_model(vllm_config):

1371 self.language_model = init_vllm_registered_model(

@@ -1437,6 +1440,8 @@ class Qwen2_5_VLForConditionalGeneration(

1440 image_embeds = image_input["image_embeds"].type(self.visual.dtype)

1441 else:

1442 pixel_values = image_input["pixel_values"]

1443 + pixel_values = self.input_norm(pixel_values, self.visual.dtype)

1444 +

1445 if self.use_data_parallel:

1446 return run_dp_sharded_mrope_vision_model(

1447 self.visual, pixel_values, grid_thw_list, rope_type="rope_3d"

@@ -1493,6 +1498,10 @@ class Qwen2_5_VLForConditionalGeneration(

1498 video_embeds = video_input["video_embeds"].type(self.visual.dtype)

1499 else:

1500 pixel_values_videos = video_input["pixel_values_videos"]

1501 + pixel_values_videos = self.input_norm(

1502 + pixel_values_videos, self.visual.dtype

1503 + )

1504 +

1505 if self.use_data_parallel:

1506 return run_dp_sharded_mrope_vision_model(

1507 self.visual,

vllm/model_executor/models/qwen2_vl.py

+9 -2

@@ -103,6 +103,7 @@ from .utils import (

103 maybe_prefix,

104 )

105 from .vision import (

106 + FusedInputNorm,

107 get_vit_attn_backend,

108 is_vit_use_data_parallel,

109 run_dp_sharded_mrope_vision_model,

@@ -1192,6 +1193,7 @@ class Qwen2VLForConditionalGeneration(

1193 )

1194

1195 supports_encoder_tp_data = True

1196 + supports_mm_device_do_normalize = True

1197

1198 def iter_mm_grid_thw(

1199 self, mm_features: list[MultiModalFeatureSpec]

@@ -1296,6 +1298,7 @@ class Qwen2VLForConditionalGeneration(

1298 quant_config=quant_config,

1299 prefix=maybe_prefix(prefix, "visual"),

1300 )

1301 + self.input_norm = FusedInputNorm.from_model_config(self.model_config)

1302

1303 with self._mark_language_model(vllm_config):

1304 self.language_model = init_vllm_registered_model(

@@ -1363,9 +1366,10 @@ class Qwen2VLForConditionalGeneration(

1366 assert grid_thw.ndim == 2

1367

1368 if image_input["type"] == "image_embeds":

1366 - image_embeds = image_input["image_embeds"]

1369 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)

1370 else:

1371 pixel_values = image_input["pixel_values"]

1372 + pixel_values = self.input_norm(pixel_values, self.visual.dtype)

1373

1374 if self.use_data_parallel:

1375 return run_dp_sharded_mrope_vision_model(

@@ -1386,9 +1390,12 @@ class Qwen2VLForConditionalGeneration(

1390 assert grid_thw.ndim == 2

1391

1392 if video_input["type"] == "video_embeds":

1389 - video_embeds = video_input["video_embeds"]

1393 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)

1394 else:

1395 pixel_values_videos = video_input["pixel_values_videos"]

1396 + pixel_values_videos = self.input_norm(

1397 + pixel_values_videos, self.visual.dtype

1398 + )

1399 if self.use_data_parallel:

1400 return run_dp_sharded_mrope_vision_model(

1401 self.visual,

vllm/model_executor/models/registry.py

@@ -814,6 +814,7 @@ class _ModelInfo:

814 supports_transcription: bool

815 supports_transcription_only: bool

816 supported_video_pruning_methods: tuple[str, ...]

817 + supports_mm_device_do_normalize: bool

818

819 @staticmethod

820 def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":

@@ -847,6 +848,9 @@ class _ModelInfo:

848 supported_video_pruning_methods=getattr(

849 model, "supported_video_pruning_methods", ()

850 ),

851 + supports_mm_device_do_normalize=getattr(

852 + model, "supports_mm_device_do_normalize", False

853 + ),

854 )

855

856

vllm/model_executor/models/vision.py

+141 -1

@@ -8,9 +8,11 @@ from collections.abc import Callable

8 from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar

10 import torch

11 +import torch.nn as nn

12 +import torch.nn.functional as F

13 from transformers import PretrainedConfig

13 -from vllm.config import MultiModalConfig, get_current_vllm_config_or_none

15 +from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none

16 from vllm.distributed import (

17 get_tensor_model_parallel_rank,

18 get_tensor_model_parallel_world_size,

@@ -18,6 +20,7 @@ from vllm.distributed import (

20 )

21 from vllm.logger import init_logger

22 from vllm.platforms import current_platform

23 +from vllm.transformers_utils.processor import get_processor, get_processor_config

24 from vllm.utils.math_utils import round_up

25 from vllm.v1.attention.backends.registry import AttentionBackendEnum

@@ -580,3 +583,140 @@ def run_dp_sharded_mrope_vision_model(

583 "Found unassigned embeddings"

584 )

585 return out_embeddings

586 +

587 +

588 +class FusedInputNorm(nn.Module):

589 + """

590 + Module that applies rescaling and normalization to input images.

591 + Equivalent to: output = (input * rescale_factor - mean) / std

592 + """

593 +

594 + def __init__(

595 + self,

596 + image_mean: list[float],

597 + image_std: list[float],

598 + rescale_factor: float,

599 + channel: int = 3,

600 + dtype: torch.dtype = torch.float32,

601 + ):

602 + super().__init__()

603 +

604 + self.channel = channel

605 +

606 + image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * (

607 + 1.0 / rescale_factor

608 + )

609 + image_std_tensor = torch.tensor(image_std, dtype=dtype) * (1.0 / rescale_factor)

610 + weight = 1.0 / image_std_tensor

611 + bias = -image_mean_tensor / image_std_tensor

612 +

613 + self.is_identity = bool(

614 + torch.allclose(weight, torch.ones_like(weight))

615 + and torch.allclose(bias, torch.zeros_like(bias))

616 + )

617 +

618 + if not self.is_identity:

619 + self.register_buffer("weight", weight)

620 + self.register_buffer("bias", bias)

621 + self.register_buffer("running_mean", torch.zeros_like(image_mean_tensor))

622 + self.register_buffer("running_var", torch.ones_like(image_mean_tensor))

623 + else:

624 + self.register_buffer("weight", None)

625 + self.register_buffer("bias", None)

626 + self.register_buffer("running_mean", None)

627 + self.register_buffer("running_var", None)

628 +

629 + @property

630 + def dtype(self) -> torch.dtype:

631 + return self.weight.dtype

632 +

633 + @classmethod

634 + def identity(

635 + cls, channel: int = 3, dtype: torch.dtype = torch.float32

636 + ) -> "FusedInputNorm":

637 + return cls(

638 + image_mean=[0.0, 0.0, 0.0],

639 + image_std=[1.0, 1.0, 1.0],

640 + rescale_factor=1.0,

641 + channel=channel,

642 + dtype=dtype,

643 + )

644 +

645 + @classmethod

646 + def from_model_config(cls, model_config: "ModelConfig") -> nn.Module:

647 + if not model_config.multimodal_config.mm_device_do_normalize:

648 + return cls.identity()

649 +

650 + model = model_config.model

651 + revision = model_config.revision

652 +

653 + # Try to read parameters from the processor config

654 + config = get_processor_config(model, revision=revision)

655 + do_rescale = config.get("do_rescale", None)

656 + do_normalize = config.get("do_normalize", None)

657 + image_mean = config.get("image_mean", None)

658 + image_std = config.get("image_std", None)

659 + rescale_factor = config.get("rescale_factor", None)

660 +

661 + # Fallback to the image_processor object if any parameter is missing

662 + if None in [do_rescale, do_normalize, image_mean, image_std, rescale_factor]:

663 + image_processor = get_processor(model, revision=revision).image_processor

664 +

665 + if do_rescale is None:

666 + do_rescale = getattr(image_processor, "do_rescale", None)

667 + if do_normalize is None:

668 + do_normalize = getattr(image_processor, "do_normalize", None)

669 + if image_mean is None:

670 + image_mean = getattr(image_processor, "image_mean", None)

671 + if image_std is None:

672 + image_std = getattr(image_processor, "image_std", None)

673 + if rescale_factor is None:

674 + rescale_factor = getattr(image_processor, "rescale_factor", None)

675 +

676 + # Apply defaults based on flags

677 + if not do_rescale:

678 + rescale_factor = 1.0

679 + if not do_normalize:

680 + image_mean = [0.0, 0.0, 0.0]

681 + image_std = [1.0, 1.0, 1.0]

682 +

683 + # Ensure all required parameters are resolved

684 + assert None not in [

685 + do_rescale,

686 + do_normalize,

687 + image_mean,

688 + image_std,

689 + rescale_factor,

690 + ], "Some normalization parameters are still None after resolution."

691 +

692 + # If no processing is needed, return an identity module

693 + if not do_rescale and not do_normalize:

694 + return cls.identity()

695 +

696 + return cls(

697 + image_mean=image_mean, image_std=image_std, rescale_factor=rescale_factor

698 + )

699 +

700 + def forward(

701 + self,

702 + grid_thw: torch.Tensor,

703 + visual_dtype: torch.dtype,

704 + ) -> torch.Tensor:

705 + if self.is_identity:

706 + return grid_thw.to(visual_dtype)

707 +

708 + assert grid_thw.ndim == 2

709 + patches, size = grid_thw.shape

710 + patch_size = size // self.channel

711 +

712 + grid_thw = grid_thw.view(patches, self.channel, patch_size)

713 + grid_thw = F.batch_norm(

714 + grid_thw.to(self.dtype),

715 + running_mean=self.running_mean,

716 + running_var=self.running_var,

717 + weight=self.weight,

718 + bias=self.bias,

719 + training=False,

720 + eps=0.0,

721 + )

722 + return grid_thw.view(patches, size).to(visual_dtype)

vllm/transformers_utils/processor.py

+16 -4

@@ -142,10 +142,10 @@ def _merge_mm_kwargs(

142 return allowed_kwargs

143

144

145 -def get_processor_cls_name_from_config(

145 +def get_processor_config(

146 processor_name: str,

147 revision: str | None = "main",

148 -) -> str | None:

148 +) -> dict:

149 config_file = [

150 "processor_config.json",

151 "preprocessor_config.json",

@@ -154,8 +154,20 @@ def get_processor_cls_name_from_config(

154 for file in config_file:

155 config = get_hf_file_to_dict(file, processor_name, revision=revision)

156 if config and "processor_class" in config:

157 - return config["processor_class"]

158 - return None

157 + return config

158 + return {}

159 +

160 +

161 +def get_processor_cls_name_from_config(

162 + processor_name: str,

163 + revision: str | None = "main",

164 +) -> str | None:

165 + config = get_processor_config(processor_name, revision=revision)

166 +

167 + if not config:

168 + return None

169 +

170 + return config["processor_class"]

171

172

173 def get_video_processor_cls_name_from_config(