@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82de9b9d46
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 22:35 UTC

[Misc] Automatically resolve HF processor init kwargs (#22005)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Aug 1, 2025 at 13:44 UTC 82de9b9d468dab451380d3e7dda88b0c40a31204
40 files changed +332 -725
examples/offline_inference/vision_language.py
+19 -19
@@ -449,25 +449,6 @@ def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
449 )
450
451
452 -# omni-research/Tarsier-7b
453 -def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
454 - assert modality == "image"
455 - model_name = "omni-research/Tarsier-7b"
456 -
457 - engine_args = EngineArgs(
458 - model=model_name,
459 - trust_remote_code=True,
460 - max_model_len=4096,
461 - limit_mm_per_prompt={modality: 1},
462 - )
463 - prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
464 -
465 - return ModelRequestData(
466 - engine_args=engine_args,
467 - prompts=prompts,
468 - )
469 -
470 -
452 # Intern-S1
453 def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
454 model_name = "internlm/Intern-S1"
@@ -1293,6 +1274,25 @@ def run_qwen2_5_omni(questions: list[str], modality: str):
1274 )
1275
1276
1277 +# omni-research/Tarsier-7b
1278 +def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
1279 + assert modality == "image"
1280 + model_name = "omni-research/Tarsier-7b"
1281 +
1282 + engine_args = EngineArgs(
1283 + model=model_name,
1284 + trust_remote_code=True,
1285 + max_model_len=4096,
1286 + limit_mm_per_prompt={modality: 1},
1287 + )
1288 + prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
1289 +
1290 + return ModelRequestData(
1291 + engine_args=engine_args,
1292 + prompts=prompts,
1293 + )
1294 +
1295 +
1296 def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
1297 model_name = "omni-research/Tarsier2-Recap-7b"
1298
tests/lora/test_qwen2vl.py
-6
@@ -4,8 +4,6 @@ from dataclasses import dataclass
4 from typing import Optional
5
6 import pytest
7 -from packaging.version import Version
8 -from transformers import __version__ as TRANSFORMERS_VERSION
7
8 import vllm
9 from vllm.assets.image import ImageAsset
@@ -185,10 +183,6 @@ def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
183 current_platform.is_rocm(),
184 reason="Qwen2.5-VL dependency xformers incompatible with ROCm",
185 )
188 -@pytest.mark.skipif(
189 - Version(TRANSFORMERS_VERSION) < Version("4.49.0"),
190 - reason="Qwen2.5-VL require transformers version no lower than 4.49.0",
191 -)
186 def test_qwen25vl_lora(qwen25vl_lora_files):
187 """Test Qwen 2.5 VL model with LoRA"""
188 config = TestConfig(model_path=QWEN25VL_MODEL_PATH,
tests/models/multimodal/generation/test_common.py
+26 -1
@@ -702,13 +702,38 @@ VLM_TEST_SETTINGS = {
702 "smolvlm": VLMTestInfo(
703 models=["HuggingFaceTB/SmolVLM2-2.2B-Instruct"],
704 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
705 - prompt_formatter=lambda img_prompt:f"<|im_start|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
705 + prompt_formatter=lambda img_prompt: f"<|im_start|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
706 img_idx_to_prompt=lambda idx: "<image>",
707 max_model_len=8192,
708 max_num_seqs=2,
709 auto_cls=AutoModelForImageTextToText,
710 hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,
711 ),
712 + "tarsier": VLMTestInfo(
713 + models=["omni-research/Tarsier-7b"],
714 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
715 + prompt_formatter=lambda img_prompt: f"USER: {img_prompt} ASSISTANT:",
716 + max_model_len=4096,
717 + max_num_seqs=2,
718 + auto_cls=AutoModelForImageTextToText,
719 + patch_hf_runner=model_utils.tarsier_patch_hf_runner,
720 + ),
721 + "tarsier2": VLMTestInfo(
722 + models=["omni-research/Tarsier2-Recap-7b"],
723 + test_type=(
724 + VLMTestType.IMAGE,
725 + VLMTestType.MULTI_IMAGE,
726 + VLMTestType.VIDEO,
727 + ),
728 + prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
729 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
730 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
731 + max_model_len=4096,
732 + max_num_seqs=2,
733 + auto_cls=AutoModelForImageTextToText,
734 + image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
735 + marks=[pytest.mark.skip("Model initialization hangs")],
736 + ),
737 ### Tensor parallel / multi-gpu broadcast tests
738 "chameleon-broadcast": VLMTestInfo(
739 models=["facebook/chameleon-7b"],
tests/models/multimodal/generation/vlm_utils/model_utils.py
+12
@@ -818,3 +818,15 @@ def qwen2_5_omni_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
818 thinker.get_output_embeddings = lambda: thinker.lm_head
819 hf_model.model = thinker
820 return hf_model
821 +
822 +
823 +def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
824 + from vllm.model_executor.models.tarsier import get_vision_encoder_info
825 +
826 + vision_encoder_info = get_vision_encoder_info(hf_model.config)
827 +
828 + hf_processor = hf_model.processor
829 + if hf_processor.patch_size is None:
830 + hf_processor.patch_size = vision_encoder_info.get_patch_size()
831 +
832 + return hf_model
tests/models/multimodal/processing/test_transformers.py
+1 -1
@@ -16,7 +16,7 @@ def test_multimodal_processor(model_id):
16 model_impl="transformers",
17 )
18
19 - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config, )
19 + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
20
21 image_pil = ImageAsset('cherry_blossom').pil_image
22 mm_data = {"image": image_pil}
tests/models/registry.py
+1 -2
@@ -465,8 +465,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
465 is_available_online=False),
466 "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
467 trust_remote_code=True),
468 - "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b", # noqa: E501
469 - hf_overrides={"architectures": ["TarsierForConditionalGeneration"]}), # noqa: E501
468 + "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"), # noqa: E501
469 "Tarsier2ForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier2-Recap-7b", # noqa: E501
470 hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]}), # noqa: E501
471 "VoxtralForConditionalGeneration": _HfExamplesInfo(
tests/multimodal/test_processing.py
+70 -37
@@ -2,16 +2,15 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from contextlib import nullcontext
5 -from types import MethodType
6 -from typing import cast
5 +from typing import Optional, cast
6 from unittest.mock import MagicMock
7
8 import numpy as np
9 import pytest
10 import torch
12 -from transformers import ProcessorMixin
11
12 from vllm.config import ModelConfig
13 +from vllm.inputs import InputProcessingContext
14 from vllm.multimodal import MULTIMODAL_REGISTRY
15 from vllm.multimodal.inputs import (MultiModalFieldElem, MultiModalKwargs,
16 MultiModalKwargsItem,
@@ -1013,57 +1012,91 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
1012 )
1013
1014
1016 -class _ProcessorProxy:
1015 +class DummyProcessor:
1016
1018 - def __init__(self, processor: ProcessorMixin) -> None:
1017 + def __init__(self, a: int = 0, b: int = 0) -> None:
1018 super().__init__()
1019
1021 - self.__processor = processor
1022 -
1023 - def __getattr__(self, key: str):
1024 - return getattr(self.__processor, key)
1020 + self.a = a
1021 + self.b = b
1022
1023 def __call__(
1024 self,
1028 - text=None,
1029 - images=None,
1030 - videos=None,
1031 - exists=None,
1032 - return_tensors=None,
1033 - ):
1034 - return dict(exists=exists)
1025 + a: int = 0,
1026 + c: int = 0,
1027 + return_tensors: Optional[str] = None,
1028 + ) -> dict[str, int]:
1029 + return dict(a=a, c=c)
1030
1031
1037 -@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
1032 # yapf: disable
1033 +@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
1034 @pytest.mark.parametrize(
1040 - ("call_kwargs", "expected_kwargs"),
1035 + ("config_kwargs", "inference_kwargs", "expected_kwargs"),
1036 [
1042 - # Should ignore invalid kwargs
1043 - ({"does_not_exist": 100}, {"exists": None}),
1044 - ({"exists": 1}, {"exists": 1}),
1045 - ({"does_not_exist": 100, "exists": 1}, {"exists": 1}),
1037 + ({"a": 1}, {}, {"a": 1, "b": 0}),
1038 + ({}, {"a": 1}, {"a": 1, "b": 0}),
1039 + # inference_kwargs should take precedence
1040 + ({"a": 1}, {"a": 2}, {"a": 2, "b": 0}),
1041 + # Should ignore extra kwargs
1042 + ({"a": 1, "c": 1}, {}, {"a": 1, "b": 0}),
1043 + ({"b": 1, "c": 1}, {}, {"a": 0, "b": 1}),
1044 ],
1045 )
1046 # yapf: enable
1049 -def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):
1050 - model_config = ModelConfig(model_id)
1047 +def test_hf_processor_init_kwargs(
1048 + model_id,
1049 + config_kwargs,
1050 + inference_kwargs,
1051 + expected_kwargs,
1052 +):
1053 + # Should not be used since there is nothing to convert to tokens
1054 + mock_tokenizer = cast(AnyTokenizer, object())
1055
1052 - processor = MULTIMODAL_REGISTRY.create_processor(model_config)
1053 - orig_get_hf_processor = processor.info.get_hf_processor
1056 + ctx = InputProcessingContext(
1057 + model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),
1058 + tokenizer=mock_tokenizer,
1059 + )
1060 +
1061 + processor = ctx.get_hf_processor(
1062 + DummyProcessor, # type: ignore[arg-type]
1063 + **inference_kwargs,
1064 + )
1065 +
1066 + for k, v in expected_kwargs.items():
1067 + assert getattr(processor, k) == v
1068
1055 - def get_hf_processor(self, **kwargs):
1056 - assert kwargs == call_kwargs
1057 - return _ProcessorProxy(orig_get_hf_processor())
1069
1059 - processor.info.get_hf_processor = MethodType(get_hf_processor,
1060 - processor.info)
1070 +# yapf: disable
1071 +@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
1072 +@pytest.mark.parametrize(
1073 + ("config_kwargs", "inference_kwargs", "expected_kwargs"),
1074 + [
1075 + ({"a": 1}, {}, {"a": 1, "c": 0}),
1076 + ({}, {"a": 1}, {"a": 1, "c": 0}),
1077 + # inference_kwargs should take precedence
1078 + ({"a": 1}, {"a": 2}, {"a": 2, "c": 0}),
1079 + # Should ignore extra kwargs
1080 + ({"a": 1, "c": 1}, {}, {"a": 1, "c": 1}),
1081 + ({"b": 1, "c": 1}, {}, {"a": 0, "c": 1}),
1082 + ],
1083 +)
1084 +# yapf: enable
1085 +def test_hf_processor_call_kwargs(
1086 + model_id,
1087 + config_kwargs,
1088 + inference_kwargs,
1089 + expected_kwargs,
1090 +):
1091 + # Should not be used since there is nothing to convert to tokens
1092 + mock_tokenizer = cast(AnyTokenizer, object())
1093
1062 - out_kwargs = processor._call_hf_processor(
1063 - prompt="",
1064 - mm_data={},
1065 - mm_kwargs=call_kwargs,
1066 - tok_kwargs={},
1094 + ctx = InputProcessingContext(
1095 + model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),
1096 + tokenizer=mock_tokenizer,
1097 )
1098
1069 - assert out_kwargs == expected_kwargs
1099 + processor = ctx.get_hf_processor(DummyProcessor) # type: ignore[arg-type]
1100 +
1101 + result = ctx.call_hf_processor(processor, {}, inference_kwargs)
1102 + assert result == expected_kwargs
vllm/config.py
+11 -1
@@ -11,6 +11,7 @@ import textwrap
11 import uuid
12 import warnings
13 from collections import Counter
14 +from collections.abc import Mapping
15 from contextlib import contextmanager
16 from dataclasses import (MISSING, Field, asdict, field, fields, is_dataclass,
17 replace)
@@ -3332,7 +3333,16 @@ class MultiModalConfig:
3333 999 if envs.VLLM_USE_V1 else 1,
3334 )
3335
3335 - # TODO: Add configs to init vision tower or not.
3336 + def merge_mm_processor_kwargs(
3337 + self,
3338 + inference_kwargs: Mapping[str, object],
3339 + ) -> dict[str, object]:
3340 + """
3341 + Get the keyword arguments to pass to the multi-modal processor
3342 + according to the extra arguments passed during inference.
3343 + """
3344 + kwargs = self.mm_processor_kwargs or {}
3345 + return kwargs | dict(inference_kwargs)
3346
3347
3348 @config
vllm/inputs/registry.py
+8 -9
@@ -11,7 +11,7 @@ from typing_extensions import TypeVar
11 from vllm.jsontree import JSONTree, json_map_leaves
12 from vllm.logger import init_logger
13 from vllm.transformers_utils.processor import cached_processor_from_config
14 -from vllm.utils import resolve_mm_processor_kwargs
14 +from vllm.utils import get_allowed_kwarg_only_overrides
15
16 if TYPE_CHECKING:
17 from vllm.config import ModelConfig
@@ -154,14 +154,11 @@ class InputProcessingContext(InputContext):
154 assert callable(hf_processor)
155
156 mm_config = self.model_config.get_multimodal_config()
157 - base_kwargs = mm_config.mm_processor_kwargs
158 - if base_kwargs is None:
159 - base_kwargs = {}
157 + merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
158
161 - merged_kwargs = resolve_mm_processor_kwargs(
162 - base_kwargs,
163 - kwargs,
159 + allowed_kwargs = get_allowed_kwarg_only_overrides(
160 hf_processor,
161 + merged_kwargs,
162 requires_kw_only=False,
163 allow_var_kwargs=True,
164 )
@@ -173,7 +170,9 @@ class InputProcessingContext(InputContext):
170 return x
171
172 try:
176 - output = hf_processor(**data, **merged_kwargs, return_tensors="pt")
173 + output = hf_processor(**data,
174 + **allowed_kwargs,
175 + return_tensors="pt")
176 # this emulates output.to(dtype=self.model_config.dtype)
177 if isinstance(output, BatchFeature):
178 cast_output = json_map_leaves(maybe_cast_dtype, output.data)
@@ -189,7 +188,7 @@ class InputProcessingContext(InputContext):
188
189 except Exception as exc:
190 msg = (f"Failed to apply {type(hf_processor).__name__} "
192 - f"on data={data} with kwargs={merged_kwargs}")
191 + f"on data={data} with kwargs={allowed_kwargs}")
192
193 raise ValueError(msg) from exc
194
vllm/model_executor/models/aya_vision.py
+3 -9
@@ -123,16 +123,10 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
123 return self.ctx.get_hf_config(AyaVisionConfig)
124
125 def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:
126 - processor = self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
126 + return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
127
128 - # Temporary workaround since this processor has multiple image tokens
129 - # See https://github.com/huggingface/transformers/issues/38350
130 - processor._check_special_mm_tokens = lambda *args, **kwargs: None
131 -
132 - return processor
133 -
134 - def get_image_processor(self) -> GotOcr2ImageProcessor:
135 - return self.get_hf_processor().image_processor
128 + def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor:
129 + return self.get_hf_processor(**kwargs).image_processor
130
131 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
132 return {"image": None}
vllm/model_executor/models/deepseek_vl2.py
+18 -18
@@ -214,25 +214,25 @@ class DeepseekVL2MultiModalProcessor(
214 mm_kwargs: Mapping[str, object],
215 tok_kwargs: Mapping[str, object],
216 ) -> BatchFeature:
217 - if mm_data:
218 - processed_outputs = self.info.ctx.call_hf_processor(
219 - self.info.get_hf_processor(**mm_kwargs),
220 - dict(prompt=prompt, **mm_data),
221 - dict(**mm_kwargs, **tok_kwargs),
222 - )
223 - pixel_values = processed_outputs["pixel_values"]
224 - # split pixel values into patches corresponding to each image
225 - images_spatial_crop = processed_outputs["images_spatial_crop"]
226 - patches_per_image = [
227 - x.prod().item() + 1 for x in images_spatial_crop
228 - ]
229 - pixel_values = pixel_values.split(patches_per_image)
230 - processed_outputs["pixel_values"] = pixel_values
231 - else:
217 + if not mm_data:
218 tokenizer = self.info.get_tokenizer()
233 - processed_outputs = tokenizer(prompt,
234 - add_special_tokens=True,
235 - return_tensors="pt")
219 + return tokenizer(prompt,
220 + add_special_tokens=True,
221 + return_tensors="pt")
222 +
223 + processed_outputs = super()._call_hf_processor(
224 + prompt=prompt,
225 + mm_data=mm_data,
226 + mm_kwargs=mm_kwargs,
227 + tok_kwargs=tok_kwargs,
228 + )
229 +
230 + pixel_values = processed_outputs["pixel_values"]
231 + # split pixel values into patches corresponding to each image
232 + images_spatial_crop = processed_outputs["images_spatial_crop"]
233 + patches_per_image = [x.prod().item() + 1 for x in images_spatial_crop]
234 + pixel_values = pixel_values.split(patches_per_image)
235 + processed_outputs["pixel_values"] = pixel_values
236
237 return processed_outputs
238
vllm/model_executor/models/florence2.py
-6
@@ -761,12 +761,6 @@ class Florence2LanguageForConditionalGeneration(nn.Module, SupportsV0Only):
761
762 class Florence2ProcessingInfo(BaseProcessingInfo):
763
764 - def get_hf_config(self):
765 - return self.ctx.get_hf_config()
766 -
767 - def get_hf_processor(self):
768 - return self.ctx.get_hf_processor()
769 -
764 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
765 return {"image": 1}
766
vllm/model_executor/models/fuyu.py
+2 -2
@@ -83,8 +83,8 @@ class FuyuProcessingInfo(BaseProcessingInfo):
83 def get_hf_processor(self, **kwargs: object):
84 return self.ctx.get_hf_processor(FuyuProcessor, **kwargs)
85
86 - def get_image_processor(self) -> FuyuImageProcessor:
87 - return self.get_hf_processor().image_processor
86 + def get_image_processor(self, **kwargs: object) -> FuyuImageProcessor:
87 + return self.get_hf_processor(**kwargs).image_processor
88
89 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
90 return {"image": 1}
vllm/model_executor/models/glm4_1v.py
+4 -4
@@ -809,11 +809,11 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
809 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
810 return {"image": None, "video": 1}
811
812 - def get_image_processor(self) -> Glm4vImageProcessor:
813 - return self.get_hf_processor().image_processor
812 + def get_image_processor(self, **kwargs: object) -> Glm4vImageProcessor:
813 + return self.get_hf_processor(**kwargs).image_processor
814
815 - def get_video_processor(self) -> Glm4vVideoProcessor:
816 - return self.get_hf_processor().video_processor
815 + def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:
816 + return self.get_hf_processor(**kwargs).video_processor
817
818 def _get_vision_info(
819 self,
vllm/model_executor/models/h2ovl.py
+1 -15
@@ -392,21 +392,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
392
393 class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
394
395 - def get_hf_processor(
396 - self,
397 - *,
398 - min_dynamic_patch: Optional[int] = None,
399 - max_dynamic_patch: Optional[int] = None,
400 - dynamic_image_size: Optional[bool] = None,
401 - **kwargs: object,
402 - ) -> H2OVLProcessor:
403 - if min_dynamic_patch is not None:
404 - kwargs["min_dynamic_patch"] = min_dynamic_patch
405 - if max_dynamic_patch is not None:
406 - kwargs["max_dynamic_patch"] = max_dynamic_patch
407 - if dynamic_image_size is not None:
408 - kwargs["dynamic_image_size"] = dynamic_image_size
409 -
395 + def get_hf_processor(self, **kwargs: object) -> H2OVLProcessor:
396 return self.ctx.init_processor(
397 H2OVLProcessor,
398 config=self.get_hf_config(),
vllm/model_executor/models/hyperclovax_vision.py
+1 -19
@@ -25,8 +25,7 @@ import torch
25 import torch.nn as nn
26 from timm.layers import LayerNorm, LayerNorm2d
27 from timm.models.regnet import RegStage
28 -from transformers import (AutoProcessor, BatchFeature, CLIPVisionConfig,
29 - SiglipVisionConfig)
28 +from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig
29 from transformers.modeling_utils import no_init_weights
30
31 from vllm.config import VllmConfig
@@ -80,26 +79,9 @@ HCXVisionMultimodalInputs = Union[HCXVisionMultimodalPixelInputs]
79
80 class HCXVisionProcessingInfo(BaseProcessingInfo):
81
83 - def get_hf_config(self):
84 - return self.ctx.get_hf_config()
85 -
82 def get_vision_encoder_info(self):
83 return get_vision_encoder_info(self.get_hf_config())
84
89 - def get_hf_processor(
90 - self,
91 - **kwargs: object,
92 - ):
93 - processor_cls = type(
94 - AutoProcessor.from_pretrained(
95 - self.ctx.model_config.model,
96 - trust_remote_code=self.ctx.model_config.trust_remote_code,
97 - ))
98 - return self.ctx.get_hf_processor(
99 - processor_cls,
100 - **kwargs,
101 - )
102 -
85 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
86 return {"image": None, "video": None}
87
vllm/model_executor/models/idefics3.py
+1 -9
@@ -88,15 +88,7 @@ ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
88
89 class Idefics3ProcessingInfo(BaseProcessingInfo):
90
91 - def get_hf_processor(
92 - self,
93 - *,
94 - size: Optional[dict[str, int]] = None,
95 - **kwargs: object,
96 - ) -> Idefics3Processor:
97 - if size is not None:
98 - kwargs["size"] = size
99 -
91 + def get_hf_processor(self, **kwargs: object) -> Idefics3Processor:
92 return self.ctx.get_hf_processor(Idefics3Processor, **kwargs)
93
94 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
vllm/model_executor/models/internvl.py
+3 -25
@@ -665,14 +665,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
665 """Basic image-only ProcessingInfo for InternVL-style models."""
666
667 @abstractmethod
668 - def get_hf_processor(
669 - self,
670 - *,
671 - min_dynamic_patch: Optional[int] = None,
672 - max_dynamic_patch: Optional[int] = None,
673 - dynamic_image_size: Optional[bool] = None,
674 - **kwargs: object,
675 - ) -> BaseInternVLProcessor:
668 + def get_hf_processor(self, **kwargs: object) -> BaseInternVLProcessor:
669 raise NotImplementedError
670
671 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
@@ -882,27 +875,12 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
875
876 return max(max_frames_per_video, 1)
877
885 - def get_hf_processor(
886 - self,
887 - *,
888 - min_dynamic_patch: Optional[int] = None,
889 - max_dynamic_patch: Optional[int] = None,
890 - dynamic_image_size: Optional[bool] = None,
891 - **kwargs: object,
892 - ) -> InternVLProcessor:
893 - if min_dynamic_patch is not None:
894 - kwargs["min_dynamic_patch"] = min_dynamic_patch
895 - if max_dynamic_patch is not None:
896 - kwargs["max_dynamic_patch"] = max_dynamic_patch
897 - if dynamic_image_size is not None:
898 - kwargs["dynamic_image_size"] = dynamic_image_size
899 -
900 - kwargs["video_token"] = self.get_video_token()
901 -
878 + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
879 return self.ctx.init_processor(
880 InternVLProcessor,
881 config=self.get_hf_config(),
882 tokenizer=self.get_tokenizer(),
883 + video_token=self.get_video_token(),
884 **kwargs,
885 )
886
vllm/model_executor/models/keye.py
+2 -82
@@ -44,8 +44,6 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
44 from vllm.platforms import _Backend
45 from vllm.sequence import IntermediateTensors
46 from vllm.transformers_utils.config import uses_mrope
47 -from vllm.transformers_utils.processor import (
48 - cached_image_processor_from_config)
47 from vllm.utils.tensor_schema import TensorSchema, TensorShape
48
49 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -980,72 +978,8 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
978
979 class KeyeProcessingInfo(BaseProcessingInfo):
980
983 - def get_hf_processor(
984 - self,
985 - *,
986 - min_pixels: Optional[int] = None,
987 - max_pixels: Optional[int] = None,
988 - size: Optional[dict[str, int]] = None,
989 - **kwargs: object,
990 - ):
991 - return self.ctx.get_hf_processor(
992 - image_processor=self.get_image_processor(
993 - min_pixels=min_pixels,
994 - max_pixels=max_pixels,
995 - size=size,
996 - ),
997 - **kwargs,
998 - )
999 -
1000 - def _get_image_processor_kwargs(
1001 - self,
1002 - *,
1003 - min_pixels: Optional[int] = None,
1004 - max_pixels: Optional[int] = None,
1005 - size: Optional[dict[str, int]] = None,
1006 - **kwargs: object,
1007 - ):
1008 - if self.ctx.model_config.mm_processor_kwargs:
1009 - kwargs.update(self.ctx.model_config.mm_processor_kwargs)
1010 -
1011 - if min_pixels is not None:
1012 - kwargs["min_pixels"] = min_pixels
1013 -
1014 - if size is None:
1015 - size = {"shortest_edge": min_pixels}
1016 - else:
1017 - size["shortest_edge"] = min_pixels
1018 -
1019 - if max_pixels is not None:
1020 - kwargs["max_pixels"] = max_pixels
1021 -
1022 - if size is None:
1023 - size = {"longest_edge": max_pixels}
1024 - else:
1025 - size["longest_edge"] = max_pixels
1026 -
1027 - if size is not None:
1028 - kwargs["size"] = size
1029 -
1030 - return kwargs
1031 -
1032 - def get_image_processor(
1033 - self,
1034 - *,
1035 - min_pixels: Optional[int] = None,
1036 - max_pixels: Optional[int] = None,
1037 - size: Optional[dict[str, int]] = None,
1038 - **kwargs: object,
1039 - ):
1040 - return cached_image_processor_from_config(
1041 - self.ctx.model_config,
1042 - **self._get_image_processor_kwargs(
1043 - min_pixels=min_pixels,
1044 - max_pixels=max_pixels,
1045 - size=size,
1046 - **kwargs,
1047 - ),
1048 - )
981 + def get_image_processor(self, **kwargs: object):
982 + return self.get_hf_processor(**kwargs).image_processor
983
984 def get_supported_mm_limits(self, ) -> Mapping[str, Optional[int]]:
985 return {"image": None, "video": None}
@@ -1246,20 +1180,6 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
1180 def _get_data_parser(self) -> MultiModalDataParser:
1181 return KeyeMultiModalDataParser()
1182
1249 - def _call_hf_processor(
1250 - self,
1251 - prompt: str,
1252 - mm_data: Mapping[str, object],
1253 - mm_kwargs: Mapping[str, object],
1254 - tok_kwargs: Mapping[str, object],
1255 - ) -> BatchFeature:
1256 - mm_kwargs = self.info._get_image_processor_kwargs(**mm_kwargs)
1257 - return self.info.ctx.call_hf_processor(
1258 - self.info.get_hf_processor(**mm_kwargs),
1259 - dict(text=prompt, **mm_data),
1260 - dict(**mm_kwargs, **tok_kwargs),
1261 - )
1262 -
1183 def _get_prompt_updates(
1184 self,
1185 mm_items: MultiModalDataItems,
vllm/model_executor/models/llava.py
+11 -35
@@ -8,11 +8,9 @@ from typing import (Final, Literal, Optional, Protocol, TypedDict, TypeVar,
8
9 import torch
10 import torch.nn as nn
11 -from packaging.version import Version
11 from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
12 PixtralVisionConfig, PretrainedConfig,
13 SiglipVisionConfig)
15 -from transformers import __version__ as TRANSFORMERS_VERSION
14 from transformers.models.llava import LlavaProcessor
15 from transformers.models.pixtral import PixtralProcessor
16
@@ -307,29 +305,14 @@ class PixtralHFMultiModalProcessor(
305
306 pixel_values = processed_outputs.get("pixel_values")
307 if pixel_values is not None:
310 - # Before/after https://github.com/huggingface/transformers/pull/35122
311 - if Version(TRANSFORMERS_VERSION) <= Version("4.48.3"):
312 - images = mm_data["images"]
313 - assert isinstance(images, list)
314 -
315 - # Original output: (1, num_images, C, H, W)
316 - # New output: (num_images, C, H, W)
317 - assert (isinstance(pixel_values, list)
318 - and len(pixel_values) == 1)
319 - assert (isinstance(pixel_values[0], list)
320 - and len(pixel_values[0]) == len(images))
321 -
322 - processed_outputs["pixel_values"] = pixel_values[0]
323 - else:
324 - # Avoid padding since we need the output for each image to be
325 - # independent of other images for the cache to work correctly
326 - image_sizes = processed_outputs["image_sizes"]
327 - assert len(pixel_values) == len(image_sizes)
308 + # Avoid padding since we need the output for each image to be
309 + # independent of other images for the cache to work correctly
310 + image_sizes = processed_outputs["image_sizes"]
311 + assert len(pixel_values) == len(image_sizes)
312
329 - processed_outputs["pixel_values"] = [
330 - p[:, :h, :w]
331 - for p, (h, w) in zip(pixel_values, image_sizes)
332 - ]
313 + processed_outputs["pixel_values"] = [
314 + p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
315 + ]
316
317 return processed_outputs
318
@@ -784,17 +767,10 @@ class MantisProcessingInfo(LlavaProcessingInfo):
767 vision_info = self.get_vision_encoder_info()
768
769 kwargs.setdefault("patch_size", vision_info.get_patch_size())
787 -
788 - if Version(TRANSFORMERS_VERSION) < Version("4.48"):
789 - # BUG: num_additional_image_tokens = 0 but treated as 1,
790 - # so we set vision_feature_select_strategy to None to offset this
791 - kwargs.setdefault("vision_feature_select_strategy", None)
792 - else:
793 - # FIXED: https://github.com/huggingface/transformers/pull/33424/files#diff-6a37acc21efcadaae622b079b2712a131131448ff64262bd219aa346aeec38faL150
794 - kwargs.setdefault(
795 - "vision_feature_select_strategy",
796 - hf_config.vision_feature_select_strategy,
797 - )
770 + kwargs.setdefault(
771 + "vision_feature_select_strategy",
772 + hf_config.vision_feature_select_strategy,
773 + )
774
775 return self.ctx.get_hf_processor(LlavaProcessor, **kwargs)
776
vllm/model_executor/models/minicpmv.py
+2 -4
@@ -331,10 +331,8 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
331
332 return hf_processor
333
334 - def get_image_processor(self):
335 - hf_processor = self.get_hf_processor()
336 - image_processor = hf_processor.image_processor # type: ignore
337 - return image_processor
334 + def get_image_processor(self, **kwargs: object):
335 + return self.get_hf_processor(**kwargs).image_processor
336
337 def get_model_version(self):
338 return get_version_by_config(self.get_hf_config())
vllm/model_executor/models/mllama4.py
+1 -1
@@ -533,7 +533,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
533
534 def get_hf_processor(self, **kwargs: object) -> Llama4Processor:
535 return self.ctx.get_hf_processor(Llama4Processor,
536 - use_fast=True,
536 + use_fast=kwargs.pop("use_fast", True),
537 **kwargs)
538
539 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
vllm/model_executor/models/nemotron_vl.py
+3 -21
@@ -137,34 +137,16 @@ class NemotronVLProcessor(InternVLProcessor):
137 class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
138 """Processing info for Nemotron VL models."""
139
140 - def get_hf_processor(
141 - self,
142 - *,
143 - min_dynamic_patch: Optional[int] = None,
144 - max_dynamic_patch: Optional[int] = None,
145 - dynamic_image_size: Optional[bool] = None,
146 - **kwargs: object,
147 - ) -> NemotronVLProcessor:
148 - if min_dynamic_patch is not None:
149 - kwargs["min_dynamic_patch"] = min_dynamic_patch
150 - if max_dynamic_patch is not None:
151 - kwargs["max_dynamic_patch"] = max_dynamic_patch
152 - if dynamic_image_size is not None:
153 - kwargs["dynamic_image_size"] = dynamic_image_size
154 -
155 - image_processor = self.get_image_processor()
140 + def get_hf_processor(self, **kwargs: object) -> NemotronVLProcessor:
141 return self.ctx.init_processor(
142 NemotronVLProcessor,
143 config=self.get_hf_config(),
144 tokenizer=self.get_tokenizer(),
160 - image_processor=image_processor,
145 + image_processor=self.get_image_processor(),
146 **kwargs,
147 )
148
164 - def get_image_processor(
165 - self,
166 - **kwargs: object,
167 - ):
149 + def get_image_processor(self, **kwargs: object):
150 return cached_image_processor_from_config(
151 self.ctx.model_config,
152 **kwargs,
vllm/model_executor/models/nvlm_d.py
+1 -15
@@ -63,21 +63,7 @@ class NVLMProcessor(BaseInternVLProcessor):
63
64 class NVLMProcessingInfo(BaseInternVLProcessingInfo):
65
66 - def get_hf_processor(
67 - self,
68 - *,
69 - min_dynamic_patch: Optional[int] = None,
70 - max_dynamic_patch: Optional[int] = None,
71 - dynamic_image_size: Optional[bool] = None,
72 - **kwargs: object,
73 - ) -> NVLMProcessor:
74 - if min_dynamic_patch is not None:
75 - kwargs["min_dynamic_patch"] = min_dynamic_patch
76 - if max_dynamic_patch is not None:
77 - kwargs["max_dynamic_patch"] = max_dynamic_patch
78 - if dynamic_image_size is not None:
79 - kwargs["dynamic_image_size"] = dynamic_image_size
80 -
66 + def get_hf_processor(self, **kwargs: object) -> NVLMProcessor:
67 return self.ctx.init_processor(
68 NVLMProcessor,
69 config=self.get_hf_config(),
vllm/model_executor/models/ovis.py
+3 -5
@@ -25,7 +25,7 @@ import torch
25 import torch.nn as nn
26 from torch import Tensor
27 from torch.nn.functional import gumbel_softmax, pad, softmax
28 -from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig
28 +from transformers import BatchFeature, PretrainedConfig
29
30 from vllm.config import VllmConfig
31 from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -245,11 +245,12 @@ class VisualEmbedding(torch.nn.Embedding):
245
246 class OvisProcessingInfo(BaseProcessingInfo):
247
248 - def get_hf_processor(self, **kwargs):
248 + def get_hf_processor(self, **kwargs: object):
249 return self.ctx.get_hf_processor(
250 OvisProcessor,
251 image_pad_token=self.get_image_pad_token(),
252 image_segment_len=self.get_image_segment_len(),
253 + **kwargs,
254 )
255
256 def get_image_segment_len(self) -> int:
@@ -269,9 +270,6 @@ class OvisProcessingInfo(BaseProcessingInfo):
270 text_model_type = hf_text_config.model_type
271 return IMAGE_PAD_TOKEN_MAP.get(text_model_type)
272
272 - def get_image_processor(self) -> BaseImageProcessor:
273 - return self.get_hf_processor().image_processor # type: ignore
274 -
273 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
274 return {"image": None}
275
vllm/model_executor/models/phi3v.py
-11
@@ -318,17 +318,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
318
319 class Phi3VProcessingInfo(BaseProcessingInfo):
320
321 - def get_hf_processor(
322 - self,
323 - *,
324 - num_crops: Optional[int] = None,
325 - **kwargs: object,
326 - ) -> ProcessorMixin:
327 - if num_crops is not None:
328 - kwargs["num_crops"] = num_crops
329 -
330 - return self.ctx.get_hf_processor(**kwargs)
331 -
321 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
322 return {"image": None}
323
vllm/model_executor/models/phi4_multimodal.py
+8 -14
@@ -696,19 +696,12 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
696 def get_hf_config(self) -> Phi4MultimodalConfig:
697 return self.ctx.get_hf_config(Phi4MultimodalConfig)
698
699 - def get_hf_processor(
700 - self,
701 - *,
702 - dynamic_hd: Optional[int] = None,
703 - **kwargs: object,
704 - ) -> Phi4MMProcessor:
705 - if dynamic_hd is not None:
706 - kwargs["dynamic_hd"] = dynamic_hd
707 -
708 - return self.ctx.get_hf_processor(**kwargs)
699 + def get_hf_processor(self, **kwargs: object) -> Phi4MMProcessor:
700 + return self.ctx.get_hf_processor(Phi4MMProcessor, **kwargs)
701
710 - def get_feature_extractor(self) -> Phi4MultimodalFeatureExtractor:
711 - return self.get_hf_processor().audio_processor
702 + def get_feature_extractor(
703 + self, **kwargs: object) -> Phi4MultimodalFeatureExtractor:
704 + return self.get_hf_processor(**kwargs).audio_processor
705
706 def get_image_processor(
707 self,
@@ -1007,7 +1000,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1000
1001 if audio_data:
1002 audio_features = processed_outputs['audio_input_features']
1010 - sr = self.info.get_feature_extractor().sampling_rate
1003 + sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
1004 feature_sizes = [
1005 self.info.get_audio_num_frames(len(audio), sr)
1006 for audio in audio_data
@@ -1043,7 +1036,8 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1036 audio_token_id = tokenizer.vocab[tokenizer.audio_token]
1037
1038 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1046 - audio_processor = self.info.get_feature_extractor()
1039 + audio_processor = self.info.get_feature_extractor(
1040 + **hf_processor_mm_kwargs)
1041
1042 def get_image_replacement_phi4mm(item_idx: int):
1043 images = mm_items.get_items(
vllm/model_executor/models/phi4mm.py
+6 -15
@@ -459,17 +459,6 @@ def cat_with_pad(tensors, dim, padding_value=0):
459
460 class Phi4MMProcessingInfo(BaseProcessingInfo):
461
462 - def get_hf_processor(
463 - self,
464 - *,
465 - dynamic_hd: Optional[int] = None,
466 - **kwargs: object,
467 - ) -> ProcessorMixin:
468 - if dynamic_hd is not None:
469 - kwargs["dynamic_hd"] = dynamic_hd
470 -
471 - return self.ctx.get_hf_processor(**kwargs)
472 -
462 @property
463 def image_tokens(self) -> list[str]:
464 return [f"<|image_{i+1}|>" for i in range(100)]
@@ -487,8 +476,9 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
476 image_processor = processor.image_processor
477 return image_processor.dynamic_hd
478
490 - def get_feature_extractor(self) -> SequenceFeatureExtractor:
491 - return self.get_hf_processor().audio_processor
479 + def get_feature_extractor(self,
480 + **kwargs: object) -> SequenceFeatureExtractor:
481 + return self.get_hf_processor(**kwargs).audio_processor
482
483 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
484 return {"audio": None, "image": None}
@@ -769,7 +759,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
759 prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
760 return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
761
772 - sr = self.info.get_feature_extractor().sampling_rate
762 + sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
763 if (audio_data := mm_data.get("audios", [])):
764 mm_data['audios'] = [(data, sr) for data in audio_data]
765
@@ -816,7 +806,8 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
806 ) -> Sequence[PromptUpdate]:
807 image_tokens: list[str] = self.info.image_tokens # type: ignore
808 audio_tokens: list[str] = self.info.audio_tokens # type: ignore
819 - feature_extractor = self.info.get_feature_extractor()
809 + feature_extractor = self.info.get_feature_extractor(
810 + **hf_processor_mm_kwargs)
811 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
812
813 def get_image_replacement_phi4mm(item_idx: int):
vllm/model_executor/models/qwen2_5_omni_thinker.py
+7 -42
@@ -132,50 +132,15 @@ class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo,
132 def get_hf_config(self):
133 return self.ctx.get_hf_config(Qwen2_5OmniConfig).thinker_config
134
135 - def get_hf_processor(
136 - self,
137 - *,
138 - sampling_rate: Optional[int] = None,
139 - min_pixels: Optional[int] = None,
140 - max_pixels: Optional[int] = None,
141 - size: Optional[dict[str, int]] = None,
142 - fps: Optional[Union[float, list[float]]] = None,
143 - **kwargs: object,
144 - ) -> Qwen2_5OmniProcessor:
145 - if fps is not None:
146 - kwargs["fps"] = fps
147 -
148 - # Monkey patch for Transformers v4.53
149 - processor_class = Qwen2_5OmniProcessor
150 - if processor_class.image_processor_class != "AutoImageProcessor":
151 - processor_class.image_processor_class = "AutoImageProcessor"
152 - if processor_class.video_processor_class != "AutoVideoProcessor":
153 - processor_class.video_processor_class = "AutoVideoProcessor"
154 -
155 - processor = self.ctx.get_hf_processor(
156 - processor_class,
157 - image_processor=self.get_image_processor(min_pixels=min_pixels,
158 - max_pixels=max_pixels,
159 - size=size,
160 - use_fast=kwargs.get(
161 - "use_fast", True)),
135 + def get_hf_processor(self, **kwargs: object) -> Qwen2_5OmniProcessor:
136 + return self.ctx.get_hf_processor(
137 + Qwen2_5OmniProcessor,
138 + use_fast=kwargs.pop("use_fast", True),
139 **kwargs,
140 )
164 - if not hasattr(processor, "audio_token"):
165 - processor.audio_token = "<|AUDIO|>"
166 - if not hasattr(processor, "image_token"):
167 - processor.image_token = "<|IMAGE|>"
168 - if not hasattr(processor, "video_token"):
169 - processor.video_token = "<|VIDEO|>"
170 - return processor
171 -
172 - def get_feature_extractor(
173 - self,
174 - *,
175 - sampling_rate: Optional[int] = None,
176 - **kwargs: object,
177 - ):
178 - hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
141 +
142 + def get_feature_extractor(self, **kwargs: object):
143 + hf_processor = self.get_hf_processor(**kwargs)
144 feature_extractor = hf_processor.feature_extractor # type: ignore
145 assert isinstance(feature_extractor, WhisperFeatureExtractor)
146 return feature_extractor
vllm/model_executor/models/qwen2_5_vl.py
+2 -17
@@ -780,25 +780,10 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
780 def get_hf_config(self):
781 return self.ctx.get_hf_config(Qwen2_5_VLConfig)
782
783 - def get_hf_processor(
784 - self,
785 - *,
786 - min_pixels: Optional[int] = None,
787 - max_pixels: Optional[int] = None,
788 - size: Optional[dict[str, int]] = None,
789 - fps: Optional[Union[float, list[float]]] = None,
790 - **kwargs: object,
791 - ) -> Qwen2_5_VLProcessor:
792 - if fps is not None:
793 - kwargs["fps"] = fps
794 -
783 + def get_hf_processor(self, **kwargs: object) -> Qwen2_5_VLProcessor:
784 return self.ctx.get_hf_processor(
785 Qwen2_5_VLProcessor,
797 - image_processor=self.get_image_processor(min_pixels=min_pixels,
798 - max_pixels=max_pixels,
799 - size=size,
800 - use_fast=kwargs.get(
801 - "use_fast", True)),
786 + use_fast=kwargs.pop("use_fast", True),
787 **kwargs,
788 )
789
vllm/model_executor/models/qwen2_audio.py
+4 -14
@@ -86,22 +86,12 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
86 def get_hf_config(self):
87 return self.ctx.get_hf_config(Qwen2AudioConfig)
88
89 - def get_hf_processor(
90 - self,
91 - *,
92 - # Ignored in initialization
93 - sampling_rate: Optional[int] = None,
94 - **kwargs: object,
95 - ) -> Qwen2AudioProcessor:
89 + def get_hf_processor(self, **kwargs: object) -> Qwen2AudioProcessor:
90 return self.ctx.get_hf_processor(Qwen2AudioProcessor, **kwargs)
91
98 - def get_feature_extractor(
99 - self,
100 - *,
101 - # Ignored in initialization
102 - sampling_rate: Optional[int] = None,
103 - ) -> WhisperFeatureExtractor:
104 - hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
92 + def get_feature_extractor(self,
93 + **kwargs: object) -> WhisperFeatureExtractor:
94 + hf_processor = self.get_hf_processor(**kwargs)
95 feature_extractor = hf_processor.feature_extractor # type: ignore
96 assert isinstance(feature_extractor, WhisperFeatureExtractor)
97 return feature_extractor
vllm/model_executor/models/qwen2_vl.py
+4 -78
@@ -69,8 +69,6 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
69 from vllm.platforms import _Backend, current_platform
70 from vllm.sequence import IntermediateTensors
71 from vllm.transformers_utils.config import uses_mrope
72 -from vllm.transformers_utils.processor import (
73 - cached_image_processor_from_config)
72 from vllm.transformers_utils.tokenizer import AnyTokenizer
73
74 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -752,73 +750,15 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
750 def get_hf_config(self):
751 return self.ctx.get_hf_config(Qwen2VLConfig)
752
755 - def get_hf_processor(
756 - self,
757 - *,
758 - min_pixels: Optional[int] = None,
759 - max_pixels: Optional[int] = None,
760 - size: Optional[dict[str, int]] = None,
761 - **kwargs: object,
762 - ) -> Qwen2VLProcessor:
753 + def get_hf_processor(self, **kwargs: object) -> Qwen2VLProcessor:
754 return self.ctx.get_hf_processor(
755 Qwen2VLProcessor,
765 - image_processor=self.get_image_processor(min_pixels=min_pixels,
766 - max_pixels=max_pixels,
767 - size=size,
768 - use_fast=kwargs.get(
769 - "use_fast", True)),
756 + use_fast=kwargs.pop("use_fast", True),
757 **kwargs,
758 )
759
773 - def _get_image_processor_kwargs(
774 - self,
775 - *,
776 - min_pixels: Optional[int] = None,
777 - max_pixels: Optional[int] = None,
778 - size: Optional[dict[str, int]] = None,
779 - **kwargs: object,
780 - ):
781 - mm_config = self.ctx.model_config.get_multimodal_config()
782 - if mm_config.mm_processor_kwargs:
783 - kwargs.update(mm_config.mm_processor_kwargs)
784 -
785 - if min_pixels is not None:
786 - kwargs["min_pixels"] = min_pixels
787 -
788 - if size is None:
789 - size = {"shortest_edge": min_pixels}
790 - else:
791 - size["shortest_edge"] = min_pixels
792 -
793 - if max_pixels is not None:
794 - kwargs["max_pixels"] = max_pixels
795 -
796 - if size is None:
797 - size = {"longest_edge": max_pixels}
798 - else:
799 - size["longest_edge"] = max_pixels
800 -
801 - if size is not None:
802 - kwargs["size"] = size
803 -
804 - return kwargs
805 -
806 - def get_image_processor(
807 - self,
808 - *,
809 - min_pixels: Optional[int] = None,
810 - max_pixels: Optional[int] = None,
811 - size: Optional[dict[str, int]] = None,
812 - **kwargs: object,
813 - ) -> Qwen2VLImageProcessor:
814 - kwargs["use_fast"] = kwargs.get("use_fast", True)
815 - return cached_image_processor_from_config(
816 - self.ctx.model_config,
817 - **self._get_image_processor_kwargs(min_pixels=min_pixels,
818 - max_pixels=max_pixels,
819 - size=size,
820 - **kwargs),
821 - )
760 + def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessor:
761 + return self.get_hf_processor(**kwargs).image_processor
762
763 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
764 return {"image": None, "video": None}
@@ -1023,20 +963,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
963 def _get_data_parser(self) -> MultiModalDataParser:
964 return Qwen2VLMultiModalDataParser()
965
1026 - def _call_hf_processor(
1027 - self,
1028 - prompt: str,
1029 - mm_data: Mapping[str, object],
1030 - mm_kwargs: Mapping[str, object],
1031 - tok_kwargs: Mapping[str, object],
1032 - ) -> BatchFeature:
1033 - mm_kwargs = self.info._get_image_processor_kwargs(**mm_kwargs)
1034 - return self.info.ctx.call_hf_processor(
1035 - self.info.get_hf_processor(**mm_kwargs),
1036 - dict(text=prompt, **mm_data),
1037 - dict(**mm_kwargs, **tok_kwargs),
1038 - )
1039 -
966 def _get_prompt_updates(
967 self,
968 mm_items: MultiModalDataItems,
vllm/model_executor/models/skyworkr1v.py
+20 -66
@@ -7,9 +7,8 @@
7 # Copyright (c) 2025 Skywork
8 # Licensed under The MIT License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from abc import ABC, abstractmethod
10 from collections.abc import Iterable, Mapping, Sequence
12 -from typing import Literal, Optional, TypedDict, TypeVar, Union
11 +from typing import Literal, Optional, TypedDict, Union
12
13 import torch
14 import torch.nn as nn
@@ -232,7 +231,7 @@ def image_to_pixel_values_skyworkr1v(
231 return pixel_values
232
233
235 -class BaseSkyworkR1VProcessor(ABC):
234 +class SkyworkR1VProcessor:
235 """
236 This model doesn't define its own HF processor,
237 so we implement our own one here.
@@ -279,17 +278,18 @@ class BaseSkyworkR1VProcessor(ABC):
278 self.use_thumbnail: bool = config.use_thumbnail
279
280 @property
282 - @abstractmethod
281 def image_token_id(self) -> int:
284 - raise NotImplementedError
282 + return self.tokenizer.get_vocab()[IMG_CONTEXT]
283
286 - @abstractmethod
284 def get_image_repl(
285 self,
286 feature_size: int,
287 num_patches: Optional[int],
288 ) -> PromptUpdateDetails[str]:
292 - raise NotImplementedError
289 + repl_features = IMG_CONTEXT * feature_size
290 + repl_full = IMG_START + repl_features + IMG_END
291 +
292 + return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
293
294 def resolve_min_max_num(
295 self,
@@ -426,35 +426,15 @@ class BaseSkyworkR1VProcessor(ABC):
426 }
427
428
429 -class SkyworkR1VProcessor(BaseSkyworkR1VProcessor):
430 -
431 - @property
432 - def image_token_id(self) -> int:
433 - return self.tokenizer.get_vocab()[IMG_CONTEXT]
434 -
435 - def get_image_repl(
436 - self,
437 - feature_size: int,
438 - num_patches: Optional[int],
439 - ) -> PromptUpdateDetails[str]:
440 - repl_features = IMG_CONTEXT * feature_size
441 - repl_full = IMG_START + repl_features + IMG_END
442 -
443 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
444 -
429 +class SkyworkR1VProcessingInfo(BaseProcessingInfo):
430
446 -class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
447 -
448 - @abstractmethod
449 - def get_hf_processor(
450 - self,
451 - *,
452 - min_dynamic_patch: Optional[int] = None,
453 - max_dynamic_patch: Optional[int] = None,
454 - dynamic_image_size: Optional[bool] = None,
455 - **kwargs: object,
456 - ) -> BaseSkyworkR1VProcessor:
457 - raise NotImplementedError
431 + def get_hf_processor(self, **kwargs: object) -> SkyworkR1VProcessor:
432 + return self.ctx.init_processor(
433 + SkyworkR1VProcessor,
434 + config=self.get_hf_config(),
435 + tokenizer=self.get_tokenizer(),
436 + **kwargs,
437 + )
438
439 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
440 return {"image": None}
@@ -464,7 +444,7 @@ class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
444 *,
445 image_width: int,
446 image_height: int,
467 - processor: Optional[BaseSkyworkR1VProcessor],
447 + processor: Optional[SkyworkR1VProcessor],
448 ) -> int:
449 if processor is None:
450 processor = self.get_hf_processor()
@@ -500,10 +480,8 @@ class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
480 return largest_feature_pinpoint
481
482
503 -_I = TypeVar("_I", bound=BaseSkyworkR1VProcessingInfo)
504 -
505 -
506 -class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
483 +class SkyworkR1VDummyInputsBuilder(
484 + BaseDummyInputsBuilder[SkyworkR1VProcessingInfo]):
485
486 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
487 num_images = mm_counts.get("image", 0)
@@ -527,7 +505,8 @@ class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
505 }
506
507
530 -class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):
508 +class SkyworkR1VMultiModalProcessor(
509 + BaseMultiModalProcessor[SkyworkR1VProcessingInfo]):
510
511 def _call_hf_processor(
512 self,
@@ -617,31 +596,6 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):
596 ]
597
598
620 -class SkyworkR1VProcessingInfo(BaseSkyworkR1VProcessingInfo):
621 -
622 - def get_hf_processor(
623 - self,
624 - *,
625 - min_dynamic_patch: Optional[int] = None,
626 - max_dynamic_patch: Optional[int] = None,
627 - dynamic_image_size: Optional[bool] = None,
628 - **kwargs: object,
629 - ) -> SkyworkR1VProcessor:
630 - if min_dynamic_patch is not None:
631 - kwargs["min_dynamic_patch"] = min_dynamic_patch
632 - if max_dynamic_patch is not None:
633 - kwargs["max_dynamic_patch"] = max_dynamic_patch
634 - if dynamic_image_size is not None:
635 - kwargs["dynamic_image_size"] = dynamic_image_size
636 -
637 - return self.ctx.init_processor(
638 - SkyworkR1VProcessor,
639 - config=self.get_hf_config(),
640 - tokenizer=self.get_tokenizer(),
641 - **kwargs,
642 - )
643 -
644 -
599 @MULTIMODAL_REGISTRY.register_processor(
600 SkyworkR1VMultiModalProcessor,
601 info=SkyworkR1VProcessingInfo,
vllm/model_executor/models/smolvlm.py
+1 -9
@@ -19,15 +19,7 @@ from .idefics3 import Idefics3ProcessingInfo
19
20 class SmolVLMProcessingInfo(Idefics3ProcessingInfo):
21
22 - def get_hf_processor(
23 - self,
24 - *,
25 - max_image_size: Optional[dict[str, int]] = None,
26 - **kwargs: object,
27 - ) -> SmolVLMProcessor:
28 - if max_image_size is not None:
29 - kwargs["max_image_size"] = max_image_size
30 -
22 + def get_hf_processor(self, **kwargs: object) -> SmolVLMProcessor:
23 return self.ctx.get_hf_processor(SmolVLMProcessor, **kwargs)
24
25 def _get_image_token(
vllm/model_executor/models/tarsier.py
+5 -7
@@ -178,13 +178,11 @@ class TarsierProcessingInfo(BaseProcessingInfo):
178 return get_vision_encoder_info(self.get_hf_config())
179
180 def get_hf_processor(self, **kwargs: object) -> TarsierProcessor:
181 - hf_processor = self.ctx.get_hf_processor(TarsierProcessor, **kwargs)
182 - # Patch for patch_size if needed (copied from vLLM LLaVA)
183 - if hasattr(hf_processor,
184 - 'patch_size') and hf_processor.patch_size is None:
185 - patch_size = self.get_vision_encoder_info().get_patch_size()
186 - hf_processor.patch_size = patch_size
187 - return hf_processor
181 + vision_info = self.get_vision_encoder_info()
182 +
183 + kwargs.setdefault("patch_size", vision_info.get_patch_size())
184 +
185 + return self.ctx.get_hf_processor(TarsierProcessor, **kwargs)
186
187 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
188 return {"image": None}
vllm/model_executor/models/transformers.py
-5
@@ -48,7 +48,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
48 BaseProcessingInfo)
49 from vllm.multimodal.profiling import BaseDummyInputsBuilder
50 from vllm.sequence import IntermediateTensors
51 -from vllm.transformers_utils.processor import cached_get_processor
51 from vllm.utils import is_list_of
52
53 from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,
@@ -189,10 +188,6 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
188 image_tokens = mm_tokens["num_image_tokens"][0]
189 return image_tokens
190
192 - def get_hf_processor(self):
193 - processor = cached_get_processor(self.ctx.model_config.model)
194 - return processor
195 -
191 def get_max_image_size(self):
192 return 10_000, 10_000 # hardcode for arbitrary very large size
193
vllm/model_executor/models/ultravox.py
+5 -15
@@ -71,13 +71,7 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
71
72 class UltravoxProcessingInfo(BaseProcessingInfo):
73
74 - def get_hf_processor(
75 - self,
76 - *,
77 - # Ignored in initialization
78 - sampling_rate: Optional[int] = None,
79 - **kwargs: object,
80 - ) -> ProcessorMixin:
74 + def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
75 config = self.ctx.model_config.hf_config
76 hf_processor = self.ctx.get_hf_processor(**kwargs)
77
@@ -89,13 +83,9 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
83
84 return hf_processor
85
92 - def get_feature_extractor(
93 - self,
94 - *,
95 - # Ignored in initialization
96 - sampling_rate: Optional[int] = None,
97 - ) -> WhisperFeatureExtractor:
98 - hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
86 + def get_feature_extractor(self,
87 + **kwargs: object) -> WhisperFeatureExtractor:
88 + hf_processor = self.get_hf_processor(**kwargs)
89 audio_processor = hf_processor.audio_processor # type: ignore
90 feature_extractor = audio_processor.feature_extractor # type: ignore
91 assert isinstance(feature_extractor, WhisperFeatureExtractor)
@@ -156,7 +146,7 @@ class UltravoxMultiModalProcessor(
146 audios = mm_data.pop("audios", [])
147 assert isinstance(audios, list)
148
159 - feature_extractor = self.info.get_feature_extractor()
149 + feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
150 mm_kwargs = dict(
151 **mm_kwargs,
152 sampling_rate=feature_extractor.sampling_rate,
vllm/model_executor/models/whisper.py
+7 -8
@@ -623,23 +623,22 @@ class WhisperProcessingInfo(BaseProcessingInfo):
623 def get_hf_config(self) -> WhisperConfig:
624 return self.ctx.get_hf_config(WhisperConfig)
625
626 - def get_hf_processor(self,
627 - sampling_rate: Optional[int] = None
628 - ) -> WhisperProcessor:
629 - # HACK: Transformers 4.53.0 has issue with whisper tokenizer to
626 + def get_hf_processor(self, **kwargs: object) -> WhisperProcessor:
627 + # HACK: Transformers 4.53.2 has issue with whisper tokenizer to
628 # initialize processor. We use a monkeypatch to fix it here.
629 # See: https://github.com/vllm-project/vllm/issues/20224
630 processor_class = WhisperProcessor
631 tokenizer_class = ("WhisperTokenizer", "WhisperTokenizerFast")
632 if processor_class.tokenizer_class != tokenizer_class:
633 processor_class.tokenizer_class = tokenizer_class
636 - return self.ctx.get_hf_processor(processor_class)
634 + return self.ctx.get_hf_processor(processor_class, **kwargs)
635
636 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
637 return {"audio": 1}
638
641 - def get_feature_extractor(self) -> WhisperFeatureExtractor:
642 - hf_processor = self.get_hf_processor()
639 + def get_feature_extractor(self,
640 + **kwargs: object) -> WhisperFeatureExtractor:
641 + hf_processor = self.get_hf_processor(**kwargs)
642 feature_extractor = hf_processor.feature_extractor # type: ignore
643 assert isinstance(feature_extractor, WhisperFeatureExtractor)
644 return feature_extractor
@@ -702,7 +701,7 @@ class WhisperMultiModalProcessor(
701 tok_kwargs: Mapping[str, object],
702 ) -> BatchFeature:
703 if mm_data:
705 - feature_extractor = self.info.get_feature_extractor()
704 + feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
705 mm_data = dict(audio=mm_data.pop("audios"))
706 mm_kwargs = dict(
707 **mm_kwargs,
vllm/transformers_utils/processor.py
+59 -35
@@ -4,9 +4,15 @@
4 from functools import lru_cache
5 from typing import TYPE_CHECKING, Any, Optional, Union, cast
6
7 +from transformers import (AutoFeatureExtractor, AutoImageProcessor,
8 + AutoProcessor)
9 +from transformers.feature_extraction_utils import FeatureExtractionMixin
10 +from transformers.image_processing_utils import BaseImageProcessor
11 from transformers.processing_utils import ProcessorMixin
12 from typing_extensions import TypeVar
13
14 +from vllm.utils import get_allowed_kwarg_only_overrides
15 +
16 if TYPE_CHECKING:
17 from vllm.config import ModelConfig
18
@@ -33,23 +39,42 @@ class HashableList(list):
39 return hash(tuple(self))
40
41
36 -def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):
37 - mm_config = model_config.get_multimodal_config()
38 - base_kwargs = mm_config.mm_processor_kwargs
39 - if base_kwargs is None:
40 - base_kwargs = {}
42 +def _get_processor_factory_fn(processor_cls: Union[type, tuple[type, ...]]):
43 + if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:
44 + return AutoProcessor.from_pretrained
45 + if hasattr(processor_cls, "from_pretrained"):
46 + return processor_cls.from_pretrained
47 +
48 + return processor_cls
49 +
50
42 - merged_kwargs = {**base_kwargs, **kwargs}
51 +def _merge_mm_kwargs(
52 + model_config: "ModelConfig",
53 + processor_cls: Union[type, tuple[type, ...]],
54 + /,
55 + **kwargs,
56 +):
57 + mm_config = model_config.get_multimodal_config()
58 + merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
59 +
60 + factory = _get_processor_factory_fn(processor_cls)
61 + allowed_kwargs = get_allowed_kwarg_only_overrides(
62 + factory,
63 + merged_kwargs,
64 + requires_kw_only=False,
65 + allow_var_kwargs=True,
66 + )
67
68 # NOTE: Pythonic dict is not hashable and will raise unhashable type
69 # error when calling `cached_get_processor`, therefore we need to
70 # wrap it to a hashable dict.
47 - for key, value in merged_kwargs.items():
71 + for key, value in allowed_kwargs.items():
72 if isinstance(value, dict):
49 - merged_kwargs[key] = HashableDict(value)
73 + allowed_kwargs[key] = HashableDict(value)
74 if isinstance(value, list):
51 - merged_kwargs[key] = HashableList(value)
52 - return merged_kwargs
75 + allowed_kwargs[key] = HashableList(value)
76 +
77 + return allowed_kwargs
78
79
80 def get_processor(
@@ -61,21 +86,29 @@ def get_processor(
86 **kwargs: Any,
87 ) -> _P:
88 """Load a processor for the given model name via HuggingFace."""
64 - # don't put this import at the top level
65 - # it will call torch.cuda.device_count()
66 - from transformers import AutoProcessor
67 -
68 - processor_factory = (AutoProcessor if processor_cls == ProcessorMixin or
69 - isinstance(processor_cls, tuple) else processor_cls)
89 + if revision is None:
90 + revision = "main"
91
92 try:
72 - processor = processor_factory.from_pretrained(
73 - processor_name,
74 - *args,
75 - revision=revision,
76 - trust_remote_code=trust_remote_code,
77 - **kwargs,
78 - )
93 + if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:
94 + processor = AutoProcessor.from_pretrained(
95 + processor_name,
96 + *args,
97 + revision=revision,
98 + trust_remote_code=trust_remote_code,
99 + **kwargs,
100 + )
101 + elif issubclass(processor_cls, ProcessorMixin):
102 + processor = processor_cls.from_pretrained(
103 + processor_name,
104 + *args,
105 + revision=revision,
106 + trust_remote_code=trust_remote_code,
107 + **kwargs,
108 + )
109 + else:
110 + # Processors that are standalone classes unrelated to HF
111 + processor = processor_cls(*args, **kwargs)
112 except ValueError as e:
113 # If the error pertains to the processor class not existing or not
114 # currently being imported, suggest using the --trust-remote-code flag.
@@ -112,7 +145,7 @@ def cached_processor_from_config(
145 revision=model_config.revision,
146 trust_remote_code=model_config.trust_remote_code,
147 processor_cls=processor_cls, # type: ignore[arg-type]
115 - **_merge_mm_kwargs(model_config, **kwargs),
148 + **_merge_mm_kwargs(model_config, processor_cls, **kwargs),
149 )
150
151
@@ -125,10 +158,6 @@ def get_feature_extractor(
158 ):
159 """Load an audio feature extractor for the given model name
160 via HuggingFace."""
128 - # don't put this import at the top level
129 - # it will call torch.cuda.device_count()
130 - from transformers import AutoFeatureExtractor
131 - from transformers.feature_extraction_utils import FeatureExtractionMixin
161 try:
162 feature_extractor = AutoFeatureExtractor.from_pretrained(
163 processor_name,
@@ -164,7 +193,7 @@ def cached_feature_extractor_from_config(
193 model_config.model,
194 revision=model_config.revision,
195 trust_remote_code=model_config.trust_remote_code,
167 - **_merge_mm_kwargs(model_config, **kwargs),
196 + **_merge_mm_kwargs(model_config, AutoFeatureExtractor, **kwargs),
197 )
198
199
@@ -176,11 +205,6 @@ def get_image_processor(
205 **kwargs: Any,
206 ):
207 """Load an image processor for the given model name via HuggingFace."""
179 - # don't put this import at the top level
180 - # it will call torch.cuda.device_count()
181 - from transformers import AutoImageProcessor
182 - from transformers.image_processing_utils import BaseImageProcessor
183 -
208 try:
209 processor = AutoImageProcessor.from_pretrained(
210 processor_name,
@@ -217,5 +241,5 @@ def cached_image_processor_from_config(
241 model_config.model,
242 revision=model_config.revision,
243 trust_remote_code=model_config.trust_remote_code,
220 - **_merge_mm_kwargs(model_config, **kwargs),
244 + **_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
245 )
vllm/utils/__init__.py
-43
@@ -2010,49 +2010,6 @@ def supports_kw(
2010 return False
2011
2012
2013 -def resolve_mm_processor_kwargs(
2014 - init_kwargs: Optional[Mapping[str, object]],
2015 - inference_kwargs: Optional[Mapping[str, object]],
2016 - callable: Callable[..., object],
2017 - *,
2018 - requires_kw_only: bool = True,
2019 - allow_var_kwargs: bool = False,
2020 -) -> dict[str, Any]:
2021 - """Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,
2022 - those who are not explicit keywords to the given callable (of one is
2023 - given; otherwise no filtering is done), then merges the kwarg dicts,
2024 - giving priority to inference_kwargs if there are any collisions.
2025 -
2026 - In the case that no kwarg overrides are provided, returns an empty
2027 - dict so that it can still be kwarg expanded into the callable later on.
2028 -
2029 - If allow_var_kwargs=True, allows for things that can be expanded into
2030 - kwargs as long as they aren't naming collision for var_kwargs or potential
2031 - positional arguments.
2032 - """
2033 - # Filter inference time multimodal processor kwargs provided
2034 - runtime_mm_kwargs = get_allowed_kwarg_only_overrides(
2035 - callable,
2036 - overrides=inference_kwargs,
2037 - requires_kw_only=requires_kw_only,
2038 - allow_var_kwargs=allow_var_kwargs,
2039 - )
2040 -
2041 - # Filter init time multimodal processor kwargs provided
2042 - init_mm_kwargs = get_allowed_kwarg_only_overrides(
2043 - callable,
2044 - overrides=init_kwargs,
2045 - requires_kw_only=requires_kw_only,
2046 - allow_var_kwargs=allow_var_kwargs,
2047 - )
2048 -
2049 - # Merge the final processor kwargs, prioritizing inference
2050 - # time values over the initialization time values.
2051 - mm_processor_kwargs = {**init_mm_kwargs, **runtime_mm_kwargs}
2052 -
2053 - return mm_processor_kwargs
2054 -
2055 -
2013 def get_allowed_kwarg_only_overrides(
2014 callable: Callable[..., object],
2015 overrides: Optional[Mapping[str, object]],