A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc]] Move processing context to multimodal directory (#25548)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 24, 2025 at 16:15 UTC
6488f3481be77bbce66a9ebfcab3f6bc5b961838
13 files changed
+262
-242
tests/models/multimodal/processing/test_common.py
+2
-2
@@ -12,11 +12,11 @@ from mistral_common.protocol.instruct.request import ChatCompletionRequest
12
from PIL import Image
13
14
from vllm.config import ModelConfig
15
-from vllm.inputs import InputProcessingContext
15
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
16
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
17
from vllm.multimodal.inputs import MultiModalInputs
19
-from vllm.multimodal.processing import BaseMultiModalProcessor
18
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
19
+ InputProcessingContext)
20
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
21
cached_tokenizer_from_config,
22
encode_tokens)
tests/models/multimodal/processing/test_tensor_schema.py
+2
-2
@@ -18,10 +18,10 @@ from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
18
from vllm.distributed import (cleanup_dist_env_and_memory,
19
init_distributed_environment,
20
initialize_model_parallel)
21
-from vllm.inputs import InputProcessingContext
21
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
22
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
24
-from vllm.multimodal.processing import BaseMultiModalProcessor
23
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
24
+ InputProcessingContext)
25
from vllm.multimodal.utils import group_mm_kwargs_by_modality
26
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
27
from vllm.utils import is_list_of
tests/models/utils.py
+9
-4
@@ -11,8 +11,9 @@ import torch.nn.functional as F
11
from transformers import PretrainedConfig
12
13
from vllm.config import ModelConfig, ModelDType, RunnerOption
14
-from vllm.inputs import InputContext
14
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
15
+from vllm.multimodal.processing import InputProcessingContext
16
+from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
17
18
from .registry import HF_EXAMPLE_MODELS
19
@@ -264,7 +265,7 @@ def build_model_context(
265
limit_mm_per_prompt: Optional[dict[str, int]] = None,
266
mm_processor_cache_gb: int = 0,
267
):
267
- """Creates an InputContext for a given model.
268
+ """Creates an InputProcessingContext for a given model.
269
270
Args:
271
model_id: ID of the model being considered.
@@ -273,7 +274,7 @@ def build_model_context(
274
limit_mm_per_prompt: Multimodal limits.
275
276
Returns:
276
- InputContext for the model being considered.
277
+ InputProcessingContext for the model being considered.
278
"""
279
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
280
model_info.check_available_online(on_fail="skip")
@@ -298,7 +299,11 @@ def build_model_context(
299
enforce_eager=model_info.enforce_eager,
300
**model_config_kwargs,
301
)
301
- return InputContext(model_config)
302
+
303
+ return InputProcessingContext(
304
+ model_config,
305
+ tokenizer=cached_tokenizer_from_config(model_config),
306
+ )
307
308
309
def check_embeddings_close(
tests/multimodal/test_processing.py
+2
-2
@@ -8,11 +8,11 @@ import numpy as np
8
import pytest
9
10
from vllm.config import ModelConfig
11
-from vllm.inputs import InputProcessingContext
11
from vllm.multimodal import MULTIMODAL_REGISTRY
12
# yapf conflicts with isort for this block
13
# yapf: disable
15
-from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
14
+from vllm.multimodal.processing import (InputProcessingContext,
15
+ PlaceholderFeaturesInfo,
16
PromptIndexTargets, PromptInsertion,
17
PromptReplacement, apply_text_matches,
18
apply_token_matches,
vllm/inputs/__init__.py
-3
@@ -7,7 +7,6 @@ from .data import (DataPrompt, DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,
7
SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
8
build_explicit_enc_dec_prompt, embeds_inputs,
9
to_enc_dec_tuple_list, token_inputs, zip_enc_dec_prompts)
10
-from .registry import InputContext, InputProcessingContext
10
11
__all__ = [
12
"DataPrompt",
@@ -28,6 +27,4 @@ __all__ = [
27
"build_explicit_enc_dec_prompt",
28
"to_enc_dec_tuple_list",
29
"zip_enc_dec_prompts",
31
- "InputContext",
32
- "InputProcessingContext",
30
]
vllm/inputs/registry.py
deleted
-206
@@ -1,206 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import time
4
-from collections.abc import Mapping
5
-from dataclasses import dataclass
6
-from typing import TYPE_CHECKING, Any, Union
7
-
8
-import torch
9
-from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
10
-from typing_extensions import TypeVar
11
-
12
-from vllm.logger import init_logger
13
-from vllm.transformers_utils.processor import cached_processor_from_config
14
-from vllm.utils import get_allowed_kwarg_only_overrides
15
-from vllm.utils.jsontree import JSONTree, json_map_leaves
16
-
17
-if TYPE_CHECKING:
18
- from vllm.config import ModelConfig
19
- from vllm.transformers_utils.tokenizer import AnyTokenizer
20
-else:
21
- ModelConfig = Any
22
- AnyTokenizer = Any
23
-
24
-_T = TypeVar("_T")
25
-_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
26
-_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
27
-
28
-logger = init_logger(__name__)
29
-
30
-
31
-@dataclass(frozen=True)
32
-class InputContext:
33
- """
34
- Contains information about the model which may be used to
35
- modify the inputs.
36
- """
37
-
38
- model_config: ModelConfig
39
- """The configuration of the model."""
40
-
41
- def get_hf_config(
42
- self,
43
- typ: Union[type[_C], tuple[type[_C], ...]] = PretrainedConfig,
44
- /,
45
- ) -> _C:
46
- """
47
- Get the HuggingFace configuration
48
- (`transformers.PretrainedConfig`) of the model,
49
- additionally checking its type.
50
-
51
- Raises:
52
- TypeError: If the configuration is not of the specified type.
53
- """
54
- hf_config = self.model_config.hf_config
55
- if not isinstance(hf_config, typ):
56
- raise TypeError("Invalid type of HuggingFace config. "
57
- f"Expected type: {typ}, but "
58
- f"found type: {type(hf_config)}")
59
-
60
- return hf_config
61
-
62
- def get_hf_image_processor_config(self) -> dict[str, Any]:
63
- """
64
- Get the HuggingFace image processor configuration of the model.
65
- """
66
- return self.model_config.hf_image_processor_config
67
-
68
- def get_mm_config(self):
69
- """
70
- Get the multimodal config of the model.
71
-
72
- Raises:
73
- RuntimeError: If the model is not a multimodal model.
74
- """
75
- mm_config = self.model_config.multimodal_config
76
- if mm_config is None:
77
- raise RuntimeError("Not a multimodal model")
78
-
79
- return mm_config
80
-
81
- def get_hf_processor(
82
- self,
83
- typ: Union[type[_P], tuple[type[_P], ...]] = ProcessorMixin,
84
- /,
85
- **kwargs: object,
86
- ) -> _P:
87
- """
88
- Get the HuggingFace processor
89
- (`transformers.ProcessorMixin`) of the model,
90
- additionally checking its type.
91
-
92
- Raises:
93
- TypeError: If the processor is not of the specified type.
94
- """
95
- return cached_processor_from_config(
96
- self.model_config,
97
- processor_cls=typ,
98
- **kwargs,
99
- )
100
-
101
- def init_processor(
102
- self,
103
- typ: type[_T],
104
- /,
105
- **kwargs: object,
106
- ) -> _T:
107
- """
108
- Initialize a HuggingFace-like processor class, merging the
109
- keyword arguments with those in the model's configuration.
110
- """
111
- mm_config = self.model_config.get_multimodal_config()
112
- base_kwargs = mm_config.mm_processor_kwargs
113
- if base_kwargs is None:
114
- base_kwargs = {}
115
-
116
- merged_kwargs = {**base_kwargs, **kwargs}
117
-
118
- return typ(**merged_kwargs)
119
-
120
-
121
-@dataclass(frozen=True)
122
-class InputProcessingContext(InputContext):
123
- tokenizer: AnyTokenizer
124
- """The tokenizer used to tokenize the inputs."""
125
-
126
- def get_hf_processor(
127
- self,
128
- typ: Union[type[_P], tuple[type[_P], ...]] = ProcessorMixin,
129
- /,
130
- **kwargs: object,
131
- ) -> _P:
132
- return super().get_hf_processor(
133
- typ,
134
- tokenizer=self.tokenizer,
135
- **kwargs,
136
- )
137
-
138
- def call_hf_processor(
139
- self,
140
- hf_processor: ProcessorMixin,
141
- data: Mapping[str, object],
142
- kwargs: Mapping[str, object] = {},
143
- *,
144
- num_tries: int = 1,
145
- max_tries: int = 5,
146
- ) -> Union[BatchFeature, JSONTree]:
147
- """
148
- Call `hf_processor` on the prompt `data`
149
- (text, image, audio...) with configurable options `kwargs`.
150
- """
151
- assert callable(hf_processor)
152
-
153
- mm_config = self.model_config.get_multimodal_config()
154
- merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
155
-
156
- allowed_kwargs = get_allowed_kwarg_only_overrides(
157
- hf_processor,
158
- merged_kwargs,
159
- requires_kw_only=False,
160
- allow_var_kwargs=True,
161
- )
162
-
163
- def maybe_cast_dtype(x):
164
- # This mimics the behavior of transformers.BatchFeature
165
- if isinstance(x, torch.Tensor) and x.is_floating_point():
166
- return x.to(dtype=self.model_config.dtype)
167
- return x
168
-
169
- try:
170
- output = hf_processor(**data,
171
- **allowed_kwargs,
172
- return_tensors="pt")
173
- # this emulates output.to(dtype=self.model_config.dtype)
174
- if isinstance(output, BatchFeature):
175
- cast_output = json_map_leaves(maybe_cast_dtype, output.data)
176
- return BatchFeature(cast_output)
177
-
178
- cast_output = json_map_leaves(maybe_cast_dtype, output)
179
-
180
- logger.warning_once(
181
- f"{type(hf_processor).__name__} did not return `BatchFeature`. "
182
- "Make sure to match the behaviour of `ProcessorMixin` when "
183
- "implementing custom processors.")
184
- return cast_output
185
-
186
- except Exception as exc:
187
- # See https://github.com/huggingface/tokenizers/issues/537
188
- if (isinstance(exc, RuntimeError) and exc
189
- and exc.args[0] == "Already borrowed"
190
- and num_tries < max_tries):
191
- logger.warning(
192
- "Failed to acquire tokenizer in current thread. "
193
- "Retrying (%d/%d)...", num_tries, max_tries)
194
- time.sleep(0.5)
195
- return self.call_hf_processor(
196
- hf_processor,
197
- data,
198
- kwargs,
199
- num_tries=num_tries + 1,
200
- max_tries=max_tries,
201
- )
202
-
203
- msg = (f"Failed to apply {type(hf_processor).__name__} "
204
- f"on data={data} with kwargs={allowed_kwargs}")
205
-
206
- raise ValueError(msg) from exc
vllm/model_executor/models/hyperclovax_vision.py
+3
-3
@@ -29,7 +29,6 @@ from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig
29
from transformers.modeling_utils import no_init_weights
30
31
from vllm.config import VllmConfig
32
-from vllm.inputs import InputProcessingContext
32
from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.multimodal import MULTIMODAL_REGISTRY
34
from vllm.multimodal.cache import BaseMultiModalProcessorCache
@@ -37,8 +36,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36
MultiModalKwargsItems)
37
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
40
- BaseProcessingInfo, PromptReplacement,
41
- PromptUpdate)
39
+ BaseProcessingInfo,
40
+ InputProcessingContext,
41
+ PromptReplacement, PromptUpdate)
42
from vllm.multimodal.profiling import BaseDummyInputsBuilder
43
from vllm.sequence import IntermediateTensors
44
vllm/model_executor/models/llava.py
+4
-3
@@ -15,7 +15,6 @@ from transformers.models.llava import LlavaProcessor
15
from transformers.models.pixtral import PixtralProcessor
16
17
from vllm.config import VllmConfig
18
-from vllm.inputs import InputProcessingContext
18
from vllm.model_executor.layers.activation import get_act_fn
19
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
RowParallelLinear)
@@ -28,8 +27,10 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28
ImageSize, MultiModalDataItems)
29
from vllm.multimodal.processing import (BaseMultiModalProcessor,
31
- BaseProcessingInfo, PromptReplacement,
32
- PromptUpdate, PromptUpdateDetails)
30
+ BaseProcessingInfo,
31
+ InputProcessingContext,
32
+ PromptReplacement, PromptUpdate,
33
+ PromptUpdateDetails)
34
from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.jsontree import json_map_leaves
vllm/model_executor/models/mistral3.py
+4
-3
@@ -13,7 +13,6 @@ from transformers import (BatchFeature, Mistral3Config, PixtralVisionConfig,
13
from transformers.models.pixtral import PixtralProcessor
14
15
from vllm.config import VllmConfig
16
-from vllm.inputs import InputProcessingContext
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.layernorm import RMSNorm
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -27,8 +26,10 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
27
MultiModalDataItems)
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
30
- BaseProcessingInfo, PromptReplacement,
31
- PromptUpdate, PromptUpdateDetails)
29
+ BaseProcessingInfo,
30
+ InputProcessingContext,
31
+ PromptReplacement, PromptUpdate,
32
+ PromptUpdateDetails)
33
from vllm.multimodal.profiling import BaseDummyInputsBuilder
34
from vllm.sequence import IntermediateTensors
35
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/mllama4.py
+4
-3
@@ -32,7 +32,6 @@ from transformers.models.llama4.image_processing_llama4_fast import (
32
from vllm.attention.layer import MultiHeadAttention
33
from vllm.config import VllmConfig
34
from vllm.distributed import get_tensor_model_parallel_world_size
35
-from vllm.inputs import InputProcessingContext
35
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
36
QKVParallelLinear,
37
ReplicatedLinear,
@@ -47,8 +46,10 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
46
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47
MultiModalDataItems)
48
from vllm.multimodal.processing import (BaseMultiModalProcessor,
50
- BaseProcessingInfo, PromptReplacement,
51
- PromptUpdate, PromptUpdateDetails)
49
+ BaseProcessingInfo,
50
+ InputProcessingContext,
51
+ PromptReplacement, PromptUpdate,
52
+ PromptUpdateDetails)
53
from vllm.multimodal.profiling import BaseDummyInputsBuilder
54
from vllm.sequence import IntermediateTensors
55
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/tarsier.py
+3
-3
@@ -17,7 +17,6 @@ from transformers.processing_utils import ProcessingKwargs, Unpack
17
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
18
19
from vllm.config import VllmConfig
20
-from vllm.inputs import InputProcessingContext
20
from vllm.model_executor.layers.activation import get_act_fn
21
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22
RowParallelLinear)
@@ -29,8 +28,9 @@ from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
28
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
29
ImageSize, MultiModalDataItems)
30
from vllm.multimodal.processing import (BaseMultiModalProcessor,
32
- BaseProcessingInfo, PromptReplacement,
33
- PromptUpdate)
31
+ BaseProcessingInfo,
32
+ InputProcessingContext,
33
+ PromptReplacement, PromptUpdate)
34
from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.jsontree import json_map_leaves
vllm/multimodal/processing.py
+226
-5
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import time
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
@@ -7,18 +8,20 @@ from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
8
from dataclasses import dataclass, field, replace
9
from enum import Enum
10
from functools import lru_cache
10
-from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
11
- TypeVar, Union, cast)
11
+from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional,
12
+ Protocol, Union, cast, overload)
13
14
import regex as re
15
import torch
15
-from typing_extensions import assert_never
16
+from typing_extensions import TypeVar, assert_never
17
17
-from vllm.inputs import InputProcessingContext
18
from vllm.logger import init_logger
19
+from vllm.transformers_utils.processor import cached_processor_from_config
20
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
21
encode_tokens)
21
-from vllm.utils import flatten_2d_lists, full_groupby
22
+from vllm.utils import (flatten_2d_lists, full_groupby,
23
+ get_allowed_kwarg_only_overrides)
24
+from vllm.utils.jsontree import JSONTree, json_map_leaves
25
26
from .hasher import MultiModalHasher
27
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
@@ -34,6 +37,8 @@ if TYPE_CHECKING:
37
from transformers.feature_extraction_utils import BatchFeature
38
from transformers.processing_utils import ProcessorMixin
39
40
+ from vllm.config import ModelConfig
41
+
42
from .cache import BaseMultiModalProcessorCache
43
from .profiling import BaseDummyInputsBuilder
44
@@ -875,6 +880,222 @@ def find_mm_placeholders(
880
return dict(full_groupby_modality(it))
881
882
883
+_T = TypeVar("_T")
884
+_C = TypeVar("_C", bound="PretrainedConfig", default="PretrainedConfig")
885
+_P = TypeVar("_P", bound="ProcessorMixin", default="ProcessorMixin")
886
+
887
+
888
+@dataclass(frozen=True)
889
+class InputProcessingContext:
890
+ """
891
+ Contains information about the model which may be used to
892
+ modify the inputs.
893
+ """
894
+
895
+ model_config: "ModelConfig"
896
+ """The configuration of the model."""
897
+
898
+ tokenizer: AnyTokenizer
899
+ """The tokenizer used to tokenize the inputs."""
900
+
901
+ @overload
902
+ def get_hf_config(self, /) -> "PretrainedConfig":
903
+ ...
904
+
905
+ @overload
906
+ def get_hf_config(
907
+ self,
908
+ typ: Union[type[_C], tuple[type[_C], ...]],
909
+ /,
910
+ ) -> _C:
911
+ ...
912
+
913
+ def get_hf_config(
914
+ self,
915
+ typ: Optional[Union[type[Any], tuple[type[Any], ...]]] = None,
916
+ /,
917
+ ) -> Any:
918
+ """
919
+ Get the HuggingFace configuration
920
+ (`transformers.PretrainedConfig`) of the model,
921
+ additionally checking its type.
922
+
923
+ Raises:
924
+ TypeError: If the configuration is not of the specified type.
925
+ """
926
+ if typ is None:
927
+ from transformers.configuration_utils import PretrainedConfig
928
+
929
+ typ = PretrainedConfig
930
+
931
+ hf_config = self.model_config.hf_config
932
+ if not isinstance(hf_config, typ):
933
+ raise TypeError("Invalid type of HuggingFace config. "
934
+ f"Expected type: {typ}, but "
935
+ f"found type: {type(hf_config)}")
936
+
937
+ return hf_config
938
+
939
+ def get_hf_image_processor_config(self) -> dict[str, Any]:
940
+ """
941
+ Get the HuggingFace image processor configuration of the model.
942
+ """
943
+ return self.model_config.hf_image_processor_config
944
+
945
+ def get_mm_config(self):
946
+ """
947
+ Get the multimodal config of the model.
948
+
949
+ Raises:
950
+ RuntimeError: If the model is not a multimodal model.
951
+ """
952
+ mm_config = self.model_config.multimodal_config
953
+ if mm_config is None:
954
+ raise RuntimeError("Not a multimodal model")
955
+
956
+ return mm_config
957
+
958
+ @overload
959
+ def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin":
960
+ ...
961
+
962
+ @overload
963
+ def get_hf_processor(
964
+ self,
965
+ typ: Union[type[_P], tuple[type[_P], ...]],
966
+ /,
967
+ **kwargs: object,
968
+ ) -> _P:
969
+ ...
970
+
971
+ def get_hf_processor(
972
+ self,
973
+ typ: Optional[Union[type[Any], tuple[type[Any], ...]]] = None,
974
+ /,
975
+ **kwargs: object,
976
+ ) -> Any:
977
+ """
978
+ Get the HuggingFace processor
979
+ (`transformers.ProcessorMixin`) of the model,
980
+ additionally checking its type.
981
+
982
+ Raises:
983
+ TypeError: If the processor is not of the specified type.
984
+ """
985
+ if typ is None:
986
+ from transformers.processing_utils import ProcessorMixin
987
+
988
+ typ = ProcessorMixin
989
+
990
+ return cached_processor_from_config(
991
+ self.model_config,
992
+ processor_cls=typ,
993
+ tokenizer=self.tokenizer,
994
+ **kwargs,
995
+ )
996
+
997
+ def init_processor(
998
+ self,
999
+ typ: type[_T],
1000
+ /,
1001
+ **kwargs: object,
1002
+ ) -> _T:
1003
+ """
1004
+ Initialize a HuggingFace-like processor class, merging the
1005
+ keyword arguments with those in the model's configuration.
1006
+ """
1007
+ mm_config = self.model_config.get_multimodal_config()
1008
+ base_kwargs = mm_config.mm_processor_kwargs
1009
+ if base_kwargs is None:
1010
+ base_kwargs = {}
1011
+
1012
+ merged_kwargs = {**base_kwargs, **kwargs}
1013
+
1014
+ return typ(**merged_kwargs)
1015
+
1016
+ def _postprocess_output(
1017
+ self,
1018
+ output: JSONTree,
1019
+ ) -> JSONTree:
1020
+
1021
+ def _postprocess_one(x: object):
1022
+ if isinstance(x, torch.Tensor): # noqa: SIM102
1023
+ # This mimics the behavior of transformers.BatchFeature
1024
+ if x.is_floating_point():
1025
+ x = x.to(dtype=self.model_config.dtype)
1026
+
1027
+ return x
1028
+
1029
+ return json_map_leaves(_postprocess_one, output)
1030
+
1031
+ def call_hf_processor(
1032
+ self,
1033
+ hf_processor: "ProcessorMixin",
1034
+ data: Mapping[str, object],
1035
+ kwargs: Mapping[str, object] = {},
1036
+ *,
1037
+ num_tries: int = 1,
1038
+ max_tries: int = 5,
1039
+ ) -> Union["BatchFeature", JSONTree]:
1040
+ """
1041
+ Call `hf_processor` on the prompt `data`
1042
+ (text, image, audio...) with configurable options `kwargs`.
1043
+ """
1044
+ assert callable(hf_processor)
1045
+
1046
+ mm_config = self.model_config.get_multimodal_config()
1047
+ merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
1048
+
1049
+ allowed_kwargs = get_allowed_kwarg_only_overrides(
1050
+ hf_processor,
1051
+ merged_kwargs,
1052
+ requires_kw_only=False,
1053
+ allow_var_kwargs=True,
1054
+ )
1055
+
1056
+ try:
1057
+ output = hf_processor(**data,
1058
+ **allowed_kwargs,
1059
+ return_tensors="pt")
1060
+ except Exception as exc:
1061
+ # See https://github.com/huggingface/tokenizers/issues/537
1062
+ if (isinstance(exc, RuntimeError) and exc
1063
+ and exc.args[0] == "Already borrowed"
1064
+ and num_tries < max_tries):
1065
+ logger.warning(
1066
+ "Failed to acquire tokenizer in current thread. "
1067
+ "Retrying (%d/%d)...", num_tries, max_tries)
1068
+ time.sleep(0.5)
1069
+ return self.call_hf_processor(
1070
+ hf_processor,
1071
+ data,
1072
+ kwargs,
1073
+ num_tries=num_tries + 1,
1074
+ max_tries=max_tries,
1075
+ )
1076
+
1077
+ msg = (f"Failed to apply {type(hf_processor).__name__} "
1078
+ f"on data={data} with kwargs={allowed_kwargs}")
1079
+
1080
+ raise ValueError(msg) from exc
1081
+
1082
+ # this emulates output.to(dtype=self.model_config.dtype)
1083
+ from transformers.feature_extraction_utils import BatchFeature
1084
+
1085
+ if isinstance(output, BatchFeature):
1086
+ output_ = self._postprocess_output(output.data)
1087
+ return BatchFeature(output_)
1088
+
1089
+ logger.warning_once(
1090
+ "%s did not return `BatchFeature`. "
1091
+ "Make sure to match the behaviour of `ProcessorMixin` when "
1092
+ "implementing custom processors.",
1093
+ type(hf_processor).__name__,
1094
+ )
1095
+
1096
+ return self._postprocess_output(output)
1097
+
1098
+
1099
class BaseProcessingInfo:
1100
"""Base class to provide the information necessary for data processing."""
1101
vllm/multimodal/registry.py
+3
-3
@@ -6,14 +6,14 @@ from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar
6
7
import torch.nn as nn
8
9
-from vllm.inputs import InputProcessingContext
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
11
cached_tokenizer_from_config)
12
from vllm.utils import ClassRegistry
13
14
from .cache import BaseMultiModalProcessorCache
16
-from .processing import BaseMultiModalProcessor, BaseProcessingInfo
15
+from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
16
+ InputProcessingContext)
17
from .profiling import (BaseDummyInputsBuilder, DummyDecoderData,
18
DummyEncoderData, MultiModalProfiler)
19
@@ -41,7 +41,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
41
...
42
43
44
-class DummyInputsBuilderFactory(Protocol[_I]):
44
+class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
45
"""
46
Constructs a
47
[`BaseDummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]