A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[VLM] Calculate maximum number of multi-modal tokens by model (#6121)
Cyrus Leung committed
Jul 5, 2024 at 07:37 UTC
ae96ef8fbd6fe8905f8ea0d3a3f9ff1738dbcbe5
12 files changed
+260
-90
docs/source/dev/multimodal/adding_multimodal_model.rst
+48
-20
@@ -51,17 +51,16 @@ As usual, follow :ref:`these steps <adding_a_new_model>` to implement the model
51
2. Register input mappers
52
-------------------------
53
54
-For each modality type to support, decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
54
+For each modality type that the model accepts as input, decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
55
This decorator accepts a function that maps multi-modal inputs to the keyword arguments you have previously defined in :meth:`~torch.nn.Module.forward`.
56
57
.. code-block:: diff
58
59
- from vllm.model_executor.models.interfaces import SupportsVision
59
+ from vllm.model_executor.models.interfaces import SupportsVision
60
+ from vllm.multimodal import MULTIMODAL_REGISTRY
61
62
- + @MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
63
- + @MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
64
- class YourModelForImage2Seq(nn.Module, SupportsVision):
62
+ + @MULTIMODAL_REGISTRY.register_image_input_mapper()
63
+ class YourModelForImage2Seq(nn.Module, SupportsVision):
64
65
A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
66
@@ -69,7 +68,33 @@ A default mapper is available for each modality in the core vLLM library. This i
68
:ref:`input_processing_pipeline`
69
70
72
-3. (Optional) Register dummy data
71
+3. Register maximum number of multimodal tokens
72
+----------------------------------------------------------
73
+
74
+For each modality type that the model accepts as input, calculate the maximum possible number of tokens
75
+and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
76
+
77
+.. code-block:: diff
78
+
79
+ from vllm.inputs import INPUT_REGISTRY
80
+ from vllm.model_executor.models.interfaces import SupportsVision
81
+ from vllm.multimodal import MULTIMODAL_REGISTRY
82
+
83
+ @MULTIMODAL_REGISTRY.register_image_input_mapper()
84
+ + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
85
+ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
86
+ class YourModelForImage2Seq(nn.Module, SupportsVision):
87
+
88
+Here are some examples:
89
+
90
+- Image inputs (static feature size): `LLaVA-1.5 Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py>`__
91
+- Image inputs (dynamic feature size): `LLaVA-NeXT Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py>`__
92
+
93
+.. seealso::
94
+ :ref:`input_processing_pipeline`
95
+
96
+
97
+4. (Optional) Register dummy data
98
---------------------------------
99
100
During startup, dummy data is passed to the vLLM model to allocate memory. This only consists of text input by default, which may not be applicable to multi-modal models.
@@ -77,14 +102,17 @@ In such cases, you can define your own dummy data by registering a factory metho
102
103
.. code-block:: diff
104
80
- from vllm.inputs import INPUT_REGISTRY
81
- from vllm.model_executor.models.interfaces import SupportsVision
82
- from vllm.multimodal import MULTIMODAL_REGISTRY
105
+ from vllm.inputs import INPUT_REGISTRY
106
+ from vllm.model_executor.models.interfaces import SupportsVision
107
+ from vllm.multimodal import MULTIMODAL_REGISTRY
108
84
- @MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
85
- @MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
109
+ @MULTIMODAL_REGISTRY.register_image_input_mapper()
110
+ @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
111
+ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
87
- class YourModelForImage2Seq(nn.Module, SupportsVision):
112
+ class YourModelForImage2Seq(nn.Module, SupportsVision):
113
+
114
+.. note::
115
+ The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
116
117
Here are some examples:
118
@@ -95,7 +123,7 @@ Here are some examples:
123
:ref:`input_processing_pipeline`
124
125
98
-4. (Optional) Register input processor
126
+5. (Optional) Register input processor
127
--------------------------------------
128
129
Sometimes, there is a need to process inputs at the :class:`~vllm.LLMEngine` level before they are passed to the model executor.
@@ -104,15 +132,15 @@ You can register input processors via :meth:`INPUT_REGISTRY.register_input_proce
132
133
.. code-block:: diff
134
107
- from vllm.inputs import INPUT_REGISTRY
108
- from vllm.model_executor.models.interfaces import SupportsVision
109
- from vllm.multimodal import MULTIMODAL_REGISTRY
135
+ from vllm.inputs import INPUT_REGISTRY
136
+ from vllm.model_executor.models.interfaces import SupportsVision
137
+ from vllm.multimodal import MULTIMODAL_REGISTRY
138
111
- @MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
112
- @MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
113
- @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
139
+ @MULTIMODAL_REGISTRY.register_image_input_mapper()
140
+ @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
141
+ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
142
+ @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
115
- class YourModelForImage2Seq(nn.Module, SupportsVision):
143
+ class YourModelForImage2Seq(nn.Module, SupportsVision):
144
145
A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
146
Here are some examples:
docs/source/models/vlm.rst
+4
-14
@@ -25,13 +25,8 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
25
26
.. important::
27
We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28
- the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
29
- every model to perform profiling with.
30
-
31
- This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
32
- :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
33
- for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
34
- with a more accurate profiling strategy in the future.
28
+ the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified as we now calculate that
29
+ internally for each model.
30
31
32
To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
@@ -104,13 +99,8 @@ Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with
99
100
.. important::
101
We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
107
- the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
108
- every model to perform profiling with.
109
-
110
- This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
111
- :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
112
- for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
113
- with a more accurate profiling strategy in the future.
102
+ the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified as we now calculate that
103
+ internally for each model.
104
105
To consume the server, you can use the OpenAI client like in the example below:
106
vllm/inputs/registry.py
+1
-1
@@ -51,7 +51,7 @@ class InputContext:
51
additionally checking its type.
52
53
Raises:
54
- ValueError: If the model is not of the specified type.
54
+ TypeError: If the model is not of the specified type.
55
"""
56
57
hf_config = self.model_config.hf_config
vllm/model_executor/models/clip.py
+4
@@ -35,6 +35,10 @@ def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
35
patch_size=hf_config.patch_size)
36
37
38
+def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
39
+ return get_clip_image_feature_size(hf_config)
40
+
41
+
42
def dummy_seq_data_for_clip(
43
hf_config: CLIPVisionConfig,
44
seq_len: int,
vllm/model_executor/models/llava.py
+13
-1
@@ -21,7 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.sequence import IntermediateTensors, SamplerOutput
22
23
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
24
- input_processor_for_clip)
24
+ get_max_clip_image_tokens, input_processor_for_clip)
25
from .interfaces import SupportsVision
26
from .utils import merge_vision_embeddings
27
@@ -62,6 +62,17 @@ class LlavaImagePixelInputs(TypedDict):
62
LlavaImageInputs = LlavaImagePixelInputs
63
64
65
+def get_max_llava_image_tokens(ctx: InputContext):
66
+ hf_config = ctx.get_hf_config(LlavaConfig)
67
+ vision_config = hf_config.vision_config
68
+
69
+ if isinstance(vision_config, CLIPVisionConfig):
70
+ return get_max_clip_image_tokens(vision_config)
71
+
72
+ msg = f"Unsupported vision config: {type(vision_config)}"
73
+ raise NotImplementedError(msg)
74
+
75
+
76
def dummy_data_for_llava(ctx: InputContext, seq_len: int):
77
hf_config = ctx.get_hf_config(LlavaConfig)
78
vision_config = hf_config.vision_config
@@ -102,6 +113,7 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
113
114
115
@MULTIMODAL_REGISTRY.register_image_input_mapper()
116
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
117
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
118
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
119
class LlavaForConditionalGeneration(nn.Module, SupportsVision):
vllm/model_executor/models/llava_next.py
+12
@@ -127,6 +127,17 @@ def get_llava_next_image_feature_size(
127
raise NotImplementedError(msg)
128
129
130
+def get_max_llava_next_image_tokens(ctx: InputContext):
131
+ # Result in the max possible feature size (2x2 grid of 336x336px tiles)
132
+ dummy_height = dummy_width = 448
133
+
134
+ return get_llava_next_image_feature_size(
135
+ ctx.get_hf_config(LlavaNextConfig),
136
+ input_height=dummy_height,
137
+ input_width=dummy_width,
138
+ )
139
+
140
+
141
def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
142
hf_config = ctx.get_hf_config(LlavaNextConfig)
143
vision_config = hf_config.vision_config
@@ -198,6 +209,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
209
210
211
@MULTIMODAL_REGISTRY.register_image_input_mapper()
212
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
213
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
214
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next)
215
class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
vllm/model_executor/models/phi3v.py
+12
@@ -321,6 +321,17 @@ def get_phi3v_image_feature_size(
321
+ (new_height // 336 + 1) * 12
322
323
324
+def get_max_phi3v_image_tokens(ctx: InputContext):
325
+ # Result in the max possible feature size (h:w = 16:1)
326
+ dummy_height, dummy_width = 8000, 50
327
+
328
+ return get_phi3v_image_feature_size(
329
+ ctx.get_hf_config(PretrainedConfig),
330
+ input_height=dummy_height,
331
+ input_width=dummy_width,
332
+ )
333
+
334
+
335
def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
336
# Result in the max possible feature size (h:w = 16:1)
337
dummy_height, dummy_width = 8000, 50
@@ -429,6 +440,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
440
441
442
@MULTIMODAL_REGISTRY.register_image_input_mapper()
443
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
444
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
445
@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
446
class Phi3VForCausalLM(nn.Module, SupportsVision):
vllm/multimodal/base.py
+99
-6
@@ -97,9 +97,19 @@ the corresponding plugin with the same modality key is applied.
97
"""
98
99
MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
100
-"""Return a dictionary to be passed as keyword arguments to
100
+"""
101
+Return a dictionary to be passed as keyword arguments to
102
:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
102
-and processors in HuggingFace Transformers."""
103
+and processors in HuggingFace Transformers.
104
+
105
+If the data is not supported, throw :exc:`TypeError`.
106
+"""
107
+
108
+MultiModalTokensCalc = Union[int, Callable[[InputContext], int]]
109
+"""
110
+Calculate the maximum number of multimodal tokens input to the language
111
+model. This does not include tokens that correspond to the input text.
112
+"""
113
114
N = TypeVar("N", bound=Type[nn.Module])
115
@@ -117,6 +127,7 @@ class MultiModalPlugin(ABC):
127
128
def __init__(self) -> None:
129
self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {}
130
+ self._max_mm_tokens: Dict[Type[nn.Module], MultiModalTokensCalc] = {}
131
132
@abstractmethod
133
def get_data_key(self) -> str:
@@ -128,9 +139,12 @@ class MultiModalPlugin(ABC):
139
@abstractmethod
140
def _default_input_mapper(self, ctx: InputContext,
141
data: object) -> MultiModalInputs:
131
- """Return a dictionary to be passed as keyword arguments to
142
+ """
143
+ Return a dictionary to be passed as keyword arguments to
144
:meth:`~torch.nn.Module.forward`. This is similar in concept to
145
tokenizers and processors in HuggingFace Transformers.
146
+
147
+ If the data is not supported, throw :exc:`TypeError`.
148
"""
149
raise NotImplementedError
150
@@ -140,9 +154,11 @@ class MultiModalPlugin(ABC):
154
):
155
"""
156
Register an input mapper to a model class.
157
+
158
When the model receives input data that matches the modality served by
144
- this plugin (see :meth:`get_data_type`), the provided function is
159
+ this plugin (see :meth:`get_data_key`), the provided function is
160
invoked to transform the data into a dictionary of model inputs.
161
+
162
If `None` is provided, then the default input mapper is used instead.
163
164
See also:
@@ -170,10 +186,11 @@ class MultiModalPlugin(ABC):
186
Apply an input mapper to a data passed
187
to the model, transforming the data into a dictionary of model inputs.
188
173
- If the data is not something that the mapper expects, throws TypeError.
174
-
189
The model is identified by ``model_config``.
190
191
+ Raises:
192
+ TypeError: If the data type is not supported.
193
+
194
See also:
195
:ref:`adding_a_new_multimodal_model`
196
"""
@@ -188,3 +205,79 @@ class MultiModalPlugin(ABC):
205
f"model class {model_cls.__name__}.")
206
207
return mapper(InputContext(model_config), data)
208
+
209
+ @abstractmethod
210
+ def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
211
+ """
212
+ Calculate the maximum number of multimodal tokens input to the language
213
+ model. This does not include tokens that correspond to the input text.
214
+ """
215
+ raise NotImplementedError
216
+
217
+ def _validate_max_multimodal_tokens(self, max_mm_tokens: int):
218
+ if max_mm_tokens < 1:
219
+ raise ValueError("You should set the number of tokens to a "
220
+ f"positive integer. Found: {max_mm_tokens}")
221
+
222
+ def register_max_multimodal_tokens(
223
+ self,
224
+ max_mm_tokens: Optional[MultiModalTokensCalc] = None,
225
+ ):
226
+ """
227
+ Register the maximum number of multi-modal tokens input to the
228
+ language model for a model class.
229
+
230
+ If `None` is provided, then the default calculation is used instead.
231
+
232
+ See also:
233
+ :ref:`adding_a_new_multimodal_model`
234
+ """
235
+
236
+ def wrapper(model_cls: N) -> N:
237
+ if model_cls in self._max_mm_tokens:
238
+ logger.warning(
239
+ "Model class %s already calculates maximum number of "
240
+ "tokens in %s. It is overwritten by the new one.",
241
+ model_cls, self)
242
+
243
+ if isinstance(max_mm_tokens, int):
244
+ self._validate_max_multimodal_tokens(max_mm_tokens)
245
+
246
+ self._max_mm_tokens[model_cls] = max_mm_tokens \
247
+ or self._default_max_multimodal_tokens
248
+
249
+ return model_cls
250
+
251
+ return wrapper
252
+
253
+ def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
254
+ """
255
+ Get the maximum number of multi-modal tokens
256
+ for profiling the memory usage of a model.
257
+
258
+ If this registry is not applicable to the model, `0` is returned.
259
+
260
+ The model is identified by ``model_config``.
261
+
262
+ See also:
263
+ :ref:`adding_a_new_multimodal_model`
264
+ """
265
+ # Avoid circular import
266
+ from vllm.model_executor.model_loader import get_model_architecture
267
+
268
+ model_cls, _ = get_model_architecture(model_config)
269
+
270
+ if model_cls not in self._input_mappers:
271
+ return 0
272
+
273
+ max_mm_tokens = self._max_mm_tokens.get(model_cls)
274
+ if max_mm_tokens is None:
275
+ raise KeyError(f"No maximum number of multi-modal tokens is given "
276
+ f"for model class {model_cls.__name__} in {self}.")
277
+
278
+ if callable(max_mm_tokens):
279
+ max_mm_tokens = max_mm_tokens(InputContext(model_config))
280
+
281
+ self._validate_max_multimodal_tokens(max_mm_tokens)
282
+
283
+ return max_mm_tokens
vllm/multimodal/image.py
+3
@@ -130,3 +130,6 @@ class ImagePlugin(MultiModalPlugin):
130
raise NotImplementedError("Embeddings input is not supported yet")
131
132
raise TypeError(f"Invalid image type: {type(data)}")
133
+
134
+ def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
135
+ return 3000
vllm/multimodal/registry.py
+39
-34
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7
from vllm.logger import init_logger
8
9
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
10
- MultiModalPlugin)
10
+ MultiModalPlugin, MultiModalTokensCalc)
11
from .image import ImagePlugin
12
13
logger = init_logger(__name__)
@@ -48,45 +48,24 @@ class MultiModalRegistry:
48
msg = f"Unknown multi-modal data type: {data_type_key}"
49
raise NotImplementedError(msg)
50
51
- def register_image_input_mapper(
51
+ def register_input_mapper(
52
self,
53
+ data_type_key: str,
54
mapper: Optional[MultiModalInputMapper] = None,
55
):
56
"""
56
- Register an input mapper for image data to a model class.
57
+ Register an input mapper for a specific modality to a model class.
58
59
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
60
"""
60
- return self.register_input_mapper("image", mapper)
61
-
62
- def _process_input(self, key: str, value: object,
63
- model_config: ModelConfig) -> MultiModalInputs:
64
- plugin = self._plugins.get(key)
65
- if plugin:
66
- return plugin.map_input(model_config, value)
67
- msg = f"Unknown multi-modal data type: {key}"
68
- raise NotImplementedError(msg)
61
+ return self._get_plugin(data_type_key).register_input_mapper(mapper)
62
70
- def register_input_mapper(
63
+ def register_image_input_mapper(
64
self,
72
- data_type: str,
65
mapper: Optional[MultiModalInputMapper] = None,
66
):
67
"""
76
- Register an input mapper for a specific modality to a model class.
77
-
78
- See :meth:`MultiModalPlugin.register_input_mapper` for more details.
79
- """
80
- plugin = self._plugins.get(data_type)
81
- if not plugin:
82
- msg = f"Unknown multi-modal data type: {data_type}"
83
- raise NotImplementedError(msg)
84
- return plugin.register_input_mapper(mapper)
85
-
86
- def register_image_input(self,
87
- mapper: Optional[MultiModalInputMapper] = None):
88
- """
89
- Register an input mapper for image pixel data to a model class.
68
+ Register an input mapper for image data to a model class.
69
70
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
71
"""
@@ -102,8 +81,8 @@ class MultiModalRegistry:
81
merged_dict: Dict[str, torch.Tensor] = {}
82
83
for data_key, data_value in data.items():
105
- input_dict = self._process_input(data_key, data_value,
106
- model_config)
84
+ input_dict = self._get_plugin(data_key) \
85
+ .map_input(model_config, data_value)
86
87
for input_key, input_tensor in input_dict.items():
88
if input_key in merged_dict:
@@ -121,9 +100,35 @@ class MultiModalRegistry:
100
"""
101
return functools.partial(self.map_input, model_config)
102
124
- def get_num_input_tokens(self):
103
+ def register_max_multimodal_tokens(
104
+ self,
105
+ data_type_key: str,
106
+ max_mm_tokens: Optional[MultiModalTokensCalc] = None,
107
+ ):
108
"""
126
- Get the number of input tokens for profiling purposes.
109
+ Register the maximum number of tokens, belonging to a
110
+ specific modality, input to the language model for a model class.
111
+ """
112
+ return self._get_plugin(data_type_key) \
113
+ .register_max_multimodal_tokens(max_mm_tokens)
114
+
115
+ def register_max_image_tokens(
116
+ self,
117
+ max_mm_tokens: Optional[MultiModalTokensCalc] = None,
118
+ ):
119
+ """
120
+ Register the maximum number of image tokens
121
+ input to the language model for a model class.
122
+ """
123
+ return self.register_max_multimodal_tokens("image", max_mm_tokens)
124
+
125
+ def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
126
+ """
127
+ Get the maximum number of multi-modal tokens
128
+ for profiling the memory usage of a model.
129
+
130
+ See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
131
"""
128
- # TODO: Provide this number on a per model basis.
129
- return 3000
132
+ return sum(
133
+ plugin.get_max_multimodal_tokens(model_config)
134
+ for plugin in self._plugins.values())
vllm/worker/model_runner.py
+13
-6
@@ -820,12 +820,19 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
820
model_config = self.model_config
821
822
if supports_vision(self.model):
823
- max_num_seqs = max(
824
- 1,
825
- min(
826
- max_num_seqs,
827
- int(max_num_batched_tokens /
828
- MULTIMODAL_REGISTRY.get_num_input_tokens())))
823
+ max_mm_tokens = MULTIMODAL_REGISTRY \
824
+ .get_max_multimodal_tokens(model_config)
825
+ max_num_seqs_orig = max_num_seqs
826
+ max_num_seqs = min(max_num_seqs,
827
+ max_num_batched_tokens // max_mm_tokens)
828
+ if max_num_seqs < 1:
829
+ expr = (f"min({max_num_seqs_orig}, "
830
+ f"{max_num_batched_tokens} // {max_mm_tokens})")
831
+ logger.warning(
832
+ "Computed max_num_seqs (%s) to be less than 1. "
833
+ "Setting it to the minimum value of 1.", expr)
834
+ max_num_seqs = 1
835
+
836
batch_size = 0
837
for group_id in range(max_num_seqs):
838
seq_len = (max_num_batched_tokens // max_num_seqs +
vllm/worker/xpu_model_runner.py
+12
-8
@@ -168,14 +168,18 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
168
model_config = self.model_config
169
170
if supports_vision(self.model):
171
- # TODO: properly inject these numbers from MultiModalRegistry.
172
- # Right now, just use an overly conservative number.
173
- max_num_seqs = max(
174
- 1,
175
- min(
176
- max_num_seqs,
177
- int(max_num_batched_tokens /
178
- MULTIMODAL_REGISTRY.get_num_input_tokens())))
171
+ max_mm_tokens = MULTIMODAL_REGISTRY \
172
+ .get_max_multimodal_tokens(model_config)
173
+ max_num_seqs_orig = max_num_seqs
174
+ max_num_seqs = min(max_num_seqs,
175
+ max_num_batched_tokens // max_mm_tokens)
176
+ if max_num_seqs < 1:
177
+ expr = (f"min({max_num_seqs_orig}, "
178
+ f"{max_num_batched_tokens} // {max_mm_tokens})")
179
+ logger.warning(
180
+ "Computed max_num_seqs (%s) to be less than 1. "
181
+ "Setting it to the minimum value of 1.", expr)
182
+ max_num_seqs = 1
183
184
for group_id in range(max_num_seqs):
185
seq_len = (max_num_batched_tokens // max_num_seqs +