@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae96ef8fbd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[VLM] Calculate maximum number of multi-modal tokens by model (#6121)

Cyrus Leung committed Jul 5, 2024 at 07:37 UTC ae96ef8fbd6fe8905f8ea0d3a3f9ff1738dbcbe5
12 files changed +260 -90
docs/source/dev/multimodal/adding_multimodal_model.rst
+48 -20
@@ -51,17 +51,16 @@ As usual, follow :ref:`these steps <adding_a_new_model>` to implement the model
51 2. Register input mappers
52 -------------------------
53
54 -For each modality type to support, decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
54 +For each modality type that the model accepts as input, decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
55 This decorator accepts a function that maps multi-modal inputs to the keyword arguments you have previously defined in :meth:`~torch.nn.Module.forward`.
56
57 .. code-block:: diff
58
59 - from vllm.model_executor.models.interfaces import SupportsVision
59 + from vllm.model_executor.models.interfaces import SupportsVision
60 + from vllm.multimodal import MULTIMODAL_REGISTRY
61
62 - + @MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
63 - + @MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
64 - class YourModelForImage2Seq(nn.Module, SupportsVision):
62 + + @MULTIMODAL_REGISTRY.register_image_input_mapper()
63 + class YourModelForImage2Seq(nn.Module, SupportsVision):
64
65 A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
66
@@ -69,7 +68,33 @@ A default mapper is available for each modality in the core vLLM library. This i
68 :ref:`input_processing_pipeline`
69
70
72 -3. (Optional) Register dummy data
71 +3. Register maximum number of multimodal tokens
72 +----------------------------------------------------------
73 +
74 +For each modality type that the model accepts as input, calculate the maximum possible number of tokens
75 +and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
76 +
77 +.. code-block:: diff
78 +
79 + from vllm.inputs import INPUT_REGISTRY
80 + from vllm.model_executor.models.interfaces import SupportsVision
81 + from vllm.multimodal import MULTIMODAL_REGISTRY
82 +
83 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
84 + + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
85 + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
86 + class YourModelForImage2Seq(nn.Module, SupportsVision):
87 +
88 +Here are some examples:
89 +
90 +- Image inputs (static feature size): `LLaVA-1.5 Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py>`__
91 +- Image inputs (dynamic feature size): `LLaVA-NeXT Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py>`__
92 +
93 +.. seealso::
94 + :ref:`input_processing_pipeline`
95 +
96 +
97 +4. (Optional) Register dummy data
98 ---------------------------------
99
100 During startup, dummy data is passed to the vLLM model to allocate memory. This only consists of text input by default, which may not be applicable to multi-modal models.
@@ -77,14 +102,17 @@ In such cases, you can define your own dummy data by registering a factory metho
102
103 .. code-block:: diff
104
80 - from vllm.inputs import INPUT_REGISTRY
81 - from vllm.model_executor.models.interfaces import SupportsVision
82 - from vllm.multimodal import MULTIMODAL_REGISTRY
105 + from vllm.inputs import INPUT_REGISTRY
106 + from vllm.model_executor.models.interfaces import SupportsVision
107 + from vllm.multimodal import MULTIMODAL_REGISTRY
108
84 - @MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
85 - @MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
109 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
110 + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
111 + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
87 - class YourModelForImage2Seq(nn.Module, SupportsVision):
112 + class YourModelForImage2Seq(nn.Module, SupportsVision):
113 +
114 +.. note::
115 + The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
116
117 Here are some examples:
118
@@ -95,7 +123,7 @@ Here are some examples:
123 :ref:`input_processing_pipeline`
124
125
98 -4. (Optional) Register input processor
126 +5. (Optional) Register input processor
127 --------------------------------------
128
129 Sometimes, there is a need to process inputs at the :class:`~vllm.LLMEngine` level before they are passed to the model executor.
@@ -104,15 +132,15 @@ You can register input processors via :meth:`INPUT_REGISTRY.register_input_proce
132
133 .. code-block:: diff
134
107 - from vllm.inputs import INPUT_REGISTRY
108 - from vllm.model_executor.models.interfaces import SupportsVision
109 - from vllm.multimodal import MULTIMODAL_REGISTRY
135 + from vllm.inputs import INPUT_REGISTRY
136 + from vllm.model_executor.models.interfaces import SupportsVision
137 + from vllm.multimodal import MULTIMODAL_REGISTRY
138
111 - @MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
112 - @MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
113 - @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
139 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
140 + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
141 + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
142 + @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
115 - class YourModelForImage2Seq(nn.Module, SupportsVision):
143 + class YourModelForImage2Seq(nn.Module, SupportsVision):
144
145 A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
146 Here are some examples:
docs/source/models/vlm.rst
+4 -14
@@ -25,13 +25,8 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
25
26 .. important::
27 We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28 - the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
29 - every model to perform profiling with.
30 -
31 - This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
32 - :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
33 - for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
34 - with a more accurate profiling strategy in the future.
28 + the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified as we now calculate that
29 + internally for each model.
30
31
32 To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
@@ -104,13 +99,8 @@ Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with
99
100 .. important::
101 We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
107 - the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
108 - every model to perform profiling with.
109 -
110 - This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
111 - :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
112 - for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
113 - with a more accurate profiling strategy in the future.
102 + the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified as we now calculate that
103 + internally for each model.
104
105 To consume the server, you can use the OpenAI client like in the example below:
106
vllm/inputs/registry.py
+1 -1
@@ -51,7 +51,7 @@ class InputContext:
51 additionally checking its type.
52
53 Raises:
54 - ValueError: If the model is not of the specified type.
54 + TypeError: If the model is not of the specified type.
55 """
56
57 hf_config = self.model_config.hf_config
vllm/model_executor/models/clip.py
+4
@@ -35,6 +35,10 @@ def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
35 patch_size=hf_config.patch_size)
36
37
38 +def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
39 + return get_clip_image_feature_size(hf_config)
40 +
41 +
42 def dummy_seq_data_for_clip(
43 hf_config: CLIPVisionConfig,
44 seq_len: int,
vllm/model_executor/models/llava.py
+13 -1
@@ -21,7 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.sequence import IntermediateTensors, SamplerOutput
22
23 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
24 - input_processor_for_clip)
24 + get_max_clip_image_tokens, input_processor_for_clip)
25 from .interfaces import SupportsVision
26 from .utils import merge_vision_embeddings
27
@@ -62,6 +62,17 @@ class LlavaImagePixelInputs(TypedDict):
62 LlavaImageInputs = LlavaImagePixelInputs
63
64
65 +def get_max_llava_image_tokens(ctx: InputContext):
66 + hf_config = ctx.get_hf_config(LlavaConfig)
67 + vision_config = hf_config.vision_config
68 +
69 + if isinstance(vision_config, CLIPVisionConfig):
70 + return get_max_clip_image_tokens(vision_config)
71 +
72 + msg = f"Unsupported vision config: {type(vision_config)}"
73 + raise NotImplementedError(msg)
74 +
75 +
76 def dummy_data_for_llava(ctx: InputContext, seq_len: int):
77 hf_config = ctx.get_hf_config(LlavaConfig)
78 vision_config = hf_config.vision_config
@@ -102,6 +113,7 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
113
114
115 @MULTIMODAL_REGISTRY.register_image_input_mapper()
116 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
117 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
118 @INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
119 class LlavaForConditionalGeneration(nn.Module, SupportsVision):
vllm/model_executor/models/llava_next.py
+12
@@ -127,6 +127,17 @@ def get_llava_next_image_feature_size(
127 raise NotImplementedError(msg)
128
129
130 +def get_max_llava_next_image_tokens(ctx: InputContext):
131 + # Result in the max possible feature size (2x2 grid of 336x336px tiles)
132 + dummy_height = dummy_width = 448
133 +
134 + return get_llava_next_image_feature_size(
135 + ctx.get_hf_config(LlavaNextConfig),
136 + input_height=dummy_height,
137 + input_width=dummy_width,
138 + )
139 +
140 +
141 def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
142 hf_config = ctx.get_hf_config(LlavaNextConfig)
143 vision_config = hf_config.vision_config
@@ -198,6 +209,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
209
210
211 @MULTIMODAL_REGISTRY.register_image_input_mapper()
212 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
213 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
214 @INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next)
215 class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
vllm/model_executor/models/phi3v.py
+12
@@ -321,6 +321,17 @@ def get_phi3v_image_feature_size(
321 + (new_height // 336 + 1) * 12
322
323
324 +def get_max_phi3v_image_tokens(ctx: InputContext):
325 + # Result in the max possible feature size (h:w = 16:1)
326 + dummy_height, dummy_width = 8000, 50
327 +
328 + return get_phi3v_image_feature_size(
329 + ctx.get_hf_config(PretrainedConfig),
330 + input_height=dummy_height,
331 + input_width=dummy_width,
332 + )
333 +
334 +
335 def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
336 # Result in the max possible feature size (h:w = 16:1)
337 dummy_height, dummy_width = 8000, 50
@@ -429,6 +440,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
440
441
442 @MULTIMODAL_REGISTRY.register_image_input_mapper()
443 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
444 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
445 @INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
446 class Phi3VForCausalLM(nn.Module, SupportsVision):
vllm/multimodal/base.py
+99 -6
@@ -97,9 +97,19 @@ the corresponding plugin with the same modality key is applied.
97 """
98
99 MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
100 -"""Return a dictionary to be passed as keyword arguments to
100 +"""
101 +Return a dictionary to be passed as keyword arguments to
102 :meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
102 -and processors in HuggingFace Transformers."""
103 +and processors in HuggingFace Transformers.
104 +
105 +If the data is not supported, throw :exc:`TypeError`.
106 +"""
107 +
108 +MultiModalTokensCalc = Union[int, Callable[[InputContext], int]]
109 +"""
110 +Calculate the maximum number of multimodal tokens input to the language
111 +model. This does not include tokens that correspond to the input text.
112 +"""
113
114 N = TypeVar("N", bound=Type[nn.Module])
115
@@ -117,6 +127,7 @@ class MultiModalPlugin(ABC):
127
128 def __init__(self) -> None:
129 self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {}
130 + self._max_mm_tokens: Dict[Type[nn.Module], MultiModalTokensCalc] = {}
131
132 @abstractmethod
133 def get_data_key(self) -> str:
@@ -128,9 +139,12 @@ class MultiModalPlugin(ABC):
139 @abstractmethod
140 def _default_input_mapper(self, ctx: InputContext,
141 data: object) -> MultiModalInputs:
131 - """Return a dictionary to be passed as keyword arguments to
142 + """
143 + Return a dictionary to be passed as keyword arguments to
144 :meth:`~torch.nn.Module.forward`. This is similar in concept to
145 tokenizers and processors in HuggingFace Transformers.
146 +
147 + If the data is not supported, throw :exc:`TypeError`.
148 """
149 raise NotImplementedError
150
@@ -140,9 +154,11 @@ class MultiModalPlugin(ABC):
154 ):
155 """
156 Register an input mapper to a model class.
157 +
158 When the model receives input data that matches the modality served by
144 - this plugin (see :meth:`get_data_type`), the provided function is
159 + this plugin (see :meth:`get_data_key`), the provided function is
160 invoked to transform the data into a dictionary of model inputs.
161 +
162 If `None` is provided, then the default input mapper is used instead.
163
164 See also:
@@ -170,10 +186,11 @@ class MultiModalPlugin(ABC):
186 Apply an input mapper to a data passed
187 to the model, transforming the data into a dictionary of model inputs.
188
173 - If the data is not something that the mapper expects, throws TypeError.
174 -
189 The model is identified by ``model_config``.
190
191 + Raises:
192 + TypeError: If the data type is not supported.
193 +
194 See also:
195 :ref:`adding_a_new_multimodal_model`
196 """
@@ -188,3 +205,79 @@ class MultiModalPlugin(ABC):
205 f"model class {model_cls.__name__}.")
206
207 return mapper(InputContext(model_config), data)
208 +
209 + @abstractmethod
210 + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
211 + """
212 + Calculate the maximum number of multimodal tokens input to the language
213 + model. This does not include tokens that correspond to the input text.
214 + """
215 + raise NotImplementedError
216 +
217 + def _validate_max_multimodal_tokens(self, max_mm_tokens: int):
218 + if max_mm_tokens < 1:
219 + raise ValueError("You should set the number of tokens to a "
220 + f"positive integer. Found: {max_mm_tokens}")
221 +
222 + def register_max_multimodal_tokens(
223 + self,
224 + max_mm_tokens: Optional[MultiModalTokensCalc] = None,
225 + ):
226 + """
227 + Register the maximum number of multi-modal tokens input to the
228 + language model for a model class.
229 +
230 + If `None` is provided, then the default calculation is used instead.
231 +
232 + See also:
233 + :ref:`adding_a_new_multimodal_model`
234 + """
235 +
236 + def wrapper(model_cls: N) -> N:
237 + if model_cls in self._max_mm_tokens:
238 + logger.warning(
239 + "Model class %s already calculates maximum number of "
240 + "tokens in %s. It is overwritten by the new one.",
241 + model_cls, self)
242 +
243 + if isinstance(max_mm_tokens, int):
244 + self._validate_max_multimodal_tokens(max_mm_tokens)
245 +
246 + self._max_mm_tokens[model_cls] = max_mm_tokens \
247 + or self._default_max_multimodal_tokens
248 +
249 + return model_cls
250 +
251 + return wrapper
252 +
253 + def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
254 + """
255 + Get the maximum number of multi-modal tokens
256 + for profiling the memory usage of a model.
257 +
258 + If this registry is not applicable to the model, `0` is returned.
259 +
260 + The model is identified by ``model_config``.
261 +
262 + See also:
263 + :ref:`adding_a_new_multimodal_model`
264 + """
265 + # Avoid circular import
266 + from vllm.model_executor.model_loader import get_model_architecture
267 +
268 + model_cls, _ = get_model_architecture(model_config)
269 +
270 + if model_cls not in self._input_mappers:
271 + return 0
272 +
273 + max_mm_tokens = self._max_mm_tokens.get(model_cls)
274 + if max_mm_tokens is None:
275 + raise KeyError(f"No maximum number of multi-modal tokens is given "
276 + f"for model class {model_cls.__name__} in {self}.")
277 +
278 + if callable(max_mm_tokens):
279 + max_mm_tokens = max_mm_tokens(InputContext(model_config))
280 +
281 + self._validate_max_multimodal_tokens(max_mm_tokens)
282 +
283 + return max_mm_tokens
vllm/multimodal/image.py
+3
@@ -130,3 +130,6 @@ class ImagePlugin(MultiModalPlugin):
130 raise NotImplementedError("Embeddings input is not supported yet")
131
132 raise TypeError(f"Invalid image type: {type(data)}")
133 +
134 + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
135 + return 3000
vllm/multimodal/registry.py
+39 -34
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7 from vllm.logger import init_logger
8
9 from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
10 - MultiModalPlugin)
10 + MultiModalPlugin, MultiModalTokensCalc)
11 from .image import ImagePlugin
12
13 logger = init_logger(__name__)
@@ -48,45 +48,24 @@ class MultiModalRegistry:
48 msg = f"Unknown multi-modal data type: {data_type_key}"
49 raise NotImplementedError(msg)
50
51 - def register_image_input_mapper(
51 + def register_input_mapper(
52 self,
53 + data_type_key: str,
54 mapper: Optional[MultiModalInputMapper] = None,
55 ):
56 """
56 - Register an input mapper for image data to a model class.
57 + Register an input mapper for a specific modality to a model class.
58
59 See :meth:`MultiModalPlugin.register_input_mapper` for more details.
60 """
60 - return self.register_input_mapper("image", mapper)
61 -
62 - def _process_input(self, key: str, value: object,
63 - model_config: ModelConfig) -> MultiModalInputs:
64 - plugin = self._plugins.get(key)
65 - if plugin:
66 - return plugin.map_input(model_config, value)
67 - msg = f"Unknown multi-modal data type: {key}"
68 - raise NotImplementedError(msg)
61 + return self._get_plugin(data_type_key).register_input_mapper(mapper)
62
70 - def register_input_mapper(
63 + def register_image_input_mapper(
64 self,
72 - data_type: str,
65 mapper: Optional[MultiModalInputMapper] = None,
66 ):
67 """
76 - Register an input mapper for a specific modality to a model class.
77 -
78 - See :meth:`MultiModalPlugin.register_input_mapper` for more details.
79 - """
80 - plugin = self._plugins.get(data_type)
81 - if not plugin:
82 - msg = f"Unknown multi-modal data type: {data_type}"
83 - raise NotImplementedError(msg)
84 - return plugin.register_input_mapper(mapper)
85 -
86 - def register_image_input(self,
87 - mapper: Optional[MultiModalInputMapper] = None):
88 - """
89 - Register an input mapper for image pixel data to a model class.
68 + Register an input mapper for image data to a model class.
69
70 See :meth:`MultiModalPlugin.register_input_mapper` for more details.
71 """
@@ -102,8 +81,8 @@ class MultiModalRegistry:
81 merged_dict: Dict[str, torch.Tensor] = {}
82
83 for data_key, data_value in data.items():
105 - input_dict = self._process_input(data_key, data_value,
106 - model_config)
84 + input_dict = self._get_plugin(data_key) \
85 + .map_input(model_config, data_value)
86
87 for input_key, input_tensor in input_dict.items():
88 if input_key in merged_dict:
@@ -121,9 +100,35 @@ class MultiModalRegistry:
100 """
101 return functools.partial(self.map_input, model_config)
102
124 - def get_num_input_tokens(self):
103 + def register_max_multimodal_tokens(
104 + self,
105 + data_type_key: str,
106 + max_mm_tokens: Optional[MultiModalTokensCalc] = None,
107 + ):
108 """
126 - Get the number of input tokens for profiling purposes.
109 + Register the maximum number of tokens, belonging to a
110 + specific modality, input to the language model for a model class.
111 + """
112 + return self._get_plugin(data_type_key) \
113 + .register_max_multimodal_tokens(max_mm_tokens)
114 +
115 + def register_max_image_tokens(
116 + self,
117 + max_mm_tokens: Optional[MultiModalTokensCalc] = None,
118 + ):
119 + """
120 + Register the maximum number of image tokens
121 + input to the language model for a model class.
122 + """
123 + return self.register_max_multimodal_tokens("image", max_mm_tokens)
124 +
125 + def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
126 + """
127 + Get the maximum number of multi-modal tokens
128 + for profiling the memory usage of a model.
129 +
130 + See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
131 """
128 - # TODO: Provide this number on a per model basis.
129 - return 3000
132 + return sum(
133 + plugin.get_max_multimodal_tokens(model_config)
134 + for plugin in self._plugins.values())
vllm/worker/model_runner.py
+13 -6
@@ -820,12 +820,19 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
820 model_config = self.model_config
821
822 if supports_vision(self.model):
823 - max_num_seqs = max(
824 - 1,
825 - min(
826 - max_num_seqs,
827 - int(max_num_batched_tokens /
828 - MULTIMODAL_REGISTRY.get_num_input_tokens())))
823 + max_mm_tokens = MULTIMODAL_REGISTRY \
824 + .get_max_multimodal_tokens(model_config)
825 + max_num_seqs_orig = max_num_seqs
826 + max_num_seqs = min(max_num_seqs,
827 + max_num_batched_tokens // max_mm_tokens)
828 + if max_num_seqs < 1:
829 + expr = (f"min({max_num_seqs_orig}, "
830 + f"{max_num_batched_tokens} // {max_mm_tokens})")
831 + logger.warning(
832 + "Computed max_num_seqs (%s) to be less than 1. "
833 + "Setting it to the minimum value of 1.", expr)
834 + max_num_seqs = 1
835 +
836 batch_size = 0
837 for group_id in range(max_num_seqs):
838 seq_len = (max_num_batched_tokens // max_num_seqs +
vllm/worker/xpu_model_runner.py
+12 -8
@@ -168,14 +168,18 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
168 model_config = self.model_config
169
170 if supports_vision(self.model):
171 - # TODO: properly inject these numbers from MultiModalRegistry.
172 - # Right now, just use an overly conservative number.
173 - max_num_seqs = max(
174 - 1,
175 - min(
176 - max_num_seqs,
177 - int(max_num_batched_tokens /
178 - MULTIMODAL_REGISTRY.get_num_input_tokens())))
171 + max_mm_tokens = MULTIMODAL_REGISTRY \
172 + .get_max_multimodal_tokens(model_config)
173 + max_num_seqs_orig = max_num_seqs
174 + max_num_seqs = min(max_num_seqs,
175 + max_num_batched_tokens // max_mm_tokens)
176 + if max_num_seqs < 1:
177 + expr = (f"min({max_num_seqs_orig}, "
178 + f"{max_num_batched_tokens} // {max_mm_tokens})")
179 + logger.warning(
180 + "Computed max_num_seqs (%s) to be less than 1. "
181 + "Setting it to the minimum value of 1.", expr)
182 + max_num_seqs = 1
183
184 for group_id in range(max_num_seqs):
185 seq_len = (max_num_batched_tokens // max_num_seqs +