@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aec9674dbe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 13:30 UTC

[Core] Remove legacy input mapper/processor from V0 (#15686)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 28, 2025 at 15:38 UTC aec9674dbe6d0b9de3a826e382d0b1c0cb5ea304
21 files changed +144 -1339
vllm/core/scheduler.py
-1
@@ -1596,7 +1596,6 @@ class Scheduler:
1596 multi_modal_placeholders=(
1597 seq_group.multi_modal_placeholders
1598 if scheduler_outputs.num_prefill_groups > 0 else None),
1599 - mm_processor_kwargs=seq_group.mm_processor_kwargs,
1599 prompt_adapter_request=seq_group.prompt_adapter_request,
1600 )
1601 else:
vllm/engine/async_llm_engine.py
+1 -2
@@ -493,12 +493,11 @@ class _AsyncLLMEngine(LLMEngine):
493 tokenizer = await self.get_tokenizer_async(lora_request)
494 self._validate_token_prompt(prompt, tokenizer=tokenizer)
495
496 - preprocessed_inputs = await self.input_preprocessor.preprocess_async(
496 + processed_inputs = await self.input_preprocessor.preprocess_async(
497 prompt,
498 lora_request=lora_request,
499 prompt_adapter_request=prompt_adapter_request,
500 )
501 - processed_inputs = self.input_processor(preprocessed_inputs)
501
502 if isinstance(params, SamplingParams) and \
503 params.guided_decoding is not None:
vllm/engine/llm_engine.py
+3 -10
@@ -29,8 +29,7 @@ from vllm.engine.output_processor.util import create_output_by_sequence_group
29 from vllm.entrypoints.openai.logits_processors import (
30 get_logits_processors as get_openai_logits_processors)
31 from vllm.executor.executor_base import ExecutorBase
32 -from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
33 - PromptType, SingletonInputs)
32 +from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
33 from vllm.inputs.parse import is_token_prompt, split_enc_dec_inputs
34 from vllm.inputs.preprocess import InputPreprocessor
35 from vllm.logger import init_logger
@@ -213,7 +212,6 @@ class LLMEngine:
212 log_stats: bool,
213 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
214 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
216 - input_registry: InputRegistry = INPUT_REGISTRY,
215 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
216 use_cached_outputs: bool = False,
217 ) -> None:
@@ -274,11 +272,7 @@ class LLMEngine:
272 self.tokenizer,
273 mm_registry)
274
277 - self.input_registry = input_registry
278 - self.input_processor = input_registry.create_input_processor(
279 - self.model_config)
280 -
281 - self.model_executor = executor_class(vllm_config=vllm_config, )
275 + self.model_executor = executor_class(vllm_config=vllm_config)
276
277 if self.model_config.runner_type != "pooling":
278 self._initialize_kv_caches()
@@ -762,12 +756,11 @@ class LLMEngine:
756 prompt,
757 tokenizer=self.get_tokenizer(lora_request=lora_request))
758
765 - preprocessed_inputs = self.input_preprocessor.preprocess(
759 + processed_inputs = self.input_preprocessor.preprocess(
760 prompt,
761 lora_request=lora_request,
762 prompt_adapter_request=prompt_adapter_request,
763 )
770 - processed_inputs = self.input_processor(preprocessed_inputs)
764
765 self._add_processed_request(
766 request_id=request_id,
vllm/inputs/__init__.py
+3 -5
@@ -2,10 +2,9 @@
2
3 from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
4 ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType,
5 - SingletonInputs, SingletonInputsAdapter, SingletonPrompt,
6 - TextPrompt, TokenInputs, TokensPrompt,
7 - build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
8 - token_inputs, zip_enc_dec_prompts)
5 + SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
6 + TokensPrompt, build_explicit_enc_dec_prompt,
7 + to_enc_dec_tuple_list, token_inputs, zip_enc_dec_prompts)
8 from .registry import (DummyData, InputContext, InputProcessingContext,
9 InputRegistry)
10
@@ -27,7 +26,6 @@ __all__ = [
26 "EncoderDecoderInputs",
27 "ProcessorInputs",
28 "SingletonInputs",
30 - "SingletonInputsAdapter",
29 "build_explicit_enc_dec_prompt",
30 "to_enc_dec_tuple_list",
31 "zip_enc_dec_prompts",
vllm/inputs/data.py
+2 -159
@@ -1,17 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -
2 from collections.abc import Iterable
4 -from dataclasses import dataclass
5 -from functools import cached_property
3 from typing import TYPE_CHECKING, Any, Generic, Literal, Optional, Union, cast
4
8 -import torch
9 -from typing_extensions import NotRequired, TypedDict, TypeVar, assert_never
5 +from typing_extensions import NotRequired, TypedDict, TypeVar
6
7 if TYPE_CHECKING:
12 - from vllm.multimodal import (MultiModalDataDict, MultiModalKwargs,
13 - MultiModalPlaceholderDict)
14 - from vllm.multimodal.inputs import MultiModalInputs
8 + from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputs
9
10
11 class TextPrompt(TypedDict):
@@ -147,46 +141,11 @@ class TokenInputs(TypedDict):
141 The original prompt text corresponding to the token IDs, if available.
142 """
143
150 - multi_modal_data: NotRequired["MultiModalDataDict"]
151 - """
152 - Optional multi-modal data to pass to the model,
153 - if the model supports it.
154 - """
155 -
156 - multi_modal_inputs: NotRequired["MultiModalKwargs"]
157 - """
158 - Optional multi-modal inputs to pass to the model,
159 - if the model supports it.
160 - """
161 -
162 - multi_modal_placeholders: NotRequired["MultiModalPlaceholderDict"]
163 - """
164 - Placeholder ranges for the multi-modal data.
165 - """
166 -
167 - multi_modal_hashes: NotRequired[list[str]]
168 - """
169 - The hashes of the multi-modal data.
170 - """
171 -
172 - mm_processor_kwargs: NotRequired[dict[str, Any]]
173 - """
174 - Optional multi-modal processor kwargs to be forwarded to the
175 - multimodal input mapper & processor. Note that if multiple modalities
176 - have registered mappers etc for the model being considered, we attempt
177 - to pass the mm_processor_kwargs to each of them.
178 - """
179 -
144
145 def token_inputs(
146 prompt_token_ids: list[int],
147 token_type_ids: Optional[list[int]] = None,
148 prompt: Optional[str] = None,
185 - multi_modal_data: Optional["MultiModalDataDict"] = None,
186 - multi_modal_inputs: Optional["MultiModalKwargs"] = None,
187 - multi_modal_hashes: Optional[list[str]] = None,
188 - multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
189 - mm_processor_kwargs: Optional[dict[str, Any]] = None,
149 ) -> TokenInputs:
150 """Construct :class:`TokenInputs` from optional values."""
151 inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
@@ -195,16 +154,6 @@ def token_inputs(
154 inputs["prompt"] = prompt
155 if token_type_ids is not None:
156 inputs["token_type_ids"] = token_type_ids
198 - if multi_modal_data is not None:
199 - inputs["multi_modal_data"] = multi_modal_data
200 - if multi_modal_inputs is not None:
201 - inputs["multi_modal_inputs"] = multi_modal_inputs
202 - if multi_modal_hashes is not None:
203 - inputs["multi_modal_hashes"] = multi_modal_hashes
204 - if multi_modal_placeholders is not None:
205 - inputs["multi_modal_placeholders"] = multi_modal_placeholders
206 - if mm_processor_kwargs is not None:
207 - inputs["mm_processor_kwargs"] = mm_processor_kwargs
157
158 return inputs
159
@@ -237,112 +186,6 @@ A processed :class:`SingletonPrompt` which can be passed to
186 :class:`vllm.sequence.Sequence`.
187 """
188
240 -
241 -@dataclass
242 -class SingletonInputsAdapter:
243 - """
244 - Unified interface to access the components of :class:`SingletonInputs`.
245 - """
246 - inputs: SingletonInputs
247 -
248 - @cached_property
249 - def prompt(self) -> Optional[str]:
250 - inputs = self.inputs
251 -
252 - if inputs["type"] == "token" or inputs["type"] == "multimodal":
253 - return inputs.get("prompt")
254 -
255 - assert_never(inputs) # type: ignore[arg-type]
256 -
257 - @cached_property
258 - def prompt_token_ids(self) -> list[int]:
259 - inputs = self.inputs
260 -
261 - if inputs["type"] == "token" or inputs["type"] == "multimodal":
262 - return inputs.get("prompt_token_ids", [])
263 -
264 - assert_never(inputs) # type: ignore[arg-type]
265 -
266 - @cached_property
267 - def token_type_ids(self) -> list[int]:
268 - inputs = self.inputs
269 -
270 - if inputs["type"] == "token" or inputs["type"] == "multimodal":
271 - return inputs.get("token_type_ids", [])
272 -
273 - assert_never(inputs) # type: ignore[arg-type]
274 -
275 - @cached_property
276 - def prompt_embeds(self) -> Optional[torch.Tensor]:
277 - inputs = self.inputs
278 -
279 - if inputs["type"] == "token" or inputs["type"] == "multimodal":
280 - return None
281 -
282 - assert_never(inputs) # type: ignore[arg-type]
283 -
284 - @cached_property
285 - def multi_modal_data(self) -> "MultiModalDataDict":
286 - inputs = self.inputs
287 -
288 - if inputs["type"] == "token":
289 - return inputs.get("multi_modal_data", {})
290 -
291 - if inputs["type"] == "multimodal":
292 - return inputs.get("mm_kwargs", {})
293 -
294 - assert_never(inputs) # type: ignore[arg-type]
295 -
296 - @cached_property
297 - def multi_modal_inputs(self) -> Union[dict, "MultiModalKwargs"]:
298 - inputs = self.inputs
299 -
300 - if inputs["type"] == "token":
301 - return inputs.get("multi_modal_inputs", {})
302 -
303 - if inputs["type"] == "multimodal":
304 - return inputs.get("mm_kwargs", {})
305 -
306 - assert_never(inputs) # type: ignore[arg-type]
307 -
308 - @cached_property
309 - def multi_modal_hashes(self) -> list[str]:
310 - inputs = self.inputs
311 -
312 - if inputs["type"] == "token":
313 - return inputs.get("multi_modal_hashes", [])
314 -
315 - if inputs["type"] == "multimodal":
316 - # only the case when we use MultiModalInputs
317 - return inputs.get("mm_hashes", []) # type: ignore[return-value]
318 -
319 - assert_never(inputs) # type: ignore[arg-type]
320 -
321 - @cached_property
322 - def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
323 - inputs = self.inputs
324 -
325 - if inputs["type"] == "token":
326 - return inputs.get("multi_modal_placeholders", {})
327 -
328 - if inputs["type"] == "multimodal":
329 - return inputs.get("mm_placeholders", {})
330 -
331 - assert_never(inputs) # type: ignore[arg-type]
332 -
333 - @cached_property
334 - def mm_processor_kwargs(self) -> dict[str, Any]:
335 - inputs = self.inputs
336 -
337 - if inputs["type"] == "token":
338 - return inputs.get("mm_processor_kwargs", {})
339 -
340 - if inputs["type"] == "multimodal":
341 - return {}
342 -
343 - assert_never(inputs) # type: ignore[arg-type]
344 -
345 -
189 ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
190 """
191 The inputs to :data:`vllm.inputs.InputProcessor`.
vllm/inputs/preprocess.py
+11 -49
@@ -223,28 +223,6 @@ class InputPreprocessor:
223 lora_request=lora_request,
224 add_special_tokens=add_special_tokens)
225
226 - def _can_process_multimodal(self) -> bool:
227 - model_config = self.model_config
228 -
229 - if not model_config.is_multimodal_model:
230 - raise ValueError("Your model does not support multi-modal inputs")
231 -
232 - # Interim measure so we can handle models that have yet to be
233 - # updated to use the new multi-modal processor
234 - can_process_multimodal = self.mm_registry.has_processor(model_config)
235 - if not can_process_multimodal:
236 - from vllm.model_executor.models.registry import _VLLM_MODELS
237 - if not any(arch in _VLLM_MODELS
238 - for arch in model_config.architectures):
239 - logger.warning_once(
240 - "Your model uses the legacy input pipeline, which will be "
241 - "removed in an upcoming release. "
242 - "Please upgrade to the new multi-modal processing pipeline "
243 - "(https://docs.vllm.ai/en/latest/design/mm_processing.html)"
244 - )
245 -
246 - return can_process_multimodal
247 -
226 def _process_multimodal(
227 self,
228 prompt: Union[str, list[int]],
@@ -258,8 +236,7 @@ class InputPreprocessor:
236 returning the corresponding token IDs and metadata.
237 """
238 # At the moment on model (PrithviGeoSpatialMAE) requires to be
261 - # initialized without a tokenizer while using also multi-modal
262 - # input.
239 + # initialized without a tokenizer while using also multi-modal input
240 if not self.tokenizer:
241 tokenizer = object() # Dummy
242 else:
@@ -285,8 +262,7 @@ class InputPreprocessor:
262 ) -> MultiModalInputs:
263 """Async version of :meth:`_process_multimodal`."""
264 # At the moment on model (PrithviGeoSpatialMAE) requires to be
288 - # initialized without a tokenizer while using also multi-modal
289 - # input.
265 + # initialized without a tokenizer while using also multi-modal input
266 if not self.tokenizer:
267 tokenizer = object() # Dummy
268 else:
@@ -343,7 +319,7 @@ class InputPreprocessor:
319 multi_modal_data = tokens_content.get("multi_modal_data")
320 mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
321
346 - if multi_modal_data is not None and self._can_process_multimodal():
322 + if multi_modal_data is not None:
323 return self._process_multimodal(
324 prompt_token_ids,
325 multi_modal_data,
@@ -355,8 +331,6 @@ class InputPreprocessor:
331 return token_inputs(
332 prompt_token_ids=prompt_token_ids,
333 token_type_ids=token_type_ids,
358 - multi_modal_data=multi_modal_data,
359 - mm_processor_kwargs=mm_processor_kwargs,
334 )
335
336 if parsed["type"] == "text":
@@ -366,7 +340,7 @@ class InputPreprocessor:
340 multi_modal_data = text_content.get("multi_modal_data")
341 mm_processor_kwargs = text_content.get("mm_processor_kwargs")
342
369 - if multi_modal_data is not None and self._can_process_multimodal():
343 + if multi_modal_data is not None:
344 return self._process_multimodal(
345 prompt_text,
346 multi_modal_data,
@@ -383,8 +357,6 @@ class InputPreprocessor:
357 return token_inputs(
358 prompt=prompt_text,
359 prompt_token_ids=prompt_token_ids,
386 - multi_modal_data=multi_modal_data,
387 - mm_processor_kwargs=mm_processor_kwargs,
360 )
361
362 assert_never(parsed)
@@ -417,7 +389,7 @@ class InputPreprocessor:
389 multi_modal_data = tokens_content.get("multi_modal_data")
390 mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
391
420 - if multi_modal_data is not None and self._can_process_multimodal():
392 + if multi_modal_data is not None:
393 return await self._process_multimodal_async(
394 prompt_token_ids,
395 multi_modal_data,
@@ -426,11 +398,7 @@ class InputPreprocessor:
398 return_mm_hashes=return_mm_hashes,
399 )
400
429 - return token_inputs(
430 - prompt_token_ids=prompt_token_ids,
431 - multi_modal_data=multi_modal_data,
432 - mm_processor_kwargs=mm_processor_kwargs,
433 - )
401 + return token_inputs(prompt_token_ids=prompt_token_ids)
402
403 if parsed["type"] == "text":
404 text_content = parsed["content"]
@@ -439,7 +407,7 @@ class InputPreprocessor:
407 multi_modal_data = text_content.get("multi_modal_data")
408 mm_processor_kwargs = text_content.get("mm_processor_kwargs")
409
442 - if multi_modal_data is not None and self._can_process_multimodal():
410 + if multi_modal_data is not None:
411 return await self._process_multimodal_async(
412 prompt_text,
413 multi_modal_data,
@@ -456,8 +424,6 @@ class InputPreprocessor:
424 return token_inputs(
425 prompt=prompt_text,
426 prompt_token_ids=prompt_token_ids,
459 - multi_modal_data=multi_modal_data,
460 - mm_processor_kwargs=mm_processor_kwargs,
427 )
428
429 assert_never(parsed)
@@ -594,15 +560,13 @@ class InputPreprocessor:
560 decoder_inputs = self._prompt_to_llm_inputs(decoder_input)
561 # For multimodal model, override decoder prompt from processor
562 # with explicit decoder prompt.
597 - if self.model_config.is_multimodal_model and (
598 - self._can_process_multimodal()):
563 + if self.model_config.is_multimodal_model:
564 encoder_inputs, decoder_inputs = (
565 self._separate_enc_dec_inputs_from_mm_processor_outputs(
566 encoder_inputs, decoder_inputs))
567 else:
568 inputs = self._prompt_to_llm_inputs(prompt)
604 - if self.model_config.is_multimodal_model and (
605 - self._can_process_multimodal()):
569 + if self.model_config.is_multimodal_model:
570 # Encoder-Decoder Multimodal model
571 encoder_inputs, decoder_inputs = (
572 self._separate_enc_dec_inputs_from_mm_processor_outputs(
@@ -637,15 +601,13 @@ class InputPreprocessor:
601
602 # For multimodal model, override decoder prompt from processor
603 # with explicit decoder prompt.
640 - if self.model_config.is_multimodal_model and (
641 - self._can_process_multimodal()):
604 + if self.model_config.is_multimodal_model:
605 encoder_inputs, decoder_inputs = (
606 self._separate_enc_dec_inputs_from_mm_processor_outputs(
607 encoder_inputs, decoder_inputs))
608 else:
609 inputs = await self._prompt_to_llm_inputs_async(prompt)
647 - if self.model_config.is_multimodal_model and (
648 - self._can_process_multimodal()):
610 + if self.model_config.is_multimodal_model:
611 # Encoder-Decoder Multimodal model
612 encoder_inputs, decoder_inputs = (
613 self._separate_enc_dec_inputs_from_mm_processor_outputs(
vllm/inputs/registry.py
+23 -298
@@ -1,24 +1,14 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -
3 -import functools
4 -from collections import UserDict
2 from collections.abc import Mapping
3 from dataclasses import dataclass
7 -from typing import (TYPE_CHECKING, Any, Callable, NamedTuple, Optional,
8 - Protocol, Union)
4 +from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
5
10 -from torch import nn
6 from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
12 -from typing_extensions import TypeVar, assert_never
7 +from typing_extensions import TypeVar
8
14 -from vllm.logger import init_logger
9 from vllm.transformers_utils.processor import cached_processor_from_config
10 from vllm.transformers_utils.tokenizer import AnyTokenizer
17 -from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
18 - resolve_mm_processor_kwargs)
19 -
20 -from .data import ProcessorInputs, SingletonInputs
21 -from .parse import split_enc_dec_inputs
11 +from vllm.utils import resolve_mm_processor_kwargs
12
13 if TYPE_CHECKING:
14 from vllm.config import ModelConfig
@@ -26,8 +16,6 @@ if TYPE_CHECKING:
16 MultiModalRegistry)
17 from vllm.sequence import SequenceData
18
29 -logger = init_logger(__name__)
30 -
19 _T = TypeVar("_T")
20 _C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
21 _P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
@@ -172,142 +160,23 @@ class InputProcessingContext(InputContext):
160 raise RuntimeError(msg) from exc
161
162
175 -N = TypeVar("N", bound=type[nn.Module])
176 -
177 -
163 class DummyData(NamedTuple):
179 - """Dummy data used for profiling."""
164 + """
165 + Dummy data used for profiling.
166 +
167 + Note: This is only used in V0.
168 + """
169
170 seq_data: "SequenceData"
171 multi_modal_data: Optional["MultiModalDataDict"] = None
172 multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None
173
174
186 -class DummyDataFactory(Protocol):
187 -
188 - def __call__(
189 - self,
190 - ctx: InputContext,
191 - seq_len: int,
192 - mm_counts: Mapping[str, int],
193 - **mm_processor_kwargs: Any,
194 - ) -> DummyData:
195 - """
196 - Create dummy data to be inputted into the model.
197 -
198 - Note:
199 - :data:`InputProcessor` is not applied to the dummy data.
200 -
201 - The :code:`mm_processor_kwargs` are overrides provided at
202 - initialization time to values in the config whose values
203 - may affect the number of tokens per instance.
204 - """
205 - ...
206 -
207 -
208 -class _MultiModalCounts(UserDict[str, int]):
209 - """
210 - Wraps `mm_counts` for a more informative error message
211 - when attempting to access a plugin that does not exist.
212 - """
213 -
214 - def __getitem__(self, key: str) -> int:
215 - try:
216 - return super().__getitem__(key)
217 - except KeyError as exc:
218 - msg = (f"There is no multi-modal plugin with the key: {key}. "
219 - f"Available keys: {set(self.keys())}")
220 - raise KeyError(msg) from exc
221 -
222 -
223 -InputProcessor = Callable[[InputContext, ProcessorInputs], ProcessorInputs]
224 -"""Preprocess the inputs to the model."""
225 -
226 -
175 class InputRegistry:
176 """
229 - A registry to dispatch data processing
230 - according to the target model.
177 + Note: This is only used in V0.
178 """
179
233 - def __init__(self) -> None:
234 - self._dummy_factories_by_model_type = \
235 - ClassRegistry[nn.Module, DummyDataFactory]()
236 - self._dummy_encoder_factories_by_model_type = \
237 - ClassRegistry[nn.Module, DummyDataFactory]()
238 - self._input_processors_by_model_type = \
239 - ClassRegistry[nn.Module, InputProcessor]()
240 -
241 - def _default_dummy_data_factory(
242 - self,
243 - ctx: InputContext,
244 - seq_len: int,
245 - mm_counts: Mapping[str, int],
246 - ) -> DummyData:
247 - """
248 - The default dummy data factory represents the longest possible text
249 - that can be inputted to the model.
250 -
251 - Note:
252 - :data:`InputProcessor` is not applied to the dummy data.
253 - """
254 - # Avoid circular import
255 - from vllm.sequence import SequenceData
256 -
257 - return DummyData(SequenceData.from_prompt_token_counts((0, seq_len)))
258 -
259 - def register_dummy_data(self, factory: DummyDataFactory):
260 - """
261 - Register a dummy data factory to a model class.
262 -
263 - During memory profiling, the provided function is invoked to create
264 - dummy data to be inputted into the model. The resulting memory usage
265 - should be an upper bound of what the model would use at inference time.
266 - """
267 -
268 - def wrapper(model_cls: N) -> N:
269 - if self._dummy_factories_by_model_type.contains(model_cls,
270 - strict=True):
271 - logger.warning(
272 - "Model class %s already has dummy data "
273 - "registered to %s. It is overwritten by the new one.",
274 - model_cls, self)
275 -
276 - self._dummy_factories_by_model_type[model_cls] = factory
277 -
278 - return model_cls
279 -
280 - return wrapper
281 -
282 - def _get_dummy_data_factory(self, model_cls: type[nn.Module]):
283 - return self._dummy_factories_by_model_type \
284 - .get(model_cls, self._default_dummy_data_factory)
285 -
286 - def register_dummy_encoder_data(self, factory: DummyDataFactory):
287 - """
288 - Register a dummy encoder data factory to a model class
289 -
290 - This is similar to :meth:`~register_dummy_data`, but for encoder input.
291 - """
292 -
293 - def wrapper(model_cls: N) -> N:
294 - if self._dummy_encoder_factories_by_model_type.contains(
295 - model_cls, strict=True):
296 - logger.warning(
297 - "Model class %s already has dummy encoder data "
298 - "registered to %s. It is overwritten by the new one.",
299 - model_cls, self)
300 -
301 - self._dummy_encoder_factories_by_model_type[model_cls] = factory
302 -
303 - return model_cls
304 -
305 - return wrapper
306 -
307 - def _get_dummy_encoder_data_factory(self, model_cls: type[nn.Module]):
308 - return self._dummy_encoder_factories_by_model_type \
309 - .get(model_cls, self._default_dummy_data_factory)
310 -
180 def dummy_data_for_profiling(
181 self,
182 model_config: "ModelConfig",
@@ -319,169 +188,25 @@ class InputRegistry:
188 Create dummy data for profiling the memory usage of a model.
189
190 The model is identified by ``model_config``.
322 -
323 - Note:
324 - This should be called after
325 - :meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
191 """
192 # Avoid circular import
328 - from vllm.model_executor.model_loader import get_model_architecture
329 - from vllm.multimodal import MultiModalKwargs
330 - from vllm.multimodal.profiling import MultiModalProfiler
193 from vllm.sequence import SequenceData
194
333 - if mm_registry.has_processor(model_config):
334 - processor = mm_registry.create_processor(model_config,
335 - disable_cache=True)
336 - profiler = MultiModalProfiler(processor)
337 -
338 - dummy_data_v1 = (profiler.get_encoder_dummy_data(seq_len)
339 - if is_encoder_data else
340 - profiler.get_decoder_dummy_data(seq_len))
341 - _seq_data = SequenceData.from_seqs(
342 - dummy_data_v1.prompt_token_ids) # type: ignore[attr-defined]
343 -
344 - dummy_data = DummyData(
345 - seq_data=_seq_data,
346 - multi_modal_data=getattr(dummy_data_v1, "multi_modal_data",
347 - None),
348 - multi_modal_placeholders=getattr(dummy_data_v1,
349 - "multi_modal_placeholders",
350 - None),
351 - )
352 - else:
353 - model_cls, _ = get_model_architecture(model_config)
354 - if is_encoder_data:
355 - dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
356 - else:
357 - dummy_factory = self._get_dummy_data_factory(model_cls)
358 - mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
359 - mm_processor_kwargs = get_allowed_kwarg_only_overrides(
360 - dummy_factory,
361 - overrides=model_config.mm_processor_kwargs,
362 - requires_kw_only=False,
363 - allow_var_kwargs=True,
364 - )
365 -
366 - dummy_data = dummy_factory(InputContext(model_config), seq_len,
367 - _MultiModalCounts(mm_counts),
368 - **mm_processor_kwargs)
369 -
370 - # Having more tokens is over-conservative but otherwise fine
371 - num_tokens = dummy_data.seq_data.prompt_token_ids
372 - if len(num_tokens) < seq_len:
373 - if is_encoder_data:
374 - logger.warning_once(
375 - f"Expected at least {seq_len} dummy encoder tokens for "
376 - f"profiling, but found {len(num_tokens)} tokens instead.")
377 - else:
378 - raise AssertionError(
379 - f"Expected at least {seq_len} dummy tokens for profiling, "
380 - f"but found {len(num_tokens)} tokens instead.")
381 -
382 - if (dummy_data.multi_modal_data is not None and
383 - not isinstance(dummy_data.multi_modal_data, MultiModalKwargs)):
384 - for k, v in dummy_data.multi_modal_data.items():
385 - num_items = len(v) if isinstance(v, list) else 1
386 - num_expected = mm_counts[k]
387 - assert num_items >= num_expected, (
388 - f"Expected at least {num_expected} dummy '{k}' instances "
389 - f"for profiling, but found {num_items} instances instead.")
390 -
391 - return dummy_data
392 -
393 - def _default_input_processor(
394 - self,
395 - ctx: InputContext,
396 - inputs: ProcessorInputs,
397 - **kwargs: object,
398 - ) -> ProcessorInputs:
399 - """The default input processor is a no-op."""
400 - return inputs
401 -
402 - def register_input_processor(self, processor: InputProcessor):
403 - """
404 - Register an input processor to a model class.
405 -
406 - The provided function is invoked on each input to the model. This
407 - happens before
408 - :meth:`~vllm.multimodal.registry.MultiModalRegistry.map_input`.
409 - """
410 -
411 - def wrapper(model_cls: N) -> N:
412 - if self._input_processors_by_model_type.contains(model_cls,
413 - strict=True):
414 - logger.warning(
415 - "Model class %s already has input processor "
416 - "registered to %s. It is overwritten by the new one.",
417 - model_cls, self)
418 -
419 - self._input_processors_by_model_type[model_cls] = processor
420 -
421 - return model_cls
195 + if not model_config.is_multimodal_model:
196 + seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
197 + return DummyData(seq_data=seq_data)
198
423 - return wrapper
199 + # Encoder dummy data does not contain multi-modal data
200 + if is_encoder_data:
201 + enc_data = mm_registry.get_encoder_dummy_data(
202 + model_config, seq_len)
203 + seq_data = SequenceData.from_seqs(enc_data.prompt_token_ids)
204 + return DummyData(seq_data=seq_data)
205
425 - def _get_model_input_processor(self, model_cls: type[nn.Module]):
426 - return self._input_processors_by_model_type \
427 - .get(model_cls, self._default_input_processor)
428 -
429 - def _ensure_mm_kwargs(
430 - self,
431 - inputs: SingletonInputs,
432 - mm_processor_kwargs: dict[str, Any],
433 - ):
434 - if inputs["type"] == "token":
435 - # In case the input processor for that model fails to set it
436 - if "mm_processor_kwargs" not in inputs:
437 - inputs["mm_processor_kwargs"] = mm_processor_kwargs
438 - elif inputs["type"] == "multimodal":
439 - # Be more strict in V2
440 - assert "mm_kwargs" in inputs
441 - else:
442 - assert_never(inputs["type"]) # type: ignore[arg-type]
443 -
444 - def process_input(self, model_config: "ModelConfig",
445 - inputs: ProcessorInputs) -> ProcessorInputs:
446 - """
447 - Apply an input processor to an instance of model inputs.
448 -
449 - The model is identified by ``model_config``.
450 - """
451 - # Avoid circular import
452 - from vllm.model_executor.model_loader import get_model_architecture
453 -
454 - model_cls, _ = get_model_architecture(model_config)
455 - processor = self._get_model_input_processor(model_cls)
456 -
457 - # Handle multimodal processor kwargs with priority:
458 - # Inference kwargs -> Init kwargs -> {}
459 - # If it's empty, it'll fall back to the default kwarg values
460 - mm_processor_kwargs = resolve_mm_processor_kwargs(
461 - model_config.mm_processor_kwargs,
462 - inputs.get("mm_processor_kwargs", {}), # type: ignore
463 - processor,
464 - requires_kw_only=False,
465 - allow_var_kwargs=True,
466 - )
206 + dec_data = mm_registry.get_decoder_dummy_data(model_config, seq_len)
207
468 - processed_inputs = processor(
469 - InputContext(model_config),
470 - inputs,
471 - **mm_processor_kwargs,
208 + return DummyData(
209 + seq_data=SequenceData.from_seqs(dec_data.prompt_token_ids),
210 + multi_modal_data=dec_data.multi_modal_data,
211 + multi_modal_placeholders=dec_data.multi_modal_placeholders,
212 )
473 -
474 - encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
475 - if encoder_inputs is not None:
476 - self._ensure_mm_kwargs(encoder_inputs, mm_processor_kwargs)
477 - if decoder_inputs is not None:
478 - self._ensure_mm_kwargs(decoder_inputs, mm_processor_kwargs)
479 -
480 - return processed_inputs
481 -
482 - def create_input_processor(self, model_config: "ModelConfig"):
483 - """
484 - Create an input processor (see :meth:`_process_input`) for a
485 - specific model.
486 - """
487 - return functools.partial(self.process_input, model_config)
vllm/multimodal/__init__.py
+1 -3
@@ -1,6 +1,5 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -
3 -from .base import MultiModalPlaceholderMap, MultiModalPlugin
2 +from .base import MultiModalPlaceholderMap
3 from .hasher import MultiModalHashDict, MultiModalHasher
4 from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
5 MultiModalDataDict, MultiModalKwargs,
@@ -26,7 +25,6 @@ __all__ = [
25 "MultiModalKwargs",
26 "MultiModalPlaceholderDict",
27 "MultiModalPlaceholderMap",
29 - "MultiModalPlugin",
28 "NestedTensors",
29 "MULTIMODAL_REGISTRY",
30 "MultiModalRegistry",
vllm/multimodal/audio.py
+1 -22
@@ -7,11 +7,9 @@ from typing import Literal, Optional
7 import numpy as np
8 import numpy.typing as npt
9
10 -from vllm.inputs.registry import InputContext
10 from vllm.utils import PlaceholderModule
11
13 -from .base import MediaIO, MultiModalPlugin
14 -from .inputs import AudioItem, ModalityData, MultiModalKwargs
12 +from .base import MediaIO
13
14 try:
15 import librosa
@@ -24,25 +22,6 @@ except ImportError:
22 soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
23
24
27 -class AudioPlugin(MultiModalPlugin):
28 - """Plugin for audio data."""
29 -
30 - def get_data_key(self) -> str:
31 - return "audio"
32 -
33 - def _default_input_mapper(
34 - self,
35 - ctx: InputContext,
36 - data: ModalityData[AudioItem],
37 - **mm_processor_kwargs,
38 - ) -> MultiModalKwargs:
39 - raise NotImplementedError("There is no default audio input mapper")
40 -
41 - def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
42 - raise NotImplementedError(
43 - "There is no default maximum multimodal tokens")
44 -
45 -
25 def resample_audio_librosa(
26 audio: npt.NDArray[np.floating],
27 *,
vllm/multimodal/base.py
+17 -267
@@ -1,247 +1,23 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from abc import ABC, abstractmethod
4 -from collections import defaultdict
4 from collections.abc import Sequence
5 from pathlib import Path
7 -from typing import (TYPE_CHECKING, Any, Callable, Generic, NamedTuple,
8 - Optional, TypeVar, Union)
9 -
10 -from torch import nn
11 -
12 -from vllm.inputs import InputContext
13 -from vllm.logger import init_logger
14 -from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
15 - resolve_mm_processor_kwargs)
6 +from typing import TYPE_CHECKING, Generic, NamedTuple, TypeVar
7
8 if TYPE_CHECKING:
18 - from vllm.config import ModelConfig
9 from vllm.sequence import SequenceGroupMetadata
10
21 -from .inputs import (ModalityData, MultiModalDataDict, MultiModalKwargs,
22 - PlaceholderRange)
23 -
24 -logger = init_logger(__name__)
25 -
26 -MultiModalInputMapper = Callable[[InputContext, ModalityData[object]],
27 - MultiModalKwargs]
28 -"""
29 -Return a dictionary to be passed as keyword arguments to
30 -:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
31 -and processors in HuggingFace Transformers.
32 -
33 -If the data is not supported, throw :exc:`TypeError`.
34 -"""
35 -
36 -MultiModalTokensCalc = Union[int, Callable[[InputContext], int]]
37 -"""
38 -Calculate the maximum number of multimodal tokens input to the language
39 -model. This does not include tokens that correspond to the input text.
40 -"""
11 +from .inputs import MultiModalKwargs, PlaceholderRange
12
13 _T = TypeVar("_T")
43 -N = TypeVar("N", bound=type[nn.Module])
44 -
45 -
46 -class MultiModalPlugin(ABC):
47 - """
48 - Base class that defines data processing logic for a specific modality.
49 -
50 - In particular, we adopt a registry pattern to dispatch data processing
51 - according to the model being used (considering that different models may
52 - process the same data differently). This registry is in turn used by
53 - :class:`~MultiModalRegistry` which acts at a higher level
54 - (i.e., the modality of the data).
55 - """
56 -
57 - def __init__(self) -> None:
58 - self._input_mappers = ClassRegistry[nn.Module, MultiModalInputMapper]()
59 - self._max_mm_tokens = ClassRegistry[nn.Module, MultiModalTokensCalc]()
60 -
61 - @abstractmethod
62 - def get_data_key(self) -> str:
63 - """
64 - Get the data key corresponding to the modality.
65 - """
66 - raise NotImplementedError
67 -
68 - @abstractmethod
69 - def _default_input_mapper(
70 - self,
71 - ctx: InputContext,
72 - data: ModalityData[Any],
73 - **mm_processor_kwargs,
74 - ) -> MultiModalKwargs:
75 - """
76 - Return a dictionary to be passed as keyword arguments to
77 - :meth:`~torch.nn.Module.forward`. This is similar in concept to
78 - tokenizers and processors in HuggingFace Transformers.
79 -
80 - If the data is not supported, throw :exc:`TypeError`.
81 - """
82 - raise NotImplementedError
83 -
84 - def register_input_mapper(
85 - self,
86 - mapper: Optional[MultiModalInputMapper] = None,
87 - ):
88 - """
89 - Register an input mapper to a model class.
90 -
91 - When the model receives input data that matches the modality served by
92 - this plugin (see :meth:`get_data_key`), the provided function is
93 - invoked to transform the data into a dictionary of model inputs.
94 -
95 - If `None` is provided, then the default input mapper is used instead.
96 - """
97 -
98 - def wrapper(model_cls: N) -> N:
99 - if self._input_mappers.contains(model_cls, strict=True):
100 - logger.warning(
101 - "Model class %s already has an input mapper "
102 - "registered to %s. It is overwritten by the new one.",
103 - model_cls,
104 - self,
105 - )
106 -
107 - self._input_mappers[model_cls] = (mapper
108 - or self._default_input_mapper)
109 -
110 - return model_cls
111 -
112 - return wrapper
113 -
114 - def map_input(
115 - self,
116 - model_config: "ModelConfig",
117 - data: ModalityData[Any],
118 - mm_processor_kwargs: Optional[dict[str, Any]],
119 - ) -> MultiModalKwargs:
120 - """
121 - Transform the data into a dictionary of model inputs using the
122 - input mapper registered for that model.
123 -
124 - The model is identified by ``model_config``.
125 -
126 - Raises:
127 - TypeError: If the data type is not supported.
128 - """
129 -
130 - # Avoid circular import
131 - from vllm.model_executor.model_loader import get_model_architecture
132 -
133 - model_cls, _ = get_model_architecture(model_config)
134 -
135 - mapper = self._input_mappers.get(model_cls)
136 -
137 - if mapper is None:
138 - raise KeyError(f"No input mapper in {self} is registered for "
139 - f"model class {model_cls.__name__}.")
140 -
141 - if mm_processor_kwargs is None:
142 - mm_processor_kwargs = {}
143 -
144 - # In the case of the default mapper, we have to get resource
145 - # processor through its HuggingFace autoclass; since this goes
146 - # through **kwargs, we can't inspect it the same way, so we allow
147 - # drop mm_processor_kwargs based on signature inspection
148 - # if we're using the default mapper.
149 - #
150 - # This should be safe in general due to the sanitation, since the
151 - # transformers resource should filter unused kwargs anyway.
152 - uses_default_mapper = mapper == self._default_input_mapper
153 - mm_processor_kwargs = resolve_mm_processor_kwargs(
154 - model_config.mm_processor_kwargs,
155 - mm_processor_kwargs,
156 - callable=mapper,
157 - allow_var_kwargs=uses_default_mapper,
158 - )
159 - return mapper(InputContext(model_config), data, **mm_processor_kwargs)
160 -
161 - @abstractmethod
162 - def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
163 - """
164 - Calculate the maximum number of tokens, corresponding to a single
165 - instance of multimodal data, that are passed to the language model.
166 - """
167 - raise NotImplementedError
168 -
169 - def _validate_max_multimodal_tokens(self, max_mm_tokens: int):
170 - if max_mm_tokens < 1:
171 - raise ValueError("You should set the number of tokens to a "
172 - f"positive integer. Found: {max_mm_tokens}")
173 -
174 - def register_max_multimodal_tokens(
175 - self,
176 - max_mm_tokens: Optional[MultiModalTokensCalc] = None,
177 - ):
178 - """
179 - Register the maximum number of tokens, corresponding to a single
180 - instance of multimodal data, that are passed to the language model
181 - for a model class.
182 -
183 - If `None` is provided, then the default calculation is used instead.
184 - """
185 -
186 - def wrapper(model_cls: N) -> N:
187 - if self._max_mm_tokens.contains(model_cls, strict=True):
188 - logger.warning(
189 - "Model class %s already calculates maximum number of "
190 - "tokens in %s. It is overwritten by the new one.",
191 - model_cls,
192 - self,
193 - )
194 -
195 - if isinstance(max_mm_tokens, int):
196 - self._validate_max_multimodal_tokens(max_mm_tokens)
197 -
198 - self._max_mm_tokens[model_cls] = (
199 - max_mm_tokens or self._default_max_multimodal_tokens)
200 -
201 - return model_cls
202 -
203 - return wrapper
204 -
205 - def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
206 - """
207 - Get the maximum number of multi-modal tokens
208 - for profiling the memory usage of a model.
209 -
210 - If this registry is not applicable to the model, `0` is returned.
211 -
212 - The model is identified by ``model_config``.
213 - """
214 - # Avoid circular import
215 - from vllm.model_executor.model_loader import get_model_architecture
216 - from vllm.model_executor.models import supports_multimodal
217 -
218 - model_cls, _ = get_model_architecture(model_config)
219 -
220 - if not supports_multimodal(model_cls):
221 - return 0
222 -
223 - max_mm_tokens = self._max_mm_tokens.get(model_cls)
224 - if max_mm_tokens is None:
225 - return 0
226 -
227 - if callable(max_mm_tokens):
228 - mm_processor_kwargs = get_allowed_kwarg_only_overrides(
229 - max_mm_tokens,
230 - overrides=model_config.mm_processor_kwargs,
231 - requires_kw_only=False,
232 - allow_var_kwargs=True,
233 - )
234 - max_mm_tokens = max_mm_tokens(InputContext(model_config),
235 - **mm_processor_kwargs)
236 -
237 - self._validate_max_multimodal_tokens(max_mm_tokens)
238 -
239 - return max_mm_tokens
14
15
16 class MultiModalPlaceholderMap:
17 """
18 Relates multi-modal embeddings to their corresponding placeholders.
19 +
20 + Note: This is only used in V0.
21 """
22
23 class IndexMap(NamedTuple):
@@ -279,8 +55,7 @@ class MultiModalPlaceholderMap:
55 @classmethod
56 def from_seq_group(
57 cls, seq_group: "SequenceGroupMetadata", positions: range
282 - ) -> tuple[Optional[MultiModalDataDict], dict[str,
283 - "MultiModalPlaceholderMap"]]:
58 + ) -> tuple[MultiModalKwargs, dict[str, "MultiModalPlaceholderMap"]]:
59 """
60 Returns the multi-modal items that intersect with the portion of a
61 prompt (``seq_group``) represented by ``positions``, as well as a
@@ -323,48 +98,24 @@ class MultiModalPlaceholderMap:
98 seq_mm_placeholders = seq_group.multi_modal_placeholders
99
100 if not seq_mm_data or not seq_mm_placeholders:
326 - return seq_mm_data, {}
327 -
328 - # For merged processor, we directly use mm_kwargs as mm_data
329 - if isinstance(seq_mm_data, MultiModalKwargs):
330 - placeholder_maps = dict[str, MultiModalPlaceholderMap]()
331 -
332 - for modality, placeholders in seq_mm_placeholders.items():
333 - placeholder_map = MultiModalPlaceholderMap()
101 + return MultiModalKwargs({}), {}
102
335 - if positions:
336 - placeholder_map.append_items_from_seq_group(
337 - positions,
338 - # Dummy, since we don't care about intersecting items
339 - [None] * len(placeholders),
340 - placeholders,
341 - )
342 -
343 - placeholder_maps[modality] = placeholder_map
344 -
345 - return seq_mm_data, placeholder_maps
346 -
347 - mm_data = {**seq_mm_data}
348 - placeholder_maps = defaultdict[str, MultiModalPlaceholderMap](
349 - MultiModalPlaceholderMap)
103 + placeholder_maps = dict[str, MultiModalPlaceholderMap]()
104
105 for modality, placeholders in seq_mm_placeholders.items():
352 - mm_items = mm_data.pop(modality)
353 - if not isinstance(mm_items, list):
354 - mm_items = [mm_items]
106 + placeholder_map = MultiModalPlaceholderMap()
107
108 if positions:
357 - intersecting_items = placeholder_maps[modality] \
358 - .append_items_from_seq_group(
359 - positions,
360 - mm_items,
361 - placeholders,
362 - )
109 + placeholder_map.append_items_from_seq_group(
110 + positions,
111 + # Dummy, since we don't care about intersecting items
112 + [None] * len(placeholders),
113 + placeholders,
114 + )
115
364 - if intersecting_items:
365 - mm_data[modality] = intersecting_items
116 + placeholder_maps[modality] = placeholder_map
117
367 - return mm_data, placeholder_maps
118 + return seq_mm_data, placeholder_maps
119
120 def append_items_from_seq_group(
121 self,
@@ -445,8 +196,7 @@ class MultiModalPlaceholderMap:
196 f"The number of source ({len(src_indices)}) and destination "
197 f"indices ({len(dest_indices)}) must be the same.")
198
448 - return MultiModalPlaceholderMap.IndexMap(src=src_indices,
449 - dest=dest_indices)
199 + return self.IndexMap(src=src_indices, dest=dest_indices)
200
201
202 class MediaIO(ABC, Generic[_T]):
vllm/multimodal/image.py
+1 -79
@@ -3,89 +3,11 @@
3 import base64
4 from io import BytesIO
5 from pathlib import Path
6 -from typing import TYPE_CHECKING, Any, Optional
6
7 import torch
8 from PIL import Image
9
11 -from vllm.inputs.registry import InputContext
12 -from vllm.logger import init_logger
13 -from vllm.transformers_utils.processor import cached_get_image_processor
14 -from vllm.utils import is_list_of
15 -
16 -from .base import MediaIO, MultiModalPlugin
17 -from .inputs import ImageItem, ModalityData, MultiModalKwargs
18 -
19 -if TYPE_CHECKING:
20 - from vllm.config import ModelConfig
21 -
22 -logger = init_logger(__name__)
23 -
24 -
25 -class ImagePlugin(MultiModalPlugin):
26 - """Plugin for image data."""
27 -
28 - def get_data_key(self) -> str:
29 - return "image"
30 -
31 - def _get_hf_image_processor(
32 - self,
33 - model_config: "ModelConfig",
34 - mm_processor_kwargs: Optional[dict[str, Any]] = None,
35 - ):
36 - if mm_processor_kwargs is None:
37 - mm_processor_kwargs = {}
38 - return cached_get_image_processor(
39 - model_config.model,
40 - trust_remote_code=model_config.trust_remote_code,
41 - **mm_processor_kwargs)
42 -
43 - def _default_input_mapper(
44 - self,
45 - ctx: InputContext,
46 - data: ModalityData[ImageItem],
47 - **mm_processor_kwargs,
48 - ) -> MultiModalKwargs:
49 - model_config = ctx.model_config
50 -
51 - # PIL image
52 - if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
53 - image_processor = self._get_hf_image_processor(
54 - model_config,
55 - mm_processor_kwargs,
56 - )
57 -
58 - if image_processor is None:
59 - raise RuntimeError("No HuggingFace processor is available "
60 - "to process the image object")
61 - try:
62 - # NOTE: It may make sense to forward the mm_processor_kwargs
63 - # here too. For now, to keep it simple, we only allow it be
64 - # used for the initialization call though, just in case the
65 - # signatures of the preprocessor initializer don't match
66 - # preprocess()
67 - batch_data = image_processor \
68 - .preprocess(data, return_tensors="pt") \
69 - .data
70 - except Exception:
71 - logger.error(
72 - "Failed to process image (%s) with the default mapper. "
73 - "This is most likely an edge-case with this model's image "
74 - "processor in transformers (type: %s), and not vLLM.",
75 - data,
76 - type(image_processor).__name__)
77 - raise
78 -
79 - return MultiModalKwargs(batch_data)
80 -
81 - # Image embedding
82 - elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
83 - return MultiModalKwargs({"image_embeds": data})
84 -
85 - raise TypeError(f"Invalid image type: {type(data)}")
86 -
87 - def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
88 - return 3000
10 +from .base import MediaIO
11
12
13 def rescale_image_size(image: Image.Image,
vllm/multimodal/registry.py
+42 -233
@@ -1,13 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -
3 -import functools
4 -import json
5 -from collections import UserDict
6 -from collections.abc import Mapping, Sequence
2 +from collections.abc import Mapping
3 from dataclasses import dataclass
8 -from typing import TYPE_CHECKING, Any, Generic, Optional, Protocol, TypeVar
4 +from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar
5
6 import torch.nn as nn
7 +from typing_extensions import deprecated
8
9 from vllm.envs import VLLM_MM_INPUT_CACHE_GIB
10 from vllm.inputs import InputProcessingContext
@@ -16,15 +13,10 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer,
13 cached_tokenizer_from_config)
14 from vllm.utils import ClassRegistry
15
19 -from .audio import AudioPlugin
20 -from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
21 -from .image import ImagePlugin
22 -from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
16 from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
17 ProcessingCache)
18 from .profiling import (BaseDummyInputsBuilder, DummyDecoderData,
19 DummyEncoderData, MultiModalProfiler)
27 -from .video import VideoPlugin
20
21 if TYPE_CHECKING:
22 from vllm.config import ModelConfig
@@ -85,169 +77,23 @@ class _ProcessorFactories(Generic[_I]):
77 return self.processor(info, dummy_inputs_builder, cache=cache)
78
79
88 -class _MultiModalLimits(UserDict["ModelConfig", dict[str, int]]):
89 - """
90 - Wraps `_limits_by_model` for a more informative error message
91 - when attempting to access a model that does not exist.
92 - """
93 -
94 - def __getitem__(self, key: "ModelConfig") -> dict[str, int]:
95 - try:
96 - return super().__getitem__(key)
97 - except KeyError as exc:
98 - msg = (f"Cannot find `mm_limits` for model={key.model}. Did you "
99 - "forget to call `init_mm_limits_per_prompt`?")
100 - raise KeyError(msg) from exc
101 -
102 -
80 class MultiModalRegistry:
81 """
82 A registry that dispatches data processing according to the model.
83 """
84
108 - DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
109 -
110 - def __init__(
111 - self,
112 - *,
113 - plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
114 - self._plugins = {p.get_data_key(): p for p in plugins}
115 -
85 + def __init__(self) -> None:
86 self._processor_factories = ClassRegistry[nn.Module,
87 _ProcessorFactories]()
88
119 - # This is used for non-multimodal models
120 - self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
121 -
122 - self._limits_by_model = _MultiModalLimits()
123 -
89 self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_GIB)
90
126 - def register_plugin(self, plugin: MultiModalPlugin) -> None:
127 - """
128 - Register a multi-modal plugin so it can be recognized by vLLM.
129 - """
130 - data_type_key = plugin.get_data_key()
131 -
132 - if data_type_key in self._plugins:
133 - logger.warning(
134 - "A plugin is already registered for data type %s, "
135 - "and will be overwritten by the new plugin %s.", data_type_key,
136 - plugin)
137 -
138 - self._plugins[data_type_key] = plugin
139 -
140 - def _get_plugin(self, data_type_key: str):
141 - plugin = self._plugins.get(data_type_key)
142 - if plugin is not None:
143 - return plugin
144 -
145 - msg = f"Unknown multi-modal data type: {data_type_key}"
146 - raise NotImplementedError(msg)
147 -
148 - def register_input_mapper(
149 - self,
150 - data_type_key: str,
151 - mapper: Optional[MultiModalInputMapper] = None,
152 - ):
153 - """
154 - Register an input mapper for a specific modality to a model class.
155 -
156 - See :meth:`MultiModalPlugin.register_input_mapper` for more details.
157 - """
158 - return self._get_plugin(data_type_key).register_input_mapper(mapper)
159 -
160 - def register_image_input_mapper(
161 - self,
162 - mapper: Optional[MultiModalInputMapper] = None,
163 - ):
164 - """
165 - Register an input mapper for image data to a model class.
166 -
167 - See :meth:`MultiModalPlugin.register_input_mapper` for more details.
168 - """
169 - return self.register_input_mapper("image", mapper)
170 -
171 - def map_input(
172 - self,
173 - model_config: "ModelConfig",
174 - data: MultiModalDataDict,
175 - mm_processor_kwargs: Optional[dict[str, Any]] = None,
176 - ) -> MultiModalKwargs:
177 - """
178 - Apply an input mapper to the data passed to the model.
179 -
180 - The data belonging to each modality is passed to the corresponding
181 - plugin which in turn converts the data into into keyword arguments
182 - via the input mapper registered for that model.
183 -
184 - See :meth:`MultiModalPlugin.map_input` for more details.
185 -
186 - Note:
187 - This should be called after :meth:`init_mm_limits_per_prompt`.
188 - """
189 - merged_dict = dict[str, NestedTensors]()
190 -
191 - for data_key, data_value in data.items():
192 - plugin = self._get_plugin(data_key)
193 -
194 - num_items = len(data_value) if isinstance(data_value, list) else 1
195 - max_items = self._limits_by_model[model_config][data_key]
196 - if num_items > max_items:
197 - raise ValueError(
198 - f"You set '{json.dumps({data_key: max_items})}' (or "
199 - "defaulted to 1) in `--limit-mm-per-prompt`, but found "
200 - f"{num_items} items in the same prompt.")
201 -
202 - input_dict = plugin.map_input(model_config, data_value,
203 - mm_processor_kwargs)
204 - for input_key, input_tensor in input_dict.items():
205 - if input_key in merged_dict:
206 - raise ValueError(f"The input mappers (keys={set(data)}) "
207 - f"resulted in a conflicting keyword "
208 - f"argument to `forward()`: {input_key}")
209 -
210 - merged_dict[input_key] = input_tensor
211 -
212 - return MultiModalKwargs(merged_dict)
213 -
91 + @deprecated("Legacy input processor/mapper pipeline has been removed. "
92 + "Please update your model runner to use "
93 + "`seq_group_metadata.multi_modal_data` directly without "
94 + "further processing.")
95 def create_input_mapper(self, model_config: "ModelConfig"):
215 - """
216 - Create an input mapper (see :meth:`map_input`) for a specific model.
217 - """
218 - # NOTE - we currently make the assumption that if a model has multiple
219 - # supported modalities, they take the same kwargs. For the default,
220 - # this could be an issue in the future if it falls back to two HF
221 - # resources and we can't inspect the signature easily since it's
222 - # getting initialized through the autoclass.
223 - #
224 - # If this is a problem in the future, we should revisit it, but since
225 - # it potentially introduces a lot of complexity for a currently
226 - # uncommon case, we do not for simplicity of both use & implementation
227 - return functools.partial(self.map_input, model_config)
228 -
229 - def register_max_multimodal_tokens(
230 - self,
231 - data_type_key: str,
232 - max_mm_tokens: Optional[MultiModalTokensCalc] = None,
233 - ):
234 - """
235 - Register the maximum number of tokens, corresponding to a single
236 - instance of multimodal data belonging to a specific modality, that are
237 - passed to the language model for a model class.
238 - """
239 - return self._get_plugin(data_type_key) \
240 - .register_max_multimodal_tokens(max_mm_tokens)
241 -
242 - def register_max_image_tokens(
243 - self,
244 - max_mm_tokens: Optional[MultiModalTokensCalc] = None,
245 - ):
246 - """
247 - Register the maximum number of image tokens, corresponding to a single
248 - image, that are passed to the language model for a model class.
249 - """
250 - return self.register_max_multimodal_tokens("image", max_mm_tokens)
96 + return lambda data, mm_processor_kwargs: data
97
98 def get_max_tokens_per_item_by_modality(
99 self,
@@ -257,25 +103,22 @@ class MultiModalRegistry:
103 Get the maximum number of tokens per data item from each modality based
104 on underlying model configuration.
105 """
260 - if self.has_processor(model_config):
261 - processor = self.create_processor(model_config, disable_cache=True)
262 - profiler = MultiModalProfiler(processor)
263 -
264 - seq_len = model_config.max_model_len
265 - mm_limits = self.get_mm_limits_per_prompt(model_config)
266 -
267 - return profiler.get_mm_max_tokens(
268 - seq_len,
269 - {
270 - modality: 1
271 - for modality, limit in mm_limits.items() if limit > 0
272 - },
273 - )
106 + if not model_config.is_multimodal_model:
107 + return {}
108
275 - return {
276 - key: plugin.get_max_multimodal_tokens(model_config)
277 - for key, plugin in self._plugins.items()
278 - }
109 + processor = self.create_processor(model_config, disable_cache=True)
110 + profiler = MultiModalProfiler(processor)
111 +
112 + seq_len = model_config.max_model_len
113 + mm_limits = self.get_mm_limits_per_prompt(model_config)
114 +
115 + return profiler.get_mm_max_tokens(
116 + seq_len,
117 + {
118 + modality: 1
119 + for modality, limit in mm_limits.items() if limit > 0
120 + },
121 + )
122
123 def get_max_tokens_per_item_by_nonzero_modality(
124 self,
@@ -308,9 +151,6 @@ class MultiModalRegistry:
151 for profiling the memory usage of a model.
152
153 See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
311 -
312 - Note:
313 - This should be called after :meth:`init_mm_limits_per_prompt`.
154 """
155 mm_limits = self.get_mm_limits_per_prompt(model_config)
156
@@ -326,47 +166,18 @@ class MultiModalRegistry:
166 for profiling the memory usage of a model.
167
168 See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
329 -
330 - Note:
331 - This should be called after :meth:`init_mm_limits_per_prompt`.
169 """
170 return sum(self.get_max_tokens_by_modality(model_config).values())
171
172 + @deprecated("Legacy input processor/mapper pipeline has been removed. "
173 + "Please update your model runner to use "
174 + "`seq_group_metadata.multi_modal_data` directly without "
175 + "further processing.")
176 def init_mm_limits_per_prompt(
177 self,
178 model_config: "ModelConfig",
179 ) -> None:
339 - """
340 - Initialize the maximum number of multi-modal input instances for each
341 - modality that are allowed per prompt for a model class.
342 - """
343 - if model_config in self._limits_by_model:
344 - logger.warning(
345 - "`mm_limits` has already been set for model=%s, and will "
346 - "be overwritten by the new values.", model_config.model)
347 -
348 - multimodal_config = model_config.multimodal_config
349 - if multimodal_config is None:
350 - limits_per_plugin = self._disabled_limits_per_plugin
351 - else:
352 - config_limits_per_plugin = multimodal_config.limit_per_prompt
353 -
354 - extra_keys = config_limits_per_plugin.keys() - self._plugins.keys()
355 - if extra_keys:
356 - logger.warning(
357 - "Detected extra keys in `--limit-mm-per-prompt` which "
358 - "are not registered as multi-modal plugins: %s. "
359 - "They will be ignored.", extra_keys)
360 -
361 - # NOTE: Currently the default is set to 1 for each plugin
362 - # TODO: Automatically determine the limits based on budget
363 - # once more models support multi-image inputs
364 - limits_per_plugin = {
365 - key: multimodal_config.get_limit_per_prompt(key)
366 - for key in self._plugins
367 - }
368 -
369 - self._limits_by_model[model_config] = limits_per_plugin
180 + pass
181
182 def get_mm_limits_per_prompt(
183 self,
@@ -375,16 +186,13 @@ class MultiModalRegistry:
186 """
187 Get the maximum number of multi-modal input instances for each modality
188 that are allowed per prompt for a model class.
378 -
379 - Note:
380 - This should be called after :meth:`init_mm_limits_per_prompt`.
189 """
382 - if self.has_processor(model_config):
383 - processor = self.create_processor(model_config, disable_cache=True)
384 - profiler = MultiModalProfiler(processor)
385 - return profiler.get_mm_limits()
190 + if not model_config.is_multimodal_model:
191 + return {}
192
387 - return self._limits_by_model[model_config]
193 + processor = self.create_processor(model_config, disable_cache=True)
194 + profiler = MultiModalProfiler(processor)
195 + return profiler.get_mm_limits()
196
197 def register_processor(
198 self,
@@ -428,14 +236,12 @@ class MultiModalRegistry:
236 model_cls, _ = get_model_architecture(model_config)
237 return model_cls
238
239 + @deprecated("Legacy input processor/mapper pipeline has been removed. "
240 + "Please update your model runner to use "
241 + "`seq_group_metadata.multi_modal_data` directly without "
242 + "further processing.")
243 def has_processor(self, model_config: "ModelConfig") -> bool:
432 - """
433 - Test whether a multi-modal processor is defined for a specific model.
434 -
435 - See also:
436 - :ref:`mm-processing`
437 - """
438 - return self._get_model_cls(model_config) in self._processor_factories
244 + return True
245
246 def create_processor(
247 self,
@@ -450,6 +256,9 @@ class MultiModalRegistry:
256 See also:
257 :ref:`mm-processing`
258 """
259 + if not model_config.is_multimodal_model:
260 + raise ValueError(f"{model_config.model} is not a multimodal model")
261 +
262 if tokenizer is None:
263 tokenizer = cached_tokenizer_from_config(model_config)
264 if disable_cache is None:
vllm/multimodal/video.py
+2 -69
@@ -4,80 +4,13 @@ import base64
4 from functools import partial
5 from io import BytesIO
6 from pathlib import Path
7 -from typing import TYPE_CHECKING, Any, Optional
7
8 import numpy as np
9 import numpy.typing as npt
10 from PIL import Image
11
13 -from vllm.inputs.registry import InputContext
14 -from vllm.logger import init_logger
15 -from vllm.transformers_utils.processor import cached_get_video_processor
16 -from vllm.utils import is_list_of
17 -
18 -from .base import MediaIO, ModalityData
19 -from .image import ImageMediaIO, ImagePlugin
20 -from .inputs import MultiModalKwargs, VideoItem
21 -
22 -if TYPE_CHECKING:
23 - from vllm.config import ModelConfig
24 -
25 -logger = init_logger(__name__)
26 -
27 -
28 -class VideoPlugin(ImagePlugin):
29 - """Plugin for video data."""
30 -
31 - def get_data_key(self) -> str:
32 - return "video"
33 -
34 - def _get_hf_video_processor(
35 - self,
36 - model_config: "ModelConfig",
37 - mm_processor_kwargs: Optional[dict[str, Any]] = None,
38 - ):
39 - if mm_processor_kwargs is None:
40 - mm_processor_kwargs = {}
41 - return cached_get_video_processor(
42 - model_config.model,
43 - trust_remote_code=model_config.trust_remote_code,
44 - **mm_processor_kwargs)
45 -
46 - def _default_input_mapper(
47 - self,
48 - ctx: InputContext,
49 - data: ModalityData[VideoItem],
50 - **mm_processor_kwargs,
51 - ) -> MultiModalKwargs:
52 - model_config = ctx.model_config
53 -
54 - if isinstance(data, list) and len(data) == 1:
55 - data = data[0] # type: ignore
56 -
57 - if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
58 - video_processor = self._get_hf_video_processor(
59 - model_config,
60 - mm_processor_kwargs,
61 - )
62 - if video_processor is None:
63 - raise RuntimeError("No HuggingFace processor is available "
64 - "to process the video object")
65 - try:
66 - # NOTE: Similar to image; it may be a good idea to filter and
67 - # pass mm_processor_kwargs here too, but for now we don't to
68 - # avoid extra complexity if the initializer and preprocess
69 - # signatures of the processor don't align
70 - batch_data = video_processor(data, return_tensors="pt").data
71 - except Exception:
72 - logger.error("Failed to process video (%s)", data)
73 - raise
74 -
75 - return MultiModalKwargs(batch_data)
76 -
77 - raise TypeError(f"Invalid video type: {type(data)}")
78 -
79 - def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
80 - return 4096
12 +from .base import MediaIO
13 +from .image import ImageMediaIO
14
15
16 def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
vllm/sequence.py
+17 -30
@@ -14,9 +14,9 @@ from typing import Any, Callable, Optional, Union
14 import msgspec
15 import torch
16
17 -from vllm.inputs import SingletonInputs, SingletonInputsAdapter
17 +from vllm.inputs import SingletonInputs
18 from vllm.lora.request import LoRARequest
19 -from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
19 +from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
20 from vllm.pooling_params import PoolingParams
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sampling_params import RequestOutputKind, SamplingParams
@@ -419,7 +419,7 @@ class Sequence:
419 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
420 ) -> None:
421 self.seq_id = seq_id
422 - self.inputs = SingletonInputsAdapter(inputs)
422 + self.inputs = inputs
423 self.block_size = block_size
424 self.eos_token_id = eos_token_id
425 self.lora_request = lora_request
@@ -448,31 +448,29 @@ class Sequence:
448
449 @property
450 def prompt(self) -> Optional[str]:
451 - return self.inputs.prompt
451 + return self.inputs.get("prompt")
452
453 @property
454 def prompt_token_ids(self) -> list[int]:
455 - return self.inputs.prompt_token_ids
456 -
457 - @property
458 - def prompt_embeds(self) -> Optional[torch.Tensor]:
459 - return self.inputs.prompt_embeds
455 + return self.inputs["prompt_token_ids"]
456
457 @property
458 def token_type_ids(self) -> list[int]:
463 - return self.inputs.token_type_ids
459 + return self.inputs.get("token_type_ids", [])
460
461 @property
466 - def multi_modal_data(self) -> "MultiModalDataDict":
467 - return self.inputs.multi_modal_data
462 + def multi_modal_data(self) -> MultiModalKwargs:
463 + if self.inputs["type"] == "multimodal":
464 + return self.inputs["mm_kwargs"]
465 +
466 + return MultiModalKwargs({})
467
468 @property
469 def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
471 - return self.inputs.multi_modal_placeholders
470 + if self.inputs["type"] == "multimodal":
471 + return self.inputs["mm_placeholders"]
472
473 - @property
474 - def mm_processor_kwargs(self) -> dict[str, Any]:
475 - return self.inputs.mm_processor_kwargs
473 + return {}
474
475 @property
476 def lora_int_id(self) -> int:
@@ -723,12 +721,12 @@ class SequenceGroup:
721 return self.first_seq.token_type_ids
722
723 @property
726 - def multi_modal_data(self) -> MultiModalDataDict:
724 + def multi_modal_data(self) -> MultiModalKwargs:
725 if self.first_seq.multi_modal_data:
726 return self.first_seq.multi_modal_data
727 elif self.encoder_seq is not None:
728 return self.encoder_seq.multi_modal_data
731 - return {}
729 + return MultiModalKwargs({})
730
731 @property
732 def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
@@ -738,14 +736,6 @@ class SequenceGroup:
736 return self.encoder_seq.multi_modal_placeholders
737 return {}
738
741 - @property
742 - def mm_processor_kwargs(self) -> dict[str, Any]:
743 - if self.first_seq.multi_modal_data:
744 - return self.first_seq.mm_processor_kwargs
745 - elif self.encoder_seq is not None:
746 - return self.encoder_seq.mm_processor_kwargs
747 - return {}
748 -
739 @property
740 def lora_int_id(self) -> int:
741 return self.lora_request.lora_int_id if self.lora_request else 0
@@ -969,12 +959,9 @@ class SequenceGroupMetadata(
959 computed_block_nums: Optional[list[int]] = None
960 state: Optional[SequenceGroupState] = msgspec.field(
961 default_factory=lambda: SequenceGroupState())
972 - # "MultiModalDataDict" types. We have to use Any due to msgspec
973 - # doesn't allow to have union of 2 different dicts.
962 token_type_ids: Optional[list[int]] = None
975 - multi_modal_data: Optional[Any] = None
963 + multi_modal_data: Optional[MultiModalKwargs] = None
964 multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
977 - mm_processor_kwargs: Optional[dict[str, Any]] = None
965 encoder_seq_data: Optional[SequenceData] = None
966 cross_block_table: Optional[list[int]] = None
967 prompt_adapter_request: Optional[PromptAdapterRequest] = None
vllm/transformers_utils/processor.py
-35
@@ -208,38 +208,3 @@ def cached_image_processor_from_config(
208 trust_remote_code=model_config.trust_remote_code,
209 **_merge_mm_kwargs(model_config, **kwargs),
210 )
211 -
212 -
213 -def get_video_processor(
214 - processor_name: str,
215 - *args: Any,
216 - trust_remote_code: bool = False,
217 - **kwargs: Any,
218 -):
219 - """Load a video processor for the given model name via HuggingFace."""
220 - # don't put this import at the top level
221 - # it will call torch.cuda.device_count()
222 - from transformers.image_processing_utils import BaseImageProcessor
223 -
224 - processor = get_processor(
225 - processor_name,
226 - *args,
227 - trust_remote_code=trust_remote_code,
228 - **kwargs,
229 - )
230 -
231 - return cast(BaseImageProcessor, processor.video_processor)
232 -
233 -
234 -cached_get_video_processor = lru_cache(get_video_processor)
235 -
236 -
237 -def cached_video_processor_from_config(
238 - model_config: "ModelConfig",
239 - **kwargs: Any,
240 -):
241 - return cached_get_video_processor(
242 - model_config.model,
243 - trust_remote_code=model_config.trust_remote_code,
244 - **_merge_mm_kwargs(model_config, **kwargs),
245 - )
vllm/worker/cpu_model_runner.py
+5 -20
@@ -22,8 +22,8 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
22 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23 from vllm.model_executor.model_loader import get_model
24 from vllm.model_executor.models import supports_lora, supports_multimodal
25 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
26 - MultiModalKwargs, MultiModalPlaceholderMap)
25 +from vllm.multimodal import (BatchedTensorInputs, MultiModalKwargs,
26 + MultiModalPlaceholderMap)
27 from vllm.sequence import (IntermediateTensors, SequenceData,
28 SequenceGroupMetadata)
29 from vllm.worker.model_runner_base import (
@@ -154,7 +154,6 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
154 self.sliding_window = self.runner.sliding_window
155 self.block_size = self.runner.block_size
156 self.device = self.runner.device
157 - self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
157 self.enable_lora = self.runner.lora_config is not None
158 if self.runner.attn_backend is not None:
159 # spec decode (e.g. Medusa) does not have atten backend
@@ -359,22 +358,14 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
358 computed_len = seq_data.get_num_computed_tokens()
359 seq_len = self.input_data.seq_lens[-1]
360
362 - # NOTE: mm_data only includes the subset of multi-modal items that
361 + # NOTE: mm_kwargs only includes the subset of multi-modal items that
362 # intersect with the current prefill positions.
364 - mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
363 + mm_kwargs, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
364 seq_group_metadata, range(computed_len, seq_len))
365
367 - if not mm_data:
366 + if not mm_kwargs:
367 return
368
370 - if self.runner.mm_registry.has_processor(self.runner.model_config):
371 - mm_kwargs = mm_data
372 - else:
373 - mm_kwargs = self.multi_modal_input_mapper(
374 - mm_data,
375 - seq_group_metadata.mm_processor_kwargs,
376 - )
377 -
369 # special processing for mrope position deltas.
370 if self.runner.model_config.uses_mrope:
371 assert not self.chunked_prefill, \
@@ -480,12 +471,6 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
471 use_mla=self.model_config.use_mla,
472 ) if needs_attn_backend else None
473
483 - # Multi-modal data support
484 - self.mm_registry = MULTIMODAL_REGISTRY
485 - self.multi_modal_input_mapper = self.mm_registry \
486 - .create_input_mapper(self.model_config)
487 - self.mm_registry.init_mm_limits_per_prompt(self.model_config)
488 -
474 # Lazy initialization.
475 self.model: nn.Module # Set after init_Model
476 # Set after load_model.
vllm/worker/enc_dec_model_runner.py
+2
@@ -100,6 +100,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
100 vllm_config=vllm_config,
101 kv_cache_dtype=kv_cache_dtype,
102 is_driver_worker=is_driver_worker,
103 + input_registry=input_registry,
104 + mm_registry=mm_registry,
105 )
106
107 # Crash for unsupported encoder/scenarios
vllm/worker/hpu_model_runner.py
+3 -9
@@ -45,8 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
45 VocabParallelEmbedding)
46 from vllm.model_executor.model_loader import get_model
47 from vllm.model_executor.sampling_metadata import SequenceGroupToSample
48 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
49 - MultiModalKwargs)
48 +from vllm.multimodal import BatchedTensorInputs, MultiModalKwargs
49 from vllm.sampling_params import SamplingParams
50 from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
51 Logprob, SequenceData, SequenceGroupMetadata,
@@ -545,10 +544,6 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
544 ]
545 gc.set_threshold(*requested_gc_thrs)
546
548 - # Multi-modal data support
549 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
550 - .create_input_mapper(self.model_config)
551 -
547 self.skip_warmup = os.environ.get('VLLM_SKIP_WARMUP',
548 'false').lower() == 'true'
549
@@ -731,9 +726,8 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
726 # is always the first token in the sequence.
727 input_positions.append(list(range(context_len, seq_len)))
728
734 - mm_data = seq_group_metadata.multi_modal_data
735 - if mm_data:
736 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
729 + mm_kwargs = seq_group_metadata.multi_modal_data
730 + if mm_kwargs:
731 multi_modal_kwargs_list.append(mm_kwargs)
732
733 if seq_group_metadata.block_tables is None:
vllm/worker/model_runner.py
+5 -17
@@ -457,7 +457,6 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
457 self.enable_lora = self.runner.lora_config is not None
458 self.enable_prompt_adapter = (self.runner.prompt_adapter_config
459 is not None)
460 - self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
460
461 # Attention metadata inputs.
462 if self.attn_backend is not None:
@@ -675,23 +674,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
674 def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
675 seq_group_metadata: SequenceGroupMetadata):
676 """If multi-modal data is given, add it to the input."""
678 - # NOTE: mm_data only includes the subset of multi-modal items that
677 + # NOTE: mm_kwargs only includes the subset of multi-modal items that
678 # intersect with the current prefill positions.
679 positions = inter_data.input_positions[0]
681 - mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
680 + mm_kwargs, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
681 seq_group_metadata,
682 range(positions[0], positions[0] + len(positions)))
684 - if not mm_data:
683 + if not mm_kwargs:
684 return
685
687 - if self.runner.mm_registry.has_processor(self.runner.model_config):
688 - mm_kwargs = mm_data
689 - else:
690 - mm_kwargs = self.multi_modal_input_mapper(
691 - mm_data,
692 - seq_group_metadata.mm_processor_kwargs,
693 - )
694 -
686 inter_data.multi_modal_kwargs = mm_kwargs
687 inter_data.multi_modal_placeholder_maps = placeholder_maps
688
@@ -1085,9 +1076,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1076 # Multi-modal data support
1077 self.input_registry = input_registry
1078 self.mm_registry = mm_registry
1088 - self.multi_modal_input_mapper = mm_registry \
1089 - .create_input_mapper(model_config)
1090 - self.mm_registry.init_mm_limits_per_prompt(self.model_config)
1079
1080 # Lazy initialization
1081 self.model: nn.Module # Set after load_model
@@ -1327,8 +1315,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1315
1316 dummy_data = self.input_registry \
1317 .dummy_data_for_profiling(self.model_config,
1330 - seq_len,
1331 - self.mm_registry)
1318 + seq_len,
1319 + self.mm_registry)
1320
1321 seq = SequenceGroupMetadata(
1322 request_id=str(group_id),
vllm/worker/neuron_model_runner.py
+3 -17
@@ -15,8 +15,7 @@ from vllm.logger import init_logger
15 from vllm.model_executor import SamplingMetadata
16 from vllm.model_executor.layers.sampler import SamplerOutput
17 from vllm.model_executor.model_loader.neuron import get_neuron_model
18 -from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19 - MultiModalKwargs)
18 +from vllm.multimodal import BatchedTensorInputs, MultiModalKwargs
19 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
20 from vllm.utils import is_pin_memory_available, make_tensor_with_pad
21 from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
@@ -69,11 +68,6 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
68 self.device = self.device_config.device
69 self.pin_memory = is_pin_memory_available()
70
72 - # Multi-modal data support
73 - self.mm_registry = MULTIMODAL_REGISTRY
74 - self.multi_modal_input_mapper = self.mm_registry \
75 - .create_input_mapper(self.model_config)
76 -
71 # Lazy initialization.
72 self.model: nn.Module # initialize after load_model.
73
@@ -149,16 +143,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
143 assert len(block_table) == 1
144 input_block_ids.append(block_table[0])
145
152 - mm_data = seq_group_metadata.multi_modal_data
153 - if mm_data:
154 - if self.mm_registry.has_processor(self.model_config):
155 - mm_kwargs = mm_data
156 - else:
157 - mm_kwargs = self.multi_modal_input_mapper(
158 - mm_data,
159 - seq_group_metadata.mm_processor_kwargs,
160 - )
161 -
146 + mm_kwargs = seq_group_metadata.multi_modal_data
147 + if mm_kwargs:
148 multi_modal_kwargs_list.append(mm_kwargs)
149
150 max_seq_len = max(seq_lens)
vllm/worker/xpu_model_runner.py
+2 -14
@@ -188,20 +188,11 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
188 input_positions.extend(list(positions_range))
189
190 if seq_group_metadata.multi_modal_data:
191 - # NOTE: mm_data only includes the subset of multi-modal items
191 + # NOTE: mm_kwargs only includes the subset of multi-modal items
192 # that intersect with the current prefill positions.
193 - mm_data, placeholder_maps = MultiModalPlaceholderMap \
193 + mm_kwargs, placeholder_maps = MultiModalPlaceholderMap \
194 .from_seq_group(seq_group_metadata, positions_range)
195
196 - if self.runner.mm_registry.has_processor(
197 - self.runner.model_config):
198 - mm_kwargs = mm_data
199 - else:
200 - mm_kwargs = self.runner.multi_modal_input_mapper(
201 - mm_data,
202 - seq_group_metadata.mm_processor_kwargs,
203 - )
204 -
196 multi_modal_kwargs_list.append(mm_kwargs)
197
198 for modality, placeholder_map in placeholder_maps.items():
@@ -404,9 +395,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
395 # Multi-modal data support
396 self.input_registry = input_registry
397 self.mm_registry = mm_registry
407 - self.multi_modal_input_mapper = mm_registry \
408 - .create_input_mapper(model_config)
409 - self.mm_registry.init_mm_limits_per_prompt(self.model_config)
398
399 # Lazy initialization.
400 self.model: nn.Module # Set after init_Model