@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b8bb86bf1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[1/N] Initial prototype for multi-modal processor (#10044)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 13, 2024 at 20:39 UTC 0b8bb86bf19d68950b4d92a99350e07a26ec0d2c
48 files changed +1133 -437
docs/source/models/enabling_multimodal_inputs.rst
+1 -1
@@ -66,7 +66,7 @@ A default mapper is available for each modality in the core vLLM library. This i
66 3. Register maximum number of multi-modal tokens
67 ------------------------------------------------
68
69 -For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data instance
69 +For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data item
70 and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
71
72 .. code-block:: diff
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen.py
+1 -1
@@ -6,7 +6,7 @@ import torch
6 from PIL.Image import Image
7
8 from vllm.inputs import InputContext, token_inputs
9 -from vllm.multimodal.base import MultiModalKwargs
9 +from vllm.multimodal import MultiModalKwargs
10 from vllm.multimodal.utils import cached_get_tokenizer
11
12 from .....conftest import IMAGE_ASSETS
tests/multimodal/test_inputs.py renamed
+1 -1
@@ -1,6 +1,6 @@
1 import torch
2
3 -from vllm.multimodal.base import MultiModalKwargs, NestedTensors
3 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
4
5
6 def assert_nested_tensors_equal(expected: NestedTensors,
tests/multimodal/test_processor_kwargs.py
+23 -14
@@ -1,12 +1,12 @@
1 from array import array
2 -from typing import Mapping
2 +from typing import Callable, Dict, Mapping, Optional
3 from unittest.mock import patch
4
5 import pytest
6 import torch
7
8 from vllm.inputs import (DecoderOnlyInputs, DummyData, InputContext,
9 - InputRegistry, token_inputs)
9 + InputRegistry, ProcessorInputs, token_inputs)
10 from vllm.multimodal import MultiModalRegistry
11 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
12
@@ -34,10 +34,9 @@ def use_processor_mock():
34 inputs: DecoderOnlyInputs,
35 *,
36 num_crops=DEFAULT_NUM_CROPS):
37 - # For testing purposes, we don't worry about the llm inputs / return
38 - # type validation, and just return the value of the kwarg that we
39 - # clobber.
40 - return num_crops
37 + # For testing purposes, we don't worry about the prompt
38 + return token_inputs(prompt_token_ids=[],
39 + mm_processor_kwargs={"num_crops": num_crops})
40
41 with patch("vllm.inputs.registry.InputRegistry._get_model_input_processor",
42 return_value=custom_processor):
@@ -109,6 +108,21 @@ def _get_num_crops_info(init_num_crops: int, inference_num_crops: int):
108 return init_kwargs, inference_kwargs, expected_seq_count
109
110
111 +def _get_processed_num_crops(
112 + processor: Callable[[ProcessorInputs], ProcessorInputs],
113 + inference_kwargs: Optional[Dict[str, int]],
114 +) -> int:
115 + processed_inputs = processor(
116 + token_inputs(prompt_token_ids=[],
117 + prompt="",
118 + mm_processor_kwargs=inference_kwargs))
119 +
120 + assert "type" in processed_inputs
121 + assert processed_inputs["type"] == "token"
122 + assert "mm_processor_kwargs" in processed_inputs
123 + return processed_inputs["mm_processor_kwargs"]["num_crops"]
124 +
125 +
126 @pytest.mark.parametrize("init_num_crops,inference_num_crops", [
127 (None, None),
128 (NUM_CROPS_OVERRIDE, None),
@@ -124,10 +138,8 @@ def test_input_processor_kwargs(use_processor_mock, init_num_crops,
138
139 ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
140 processor = dummy_registry.create_input_processor(ctx.model_config)
127 - num_crops_val = processor(
128 - token_inputs(prompt_token_ids=[],
129 - prompt="",
130 - mm_processor_kwargs=inference_kwargs))
141 + num_crops_val = _get_processed_num_crops(processor, inference_kwargs)
142 +
143 assert num_crops_val == expected_seq_count
144
145
@@ -153,10 +165,7 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
165
166 processor = dummy_registry.create_input_processor(ctx.model_config)
167 # Should filter out the inference time kwargs
156 - num_crops_val = processor(
157 - token_inputs(prompt_token_ids=[],
158 - prompt="",
159 - mm_processor_kwargs=mm_processor_kwargs))
168 + num_crops_val = _get_processed_num_crops(processor, mm_processor_kwargs)
169 assert num_crops_val == DEFAULT_NUM_CROPS
170
171
tests/v1/core/test_prefix_caching.py
+2 -2
@@ -1,5 +1,5 @@
1 """Compare the with and without prefix caching."""
2 -from vllm.inputs import DecoderOnlyInputs
2 +from vllm.inputs import token_inputs
3 from vllm.sampling_params import SamplingParams
4 from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
5 from vllm.v1.core.kv_cache_utils import hash_block_tokens
@@ -8,7 +8,7 @@ from vllm.v1.core.kv_cache_utils import hash_block_tokens
8 def make_request(request_id, prompt_token_ids):
9 return Request(
10 request_id=request_id,
11 - inputs=DecoderOnlyInputs(prompt_token_ids=prompt_token_ids),
11 + inputs=token_inputs(prompt_token_ids=prompt_token_ids),
12 sampling_params=SamplingParams(max_tokens=17),
13 eos_token_id=100,
14 arrival_time=0,
vllm/config.py
+1 -1
@@ -107,7 +107,7 @@ class ModelConfig:
107 matches the model name exposed via the APIs. If multiple model
108 names provided, the first name will be used. If not specified,
109 the model name will be the same as `model`.
110 - limit_mm_per_prompt: Maximum number of data instances per modality
110 + limit_mm_per_prompt: Maximum number of data items per modality
111 per prompt. Only applicable for multimodal models.
112 override_neuron_config: Initialize non default neuron config or
113 override default neuron config that are specific to Neuron devices,
vllm/engine/async_llm_engine.py
+4
@@ -19,6 +19,7 @@ from vllm.executor.executor_base import ExecutorAsyncBase
19 from vllm.executor.gpu_executor import GPUExecutorAsync
20 from vllm.executor.ray_utils import initialize_ray_cluster
21 from vllm.inputs import PromptType
22 +from vllm.inputs.preprocess import InputPreprocessor
23 from vllm.logger import init_logger
24 from vllm.lora.request import LoRARequest
25 from vllm.model_executor.guided_decoding import (
@@ -729,6 +730,9 @@ class AsyncLLMEngine(EngineClient):
730 self.set_errored(exc)
731 self._request_tracker.propagate_exception(exc)
732
733 + async def get_input_preprocessor(self) -> InputPreprocessor:
734 + return self.engine.input_preprocessor
735 +
736 async def get_tokenizer(
737 self,
738 lora_request: Optional[LoRARequest] = None,
vllm/engine/llm_engine.py
+6 -10
@@ -30,7 +30,7 @@ from vllm.executor.executor_base import ExecutorBase
30 from vllm.executor.gpu_executor import GPUExecutor
31 from vllm.executor.ray_utils import initialize_ray_cluster
32 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
33 - PromptType)
33 + PromptType, SingletonInputsAdapter)
34 from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
35 from vllm.inputs.preprocess import InputPreprocessor
36 from vllm.logger import init_logger
@@ -39,6 +39,7 @@ from vllm.lora.request import LoRARequest
39 from vllm.model_executor.guided_decoding import (
40 get_local_guided_decoding_logits_processor)
41 from vllm.model_executor.layers.sampler import SamplerOutput
42 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
43 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
44 RequestOutputFactory)
45 from vllm.pooling_params import PoolingParams
@@ -226,6 +227,7 @@ class LLMEngine:
227 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
228 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
229 input_registry: InputRegistry = INPUT_REGISTRY,
230 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
231 use_cached_outputs: bool = False,
232 ) -> None:
233
@@ -335,7 +337,8 @@ class LLMEngine:
337 model_config)
338
339 self.input_preprocessor = InputPreprocessor(model_config,
338 - self.tokenizer)
340 + self.tokenizer,
341 + mm_registry)
342
343 self.input_registry = input_registry
344 self.input_processor = input_registry.create_input_processor(
@@ -851,13 +854,6 @@ class LLMEngine:
854 )
855 processed_inputs = self.input_processor(preprocessed_inputs)
856
854 - # This is a bit of a hack - copy the mm_processor_kwargs that were
855 - # used in the input processor to the processed output, since these
856 - # kwargs are presumed to be immutable and the values should be aligned
857 - # between the input processor (here) and the input mapper.
858 - processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
859 - "mm_processor_kwargs")
860 -
857 self._add_processed_request(
858 request_id=request_id,
859 processed_inputs=processed_inputs,
@@ -2019,7 +2015,7 @@ class LLMEngine:
2015 else:
2016 prompt_inputs = inputs
2017
2022 - prompt_ids = prompt_inputs.get("prompt_token_ids")
2018 + prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids
2019
2020 if prompt_ids is None or len(prompt_ids) == 0:
2021 raise ValueError("Prompt cannot be empty")
vllm/engine/multiprocessing/client.py
+6
@@ -31,6 +31,7 @@ from vllm.engine.protocol import EngineClient
31 # yapf: enable
32 from vllm.envs import VLLM_RPC_TIMEOUT
33 from vllm.inputs import PromptType
34 +from vllm.inputs.preprocess import InputPreprocessor
35 from vllm.logger import init_logger
36 from vllm.lora.request import LoRARequest
37 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -94,6 +95,8 @@ class MQLLMEngineClient(EngineClient):
95 parallel_config=engine_config.parallel_config,
96 enable_lora=bool(engine_config.lora_config),
97 )
98 + self.input_preprocessor = InputPreprocessor(self.model_config,
99 + self.tokenizer)
100
101 # Send RPCGenerateRequest to the MQLLMEngine.
102 self.input_socket: Socket = self.context.socket(zmq.constants.PUSH)
@@ -345,6 +348,9 @@ class MQLLMEngineClient(EngineClient):
348 or response != VLLM_RPC_SUCCESS_STR):
349 raise ValueError(error_message)
350
351 + async def get_input_preprocessor(self) -> InputPreprocessor:
352 + return self.input_preprocessor
353 +
354 async def get_tokenizer(self, lora_request: Optional[LoRARequest] = None):
355 return await self.tokenizer.get_lora_tokenizer_async(lora_request)
356
vllm/engine/protocol.py
+11 -5
@@ -62,7 +62,6 @@ class EngineClient(ABC):
62 async def beam_search(
63 self,
64 prompt: PromptType,
65 - model_config: ModelConfig,
65 request_id: str,
66 params: BeamSearchParams,
67 ) -> AsyncGenerator[RequestOutput, None]:
@@ -74,13 +73,14 @@ class EngineClient(ABC):
73 length_penalty = params.length_penalty
74 include_stop_str_in_output = params.include_stop_str_in_output
75
77 - tokenizer = await self.get_tokenizer()
78 - input_preprocessor = InputPreprocessor(model_config, tokenizer)
76 + preprocessor = await self.get_input_preprocessor()
77 + tokenizer_group = preprocessor.get_tokenizer_group()
78 + tokenizer = await tokenizer_group.get_lora_tokenizer_async()
79
80 if is_explicit_encoder_decoder_prompt(prompt):
81 raise NotImplementedError
82 else:
83 - processed_inputs = input_preprocessor._prompt_to_llm_inputs(
83 + processed_inputs = preprocessor._prompt_to_llm_inputs(
84 prompt,
85 request_id=request_id,
86 )
@@ -220,6 +220,7 @@ class EngineClient(ABC):
220 Args:
221 request_id: The unique id of the request.
222 """
223 + ...
224
225 @abstractmethod
226 async def get_model_config(self) -> ModelConfig:
@@ -228,8 +229,13 @@ class EngineClient(ABC):
229
230 @abstractmethod
231 async def get_decoding_config(self) -> DecodingConfig:
231 - ...
232 """Get the decoding configuration of the vLLM engine."""
233 + ...
234 +
235 + @abstractmethod
236 + async def get_input_preprocessor(self) -> InputPreprocessor:
237 + """Get the input processor of the vLLM engine."""
238 + ...
239
240 @abstractmethod
241 async def get_tokenizer(
vllm/entrypoints/openai/serving_chat.py
-1
@@ -190,7 +190,6 @@ class OpenAIServingChat(OpenAIServing):
190 if isinstance(sampling_params, BeamSearchParams):
191 generator = self.engine_client.beam_search(
192 prompt=engine_prompt,
193 - model_config=self.model_config,
193 request_id=request_id,
194 params=sampling_params,
195 )
vllm/entrypoints/openai/serving_completion.py
-1
@@ -140,7 +140,6 @@ class OpenAIServingCompletion(OpenAIServing):
140 if isinstance(sampling_params, BeamSearchParams):
141 generator = self.engine_client.beam_search(
142 prompt=engine_prompt,
143 - model_config=self.model_config,
143 request_id=request_id,
144 params=sampling_params,
145 )
vllm/inputs/__init__.py
+8 -4
@@ -1,9 +1,11 @@
1 from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
2 ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType,
3 - SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
4 - TokensPrompt, build_explicit_enc_dec_prompt,
5 - to_enc_dec_tuple_list, token_inputs, zip_enc_dec_prompts)
6 -from .registry import DummyData, InputContext, InputRegistry
3 + SingletonInputs, SingletonInputsAdapter, SingletonPrompt,
4 + TextPrompt, TokenInputs, TokensPrompt,
5 + build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
6 + token_inputs, zip_enc_dec_prompts)
7 +from .registry import (DummyData, InputContext, InputProcessingContext,
8 + InputRegistry)
9
10 INPUT_REGISTRY = InputRegistry()
11 """
@@ -26,12 +28,14 @@ __all__ = [
28 "EncoderDecoderInputs",
29 "ProcessorInputs",
30 "SingletonInputs",
31 + "SingletonInputsAdapter",
32 "build_explicit_enc_dec_prompt",
33 "to_enc_dec_tuple_list",
34 "zip_enc_dec_prompts",
35 "INPUT_REGISTRY",
36 "DummyData",
37 "InputContext",
38 + "InputProcessingContext",
39 "InputRegistry",
40 ]
41
vllm/inputs/data.py
+88 -11
@@ -1,10 +1,14 @@
1 +from dataclasses import dataclass
2 +from functools import cached_property
3 from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List, Literal,
4 Optional, Tuple, Union, cast)
5
4 -from typing_extensions import NotRequired, TypedDict, TypeVar
6 +import torch
7 +from typing_extensions import NotRequired, TypedDict, TypeVar, assert_never
8
9 if TYPE_CHECKING:
10 from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
11 + from vllm.multimodal.inputs import MultiModalInputsV2
12
13
14 class TextPrompt(TypedDict):
@@ -36,13 +40,13 @@ class TokensPrompt(TypedDict):
40
41 multi_modal_data: NotRequired["MultiModalDataDict"]
42 """
39 - Optional multi-modal data to pass to the model,
43 + DEPRECATED: Optional multi-modal data to pass to the model,
44 if the model supports it.
45 """
46
47 mm_processor_kwargs: NotRequired[Dict[str, Any]]
48 """
45 - Optional multi-modal processor kwargs to be forwarded to the
49 + DEPRECATED: Optional multi-modal processor kwargs to be forwarded to the
50 multimodal input mapper & processor. Note that if multiple modalities
51 have registered mappers etc for the model being considered, we attempt
52 to pass the mm_processor_kwargs to each of them.
@@ -176,7 +180,7 @@ def token_inputs(
180 return inputs
181
182
179 -DecoderOnlyInputs = TokenInputs
183 +DecoderOnlyInputs = Union[TokenInputs, "MultiModalInputsV2"]
184 """
185 The inputs in :class:`~vllm.LLMEngine` before they are
186 passed to the model executor.
@@ -191,19 +195,91 @@ class EncoderDecoderInputs(TypedDict):
195
196 This specifies the required data for encoder-decoder models.
197 """
194 - encoder: TokenInputs
198 + encoder: Union[TokenInputs, "MultiModalInputsV2"]
199 """The inputs for the encoder portion."""
200
197 - decoder: TokenInputs
201 + decoder: Union[TokenInputs, "MultiModalInputsV2"]
202 """The inputs for the decoder portion."""
203
204
201 -SingletonInputs = TokenInputs
205 +SingletonInputs = Union[TokenInputs, "MultiModalInputsV2"]
206 """
207 A processed :class:`SingletonPrompt` which can be passed to
208 :class:`vllm.sequence.Sequence`.
209 """
210
211 +
212 +@dataclass
213 +class SingletonInputsAdapter:
214 + """
215 + Unified interface to access the components of :class:`SingletonInputs`.
216 + """
217 + inputs: SingletonInputs
218 +
219 + @cached_property
220 + def prompt(self) -> Optional[str]:
221 + inputs = self.inputs
222 +
223 + if inputs["type"] == "token" or inputs["type"] == "multimodal":
224 + return inputs.get("prompt")
225 +
226 + assert_never(inputs)
227 +
228 + @cached_property
229 + def prompt_token_ids(self) -> List[int]:
230 + inputs = self.inputs
231 +
232 + if inputs["type"] == "token" or inputs["type"] == "multimodal":
233 + return inputs.get("prompt_token_ids", [])
234 +
235 + assert_never(inputs)
236 +
237 + @cached_property
238 + def prompt_embeds(self) -> Optional[torch.Tensor]:
239 + inputs = self.inputs
240 +
241 + if inputs["type"] == "token" or inputs["type"] == "multimodal":
242 + return None
243 +
244 + assert_never(inputs)
245 +
246 + @cached_property
247 + def multi_modal_data(self) -> "MultiModalDataDict":
248 + inputs = self.inputs
249 +
250 + if inputs["type"] == "token":
251 + return inputs.get("multi_modal_data", {})
252 +
253 + if inputs["type"] == "multimodal":
254 + return inputs.get("mm_kwargs", {})
255 +
256 + assert_never(inputs)
257 +
258 + @cached_property
259 + def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
260 + inputs = self.inputs
261 +
262 + if inputs["type"] == "token":
263 + return inputs.get("multi_modal_placeholders", {})
264 +
265 + if inputs["type"] == "multimodal":
266 + return inputs.get("mm_placeholders", {})
267 +
268 + assert_never(inputs)
269 +
270 + @cached_property
271 + def mm_processor_kwargs(self) -> Dict[str, Any]:
272 + inputs = self.inputs
273 +
274 + if inputs["type"] == "token":
275 + return inputs.get("mm_processor_kwargs", {})
276 +
277 + if inputs["type"] == "multimodal":
278 + return {}
279 +
280 + assert_never(inputs)
281 +
282 +
283 ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
284 """
285 The inputs to :data:`vllm.inputs.InputProcessor`.
@@ -234,10 +310,11 @@ def zip_enc_dec_prompts(
310 ) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
311 """
312 Zip encoder and decoder prompts together into a list of
237 - :class:`ExplicitEncoderDecoderPrompt` instances. mm_processor_kwargs
238 - may also be provided; if a dict is passed, the same dictionary will be
239 - used for every encoder/decoder prompt. If an iterable is provided, it will
240 - be zipped with the encoder/decoder prompts.
313 + :class:`ExplicitEncoderDecoderPrompt` instances.
314 +
315 + ``mm_processor_kwargs`` may also be provided; if a dict is passed, the same
316 + dictionary will be used for every encoder/decoder prompt. If an iterable is
317 + provided, it will be zipped with the encoder/decoder prompts.
318 """
319 if mm_processor_kwargs is None:
320 mm_processor_kwargs = cast(Dict[str, Any], {})
vllm/inputs/preprocess.py
+122 -21
@@ -1,11 +1,13 @@
1 import asyncio
2 -from typing import List, Optional
2 +from typing import List, Mapping, Optional, Union
3
4 from typing_extensions import assert_never
5
6 from vllm.config import ModelConfig
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
9 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10 +from vllm.multimodal.processing import MultiModalDataDict, MultiModalInputsV2
11 from vllm.prompt_adapter.request import PromptAdapterRequest
12 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
13 from vllm.utils import print_warning_once
@@ -23,11 +25,13 @@ class InputPreprocessor:
25 self,
26 model_config: ModelConfig,
27 tokenizer: Optional[BaseTokenizerGroup],
28 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
29 ) -> None:
30 super().__init__()
31
32 self.model_config = model_config
33 self.tokenizer = tokenizer
34 + self.mm_registry = mm_registry
35
36 def get_tokenizer_group(self) -> BaseTokenizerGroup:
37 if self.tokenizer is None:
@@ -198,14 +202,79 @@ class InputPreprocessor:
202 prompt=prompt,
203 lora_request=lora_request)
204
205 + def _can_process_multimodal(self) -> bool:
206 + model_config = self.model_config
207 +
208 + if not model_config.is_multimodal_model:
209 + raise ValueError("Your model does not support multi-modal inputs")
210 +
211 + # Interim measure so we can handle models that have yet to be
212 + # updated to use the new multi-modal processor
213 + can_process_multimodal = self.mm_registry.has_processor(model_config)
214 + if not can_process_multimodal:
215 + logger.info(
216 + "Your model uses the legacy input pipeline instead of the new "
217 + "multi-modal processor. Please note that the legacy pipeline "
218 + "will be removed in a future release. For more details, see: "
219 + "https://github.com/vllm-project/vllm/issues/10114")
220 +
221 + return can_process_multimodal
222 +
223 + def _process_multimodal(
224 + self,
225 + prompt: Union[str, List[int]],
226 + mm_data: MultiModalDataDict,
227 + mm_processor_kwargs: Optional[Mapping[str, object]],
228 + lora_request: Optional[LoRARequest],
229 + ) -> MultiModalInputsV2:
230 + """
231 + Apply the model's multi-modal processor to a multi-modal prompt,
232 + returning the corresponding token IDs and metadata.
233 + """
234 + tokenizer_group = self.get_tokenizer_group()
235 + tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
236 +
237 + mm_processor = self.mm_registry.create_processor(
238 + self.model_config, tokenizer)
239 +
240 + if isinstance(prompt, list):
241 + prompt = tokenizer.decode(prompt)
242 + if mm_processor_kwargs is None:
243 + mm_processor_kwargs = {}
244 +
245 + return mm_processor.apply(prompt, mm_data, mm_processor_kwargs)
246 +
247 + async def _process_multimodal_async(
248 + self,
249 + prompt: Union[str, List[int]],
250 + mm_data: MultiModalDataDict,
251 + mm_processor_kwargs: Optional[Mapping[str, object]],
252 + lora_request: Optional[LoRARequest],
253 + ) -> MultiModalInputsV2:
254 + """Async version of :meth:`_process_multimodal`."""
255 + tokenizer_group = self.get_tokenizer_group()
256 + tokenizer = await tokenizer_group.get_lora_tokenizer_async(lora_request
257 + )
258 +
259 + mm_processor = self.mm_registry.create_processor(
260 + self.model_config, tokenizer)
261 + if isinstance(prompt, list):
262 + logger.warning("Passing `multi_modal_data` in TokensPrompt is"
263 + "deprecated and will be removed in a future update")
264 + prompt = tokenizer.decode(prompt)
265 + if mm_processor_kwargs is None:
266 + mm_processor_kwargs = {}
267 +
268 + return mm_processor.apply(prompt, mm_data, mm_processor_kwargs)
269 +
270 def _prompt_to_llm_inputs(
271 self,
272 prompt: SingletonPrompt,
273 request_id: str,
274 lora_request: Optional[LoRARequest] = None,
275 ) -> SingletonInputs:
207 - '''
208 - Extract the components of any single encoder or decoder input prompt.
276 + """
277 + Extract the singleton inputs from a prompt.
278
279 Arguments:
280
@@ -215,12 +284,8 @@ class InputPreprocessor:
284
285 Returns:
286
218 - * prompt
219 - * prompt_token_ids
220 - * multi_modal_data
221 - * mm_processor_kwargs (request-level input processor/mapper overrides)
222 - '''
223 -
287 + * :class:`SingletonInputs` instance
288 + """
289 parsed = parse_singleton_prompt(prompt)
290
291 if parsed["type"] == "str":
@@ -243,6 +308,14 @@ class InputPreprocessor:
308 multi_modal_data = tokens_content.get("multi_modal_data")
309 mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
310
311 + if multi_modal_data is not None and self._can_process_multimodal():
312 + return self._process_multimodal(
313 + prompt_token_ids,
314 + multi_modal_data,
315 + mm_processor_kwargs,
316 + lora_request=lora_request,
317 + )
318 +
319 return token_inputs(
320 prompt_token_ids=prompt_token_ids,
321 multi_modal_data=multi_modal_data,
@@ -253,13 +326,22 @@ class InputPreprocessor:
326 text_content = parsed["content"]
327
328 prompt_text = text_content["prompt"]
329 + multi_modal_data = text_content.get("multi_modal_data")
330 + mm_processor_kwargs = text_content.get("mm_processor_kwargs")
331 +
332 + if multi_modal_data is not None and self._can_process_multimodal():
333 + return self._process_multimodal(
334 + prompt_text,
335 + multi_modal_data,
336 + mm_processor_kwargs,
337 + lora_request=lora_request,
338 + )
339 +
340 prompt_token_ids = self._tokenize_prompt(
341 prompt_text,
342 request_id=request_id,
343 lora_request=lora_request,
344 )
261 - multi_modal_data = text_content.get("multi_modal_data")
262 - mm_processor_kwargs = text_content.get("mm_processor_kwargs")
345
346 return token_inputs(
347 prompt=prompt_text,
@@ -299,6 +381,14 @@ class InputPreprocessor:
381 multi_modal_data = tokens_content.get("multi_modal_data")
382 mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
383
384 + if multi_modal_data is not None and self._can_process_multimodal():
385 + return await self._process_multimodal_async(
386 + prompt_token_ids,
387 + multi_modal_data,
388 + mm_processor_kwargs,
389 + lora_request=lora_request,
390 + )
391 +
392 return token_inputs(
393 prompt_token_ids=prompt_token_ids,
394 multi_modal_data=multi_modal_data,
@@ -309,13 +399,22 @@ class InputPreprocessor:
399 text_content = parsed["content"]
400
401 prompt_text = text_content["prompt"]
402 + multi_modal_data = text_content.get("multi_modal_data")
403 + mm_processor_kwargs = text_content.get("mm_processor_kwargs")
404 +
405 + if multi_modal_data is not None and self._can_process_multimodal():
406 + return await self._process_multimodal_async(
407 + prompt_text,
408 + multi_modal_data,
409 + mm_processor_kwargs,
410 + lora_request=lora_request,
411 + )
412 +
413 prompt_token_ids = await self._tokenize_prompt_async(
414 prompt_text,
415 request_id=request_id,
416 lora_request=lora_request,
417 )
317 - multi_modal_data = text_content.get("multi_modal_data")
318 - mm_processor_kwargs = text_content.get("mm_processor_kwargs")
418
419 return token_inputs(
420 prompt=prompt_text,
@@ -331,7 +430,8 @@ class InputPreprocessor:
430 encoder_inputs: SingletonInputs,
431 decoder_inputs: Optional[SingletonInputs],
432 ) -> EncoderDecoderInputs:
334 - if encoder_inputs["type"] == "token":
433 + if (encoder_inputs["type"] == "token"
434 + or encoder_inputs["type"] == "multimodal"):
435 pass
436 else:
437 assert_never(encoder_inputs)
@@ -340,7 +440,8 @@ class InputPreprocessor:
440 dec_token_ids = self._prepare_decoder_input_ids_for_generation(
441 None)
442 decoder_inputs = token_inputs(dec_token_ids)
343 - elif decoder_inputs["type"] == "token":
443 + elif (decoder_inputs["type"] == "token"
444 + or decoder_inputs["type"] == "multimodal"):
445 dec_token_ids = self._prepare_decoder_input_ids_for_generation(
446 decoder_inputs["prompt_token_ids"])
447 decoder_inputs["prompt_token_ids"] = dec_token_ids
@@ -361,7 +462,7 @@ class InputPreprocessor:
462 prompt: PromptType,
463 request_id: str,
464 ) -> EncoderDecoderInputs:
364 - '''
465 + """
466 For encoder/decoder models only:
467 Process an input prompt into an :class:`EncoderDecoderInputs` instance.
468
@@ -391,8 +492,7 @@ class InputPreprocessor:
492 Returns:
493
494 * :class:`EncoderDecoderInputs` instance
394 - '''
395 -
495 + """
496 encoder_inputs: SingletonInputs
497 decoder_inputs: Optional[SingletonInputs]
498
@@ -460,7 +560,8 @@ class InputPreprocessor:
560 prompt_inputs: DecoderOnlyInputs,
561 prompt_adapter_request: Optional[PromptAdapterRequest],
562 ) -> DecoderOnlyInputs:
463 - if prompt_inputs["type"] == "token":
563 + if (prompt_inputs["type"] == "token"
564 + or prompt_inputs["type"] == "multimodal"):
565 prompt_inputs["prompt_token_ids"] = self._apply_prompt_adapter(
566 prompt_inputs["prompt_token_ids"],
567 prompt_adapter_request=prompt_adapter_request,
@@ -477,7 +578,7 @@ class InputPreprocessor:
578 lora_request: Optional[LoRARequest] = None,
579 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
580 ) -> DecoderOnlyInputs:
480 - '''
581 + """
582 For decoder-only models:
583 Process an input prompt into an :class:`DecoderOnlyInputs` instance.
584
@@ -491,7 +592,7 @@ class InputPreprocessor:
592 Returns:
593
594 * :class:`DecoderOnlyInputs` instance
494 - '''
595 + """
596
597 prompt_comps = self._prompt_to_llm_inputs(
598 prompt,
vllm/inputs/registry.py
+50 -6
@@ -5,14 +5,17 @@ from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5 Optional, Protocol, Type, cast)
6
7 from torch import nn
8 -from transformers import PretrainedConfig
9 -from typing_extensions import TypeVar
8 +from transformers import PretrainedConfig, ProcessorMixin
9 +from typing_extensions import TypeVar, assert_never
10
11 from vllm.logger import init_logger
12 +from vllm.transformers_utils.processor import cached_get_processor
13 +from vllm.transformers_utils.tokenizer import AnyTokenizer
14 from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
15 resolve_mm_processor_kwargs)
16
15 -from .data import ProcessorInputs
17 +from .data import ProcessorInputs, SingletonInputs
18 +from .parse import is_encoder_decoder_inputs
19
20 if TYPE_CHECKING:
21 from vllm.config import ModelConfig
@@ -61,6 +64,19 @@ class InputContext:
64 return self.model_config.hf_image_processor_config
65
66
67 +@dataclass(frozen=True)
68 +class InputProcessingContext(InputContext):
69 + tokenizer: AnyTokenizer
70 + """The tokenizer used to tokenize the inputs."""
71 +
72 + def get_hf_processor(self) -> ProcessorMixin:
73 + return cached_get_processor(
74 + self.model_config.tokenizer,
75 + tokenizer=self.tokenizer, # Override the tokenizer with ours
76 + trust_remote_code=self.model_config.trust_remote_code,
77 + )
78 +
79 +
80 N = TypeVar("N", bound=Type[nn.Module])
81
82
@@ -94,7 +110,7 @@ class DummyDataFactory(Protocol):
110 ...
111
112
97 -class _MultiModalCounts(UserDict):
113 +class _MultiModalCounts(UserDict[str, int]):
114 """
115 Wraps `mm_counts` for a more informative error message
116 when attempting to access a plugin that does not exist.
@@ -287,6 +303,21 @@ class InputRegistry:
303 return self._input_processors_by_model_type \
304 .get(model_cls, self._default_input_processor)
305
306 + def _ensure_mm_kwargs(
307 + self,
308 + inputs: SingletonInputs,
309 + mm_processor_kwargs: Dict[str, Any],
310 + ):
311 + if inputs["type"] == "token":
312 + # In case the input processor for that model fails to set it
313 + if "mm_processor_kwargs" not in inputs:
314 + inputs["mm_processor_kwargs"] = mm_processor_kwargs
315 + elif inputs["type"] == "multimodal":
316 + # Be more strict in V2
317 + assert "mm_kwargs" in inputs
318 + else:
319 + assert_never(inputs["type"])
320 +
321 def process_input(self, model_config: "ModelConfig",
322 inputs: ProcessorInputs) -> ProcessorInputs:
323 """
@@ -312,8 +343,21 @@ class InputRegistry:
343 processor,
344 )
345
315 - return processor(InputContext(model_config), inputs,
316 - **mm_processor_kwargs)
346 + processed_inputs = processor(
347 + InputContext(model_config),
348 + inputs,
349 + **mm_processor_kwargs,
350 + )
351 +
352 + if is_encoder_decoder_inputs(processed_inputs):
353 + self._ensure_mm_kwargs(processed_inputs["encoder"],
354 + mm_processor_kwargs)
355 + self._ensure_mm_kwargs(processed_inputs["decoder"],
356 + mm_processor_kwargs)
357 + else:
358 + self._ensure_mm_kwargs(processed_inputs, mm_processor_kwargs)
359 +
360 + return processed_inputs
361
362 def create_input_processor(self, model_config: "ModelConfig"):
363 """
vllm/model_executor/models/chatglm.py
+2 -2
@@ -30,8 +30,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31 from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
32 from vllm.model_executor.sampling_metadata import SamplingMetadata
33 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
34 -from vllm.multimodal.base import MultiModalData
33 +from vllm.multimodal import MULTIMODAL_REGISTRY
34 +from vllm.multimodal.inputs import MultiModalData, MultiModalKwargs
35 from vllm.multimodal.utils import cached_get_tokenizer
36 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
37 SequenceData)
vllm/model_executor/models/fuyu.py
+1 -2
@@ -32,8 +32,7 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear
32 from vllm.model_executor.layers.sampler import SamplerOutput
33 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
35 -from vllm.multimodal import MULTIMODAL_REGISTRY
36 -from vllm.multimodal.base import MultiModalKwargs
35 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
36 from vllm.multimodal.image import cached_get_image_processor
37 from vllm.multimodal.utils import (cached_get_tokenizer,
38 consecutive_placeholder_ranges)
vllm/model_executor/models/h2ovl.py
+1 -2
@@ -15,8 +15,7 @@ from transformers import PretrainedConfig
15 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
16 token_inputs)
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 -from vllm.multimodal import MULTIMODAL_REGISTRY
19 -from vllm.multimodal.base import MultiModalKwargs
18 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
19 from vllm.multimodal.utils import cached_get_tokenizer
20 from vllm.utils import is_list_of
21
vllm/model_executor/models/internvl.py
+1 -2
@@ -25,8 +25,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 -from vllm.multimodal import MULTIMODAL_REGISTRY
29 -from vllm.multimodal.base import MultiModalKwargs
28 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
29 from vllm.multimodal.utils import cached_get_tokenizer
30 from vllm.sequence import IntermediateTensors
31 from vllm.utils import is_list_of
vllm/model_executor/models/llava.py
+1 -1
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 -from vllm.multimodal.base import NestedTensors
20 +from vllm.multimodal.inputs import NestedTensors
21 from vllm.sequence import IntermediateTensors
22 from vllm.utils import is_list_of
23
vllm/model_executor/models/minicpmv.py
+1 -2
@@ -51,8 +51,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
51 from vllm.model_executor.models.qwen2 import Qwen2Model
52 from vllm.model_executor.models.utils import LLMWrapper
53 from vllm.model_executor.sampling_metadata import SamplingMetadata
54 -from vllm.multimodal import MULTIMODAL_REGISTRY
55 -from vllm.multimodal.base import MultiModalKwargs
54 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
55 from vllm.multimodal.image import cached_get_image_processor
56 from vllm.multimodal.utils import cached_get_tokenizer
57 from vllm.sequence import IntermediateTensors, SequenceData
vllm/model_executor/models/phi3v.py
+1 -1
@@ -39,7 +39,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
39 from vllm.model_executor.pooling_metadata import PoolingMetadata
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 from vllm.multimodal import MULTIMODAL_REGISTRY
42 -from vllm.multimodal.base import NestedTensors, PlaceholderRange
42 +from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
43 from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
44 from vllm.sequence import IntermediateTensors, PoolerOutput
45 from vllm.utils import is_list_of
vllm/model_executor/models/pixtral.py
+1 -2
@@ -29,8 +29,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.models.utils import merge_multimodal_embeddings
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 -from vllm.multimodal import MULTIMODAL_REGISTRY
33 -from vllm.multimodal.base import MultiModalKwargs
32 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
33 from vllm.multimodal.utils import (cached_get_tokenizer,
34 consecutive_placeholder_ranges)
35 from vllm.sequence import IntermediateTensors, SequenceData
vllm/model_executor/models/qwen.py
+1 -2
@@ -42,8 +42,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 from vllm.model_executor.models.module_mapping import MultiModelKeys
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 -from vllm.multimodal import MULTIMODAL_REGISTRY
46 -from vllm.multimodal.base import MultiModalKwargs
45 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
46 from vllm.multimodal.utils import cached_get_tokenizer
47 from vllm.sequence import IntermediateTensors, SequenceData
48 from vllm.utils import is_list_of
vllm/model_executor/models/qwen2_vl.py
+3 -3
@@ -60,10 +60,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
60 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
61 from vllm.model_executor.models.qwen2 import Qwen2Model
62 from vllm.model_executor.pooling_metadata import PoolingMetadata
63 -from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
64 - MultiModalKwargs)
65 -from vllm.multimodal.base import MultiModalData
63 +from vllm.multimodal import MULTIMODAL_REGISTRY
64 from vllm.multimodal.image import cached_get_image_processor
65 +from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
66 + MultiModalKwargs)
67 from vllm.multimodal.utils import cached_get_tokenizer
68 from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
69 from vllm.transformers_utils.config import uses_mrope
vllm/model_executor/models/utils.py
+1 -1
@@ -15,7 +15,7 @@ from vllm.config import VllmConfig
15 from vllm.logger import init_logger
16 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.model_executor.models import ModelRegistry
18 -from vllm.multimodal.base import MultiModalPlaceholderMap, NestedTensors
18 +from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
19 from vllm.platforms import current_platform
20 from vllm.sequence import IntermediateTensors
21 from vllm.utils import is_pin_memory_available
vllm/multimodal/__init__.py
+6 -4
@@ -1,7 +1,8 @@
1 -from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2 - MultiModalDataDict, MultiModalKwargs,
3 - MultiModalPlaceholderDict, MultiModalPlaceholderMap,
4 - MultiModalPlugin, NestedTensors)
1 +from .base import MultiModalPlaceholderMap, MultiModalPlugin
2 +from .inputs import (BatchedTensorInputs, MultiModalData,
3 + MultiModalDataBuiltins, MultiModalDataDict,
4 + MultiModalKwargs, MultiModalPlaceholderDict,
5 + NestedTensors)
6 from .registry import MultiModalRegistry
7
8 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -15,6 +16,7 @@ See also:
16
17 __all__ = [
18 "BatchedTensorInputs",
19 + "MultiModalData",
20 "MultiModalDataBuiltins",
21 "MultiModalDataDict",
22 "MultiModalKwargs",
vllm/multimodal/audio.py
+9 -3
@@ -1,5 +1,7 @@
1 from vllm.inputs.registry import InputContext
2 -from vllm.multimodal.base import MultiModalKwargs, MultiModalPlugin
2 +
3 +from .base import MultiModalPlugin
4 +from .inputs import AudioItem, MultiModalData, MultiModalKwargs
5
6
7 class AudioPlugin(MultiModalPlugin):
@@ -8,8 +10,12 @@ class AudioPlugin(MultiModalPlugin):
10 def get_data_key(self) -> str:
11 return "audio"
12
11 - def _default_input_mapper(self, ctx: InputContext, data: object,
12 - **mm_processor_kwargs) -> MultiModalKwargs:
13 + def _default_input_mapper(
14 + self,
15 + ctx: InputContext,
16 + data: MultiModalData[AudioItem],
17 + **mm_processor_kwargs,
18 + ) -> MultiModalKwargs:
19 raise NotImplementedError("There is no default audio input mapper")
20
21 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
vllm/multimodal/base.py
+18 -170
@@ -1,180 +1,23 @@
1 from abc import ABC, abstractmethod
2 -from collections import UserDict, defaultdict
3 -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
4 - NamedTuple, Optional, Tuple, Type, TypedDict, TypeVar,
5 - Union, cast, final)
6 -
7 -import numpy as np
8 -import torch
9 -import torch.types
10 -from PIL import Image
2 +from collections import defaultdict
3 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
4 + Optional, Sequence, Tuple, Type, TypeVar, Union)
5 +
6 from torch import nn
12 -from typing_extensions import TypeAlias
7
8 from vllm.inputs import InputContext
9 from vllm.logger import init_logger
16 -from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
17 - json_map_leaves, resolve_mm_processor_kwargs)
10 +from vllm.utils import (get_allowed_kwarg_only_overrides,
11 + resolve_mm_processor_kwargs)
12
13 if TYPE_CHECKING:
14 from vllm.config import ModelConfig
15 from vllm.sequence import SequenceGroupMetadata
16
23 -logger = init_logger(__name__)
24 -
25 -NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
26 -"""
27 -Uses a list instead of a tensor if the dimensions of each element do not match.
28 -"""
29 -
30 -BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
31 -"""
32 -A dictionary containing nested tensors which have been batched via
33 -:meth:`MultiModalKwargs.batch`.
34 -"""
35 -
36 -
37 -class _MultiModalKwargsBase(UserDict[str, NestedTensors]):
38 - pass
39 -
40 -
41 -class MultiModalKwargs(_MultiModalKwargsBase):
42 - """
43 - A dictionary that represents the keyword arguments to
44 - :meth:`~torch.nn.Module.forward`.
45 - """
46 -
47 - @staticmethod
48 - def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
49 - """
50 - Recursively stacks lists of tensors when they all have the same shape.
51 - """
52 - if isinstance(nested_tensors, torch.Tensor):
53 - return nested_tensors
54 -
55 - if isinstance(nested_tensors, np.ndarray):
56 - return torch.from_numpy(nested_tensors)
57 -
58 - if isinstance(nested_tensors, (int, float)):
59 - return torch.tensor(nested_tensors)
17 +from .inputs import (MultiModalData, MultiModalDataDict, MultiModalKwargs,
18 + PlaceholderRange)
19
61 - stacked = [MultiModalKwargs._try_stack(t) for t in nested_tensors]
62 - if not is_list_of(stacked, torch.Tensor, check="all"):
63 - # Only tensors (not lists) can be stacked.
64 - return stacked
65 -
66 - tensors_ = cast(List[torch.Tensor], stacked)
67 - if any(t.shape != tensors_[0].shape for t in tensors_):
68 - # The tensors have incompatible shapes and can't be stacked.
69 - return tensors_
70 -
71 - return torch.stack(tensors_)
72 -
73 - @staticmethod
74 - def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
75 - """
76 - Batch multiple inputs together into a dictionary.
77 -
78 - The resulting dictionary has the same keys as the inputs.
79 - If the corresponding value from each input is a tensor and they all
80 - share the same shape, the output value is a single batched tensor;
81 - otherwise, the output value is a list containing the original value
82 - from each input.
83 - """
84 - if len(inputs_list) == 0:
85 - return {}
86 -
87 - item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
88 -
89 - for inputs in inputs_list:
90 - # For models that supports multiple modalities (e.g. Qwen2-VL),
91 - # different modalities will return different data keys,
92 - # so batch() should skip the same key check.
93 -
94 - for k, v in inputs.items():
95 - item_lists[k].append(v)
96 -
97 - return {
98 - k: MultiModalKwargs._try_stack(item_list)
99 - for k, item_list in item_lists.items()
100 - }
101 -
102 - @staticmethod
103 - def as_kwargs(
104 - batched_inputs: BatchedTensorInputs,
105 - *,
106 - device: torch.types.Device,
107 - ) -> BatchedTensorInputs:
108 - json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
109 -
110 - json_mapped = json_map_leaves(
111 - lambda x: x.to(device, non_blocking=True),
112 - json_inputs,
113 - )
114 -
115 - return cast(BatchedTensorInputs, json_mapped)
116 -
117 -
118 -_T = TypeVar("_T")
119 -
120 -MultiModalData: TypeAlias = Union[_T, List[_T]]
121 -"""
122 -Either a single data instance, or a list of data instances.
123 -
124 -The number of data instances allowed per modality is restricted by
125 -`--limit-mm-per-prompt`.
126 -"""
127 -
128 -
129 -@final
130 -class MultiModalDataBuiltins(TypedDict, total=False):
131 - """Modality types that are predefined by vLLM."""
132 -
133 - image: MultiModalData[Image.Image]
134 - """The input image(s)."""
135 -
136 - audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]]
137 - """The input audio item(s) and corresponding sampling rate(s)."""
138 -
139 - video: MultiModalData[Tuple[np.ndarray]]
140 - """The input video(s)."""
141 -
142 -
143 -MultiModalDataDict = Union[MultiModalDataBuiltins,
144 - Mapping[str, MultiModalData[object]]]
145 -"""
146 -A dictionary containing an item for each modality type to input.
147 -
148 -Note:
149 - This dictionary also accepts modality keys defined outside
150 - :class:`MultiModalDataBuiltins` as long as a customized plugin is registered
151 - through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
152 - Read more on that :ref:`here <adding_multimodal_plugin>`.
153 -"""
154 -
155 -
156 -class PlaceholderRange(TypedDict):
157 - """
158 - Placeholder location information for multi-modal data.
159 -
160 - For example:
161 - Prompt: AAAA BBBB What is in these images?
162 - Images A and B will have:
163 - A: { "offset": 0, "length": 4 }
164 - B: { "offset": 5, "length": 4 }
165 - """
166 -
167 - offset: int
168 - """The start index of the placeholder in the prompt."""
169 -
170 - length: int
171 - """The length of the placeholder."""
172 -
173 -
174 -MultiModalPlaceholderDict = Mapping[str, List[PlaceholderRange]]
175 -"""
176 -A dictionary containing placeholder ranges.
177 -"""
20 +logger = init_logger(__name__)
21
22 MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
23 MultiModalKwargs]
@@ -192,6 +35,7 @@ Calculate the maximum number of multimodal tokens input to the language
35 model. This does not include tokens that correspond to the input text.
36 """
37
38 +_T = TypeVar("_T")
39 N = TypeVar("N", bound=Type[nn.Module])
40
41
@@ -224,7 +68,7 @@ class MultiModalPlugin(ABC):
68 def _default_input_mapper(
69 self,
70 ctx: InputContext,
227 - data: MultiModalData[object],
71 + data: MultiModalData[Any],
72 **mm_processor_kwargs,
73 ) -> MultiModalKwargs:
74 """
@@ -273,8 +117,8 @@ class MultiModalPlugin(ABC):
117 def map_input(
118 self,
119 model_config: "ModelConfig",
276 - data: MultiModalData[object],
277 - mm_processor_kwargs: Dict[str, Any],
120 + data: MultiModalData[Any],
121 + mm_processor_kwargs: Optional[Dict[str, Any]],
122 ) -> MultiModalKwargs:
123 """
124 Transform the data into a dictionary of model inputs using the
@@ -289,6 +133,7 @@ class MultiModalPlugin(ABC):
133 - :ref:`input_processing_pipeline`
134 - :ref:`enabling_multimodal_inputs`
135 """
136 +
137 # Avoid circular import
138 from vllm.model_executor.model_loader import get_model_architecture
139
@@ -300,6 +145,9 @@ class MultiModalPlugin(ABC):
145 raise KeyError(f"No input mapper in {self} is registered for "
146 f"model class {model_cls.__name__}.")
147
148 + if mm_processor_kwargs is None:
149 + mm_processor_kwargs = {}
150 +
151 # In the case of the default mapper, we have to get resource
152 # processor through its HuggingFace autoclass; since this goes
153 # through **kwargs, we can't inspect it the same way, so we allow
@@ -508,7 +356,7 @@ class MultiModalPlaceholderMap:
356 self,
357 positions: range,
358 multi_modal_items: List[_T],
511 - multi_modal_placeholders: List[PlaceholderRange],
359 + multi_modal_placeholders: Sequence[PlaceholderRange],
360 ) -> List[_T]:
361 """
362 Adds the multi-modal items that intersect ```positions`` to this
vllm/multimodal/image.py
+3 -7
@@ -3,14 +3,14 @@ from typing import TYPE_CHECKING, Any, Dict, Optional
3
4 import torch
5 from PIL import Image
6 -from transformers.image_processing_base import BatchFeature
6
7 from vllm.inputs.registry import InputContext
8 from vllm.logger import init_logger
9 from vllm.transformers_utils.processor import get_image_processor
10 from vllm.utils import is_list_of
11
13 -from .base import MultiModalData, MultiModalKwargs, MultiModalPlugin
12 +from .base import MultiModalPlugin
13 +from .inputs import ImageItem, MultiModalData, MultiModalKwargs
14
15 if TYPE_CHECKING:
16 from vllm.config import ModelConfig
@@ -41,15 +41,11 @@ class ImagePlugin(MultiModalPlugin):
41 def _default_input_mapper(
42 self,
43 ctx: InputContext,
44 - data: MultiModalData[object],
44 + data: MultiModalData[ImageItem],
45 **mm_processor_kwargs,
46 ) -> MultiModalKwargs:
47 model_config = ctx.model_config
48
49 - # Processed by input processor
50 - if isinstance(data, BatchFeature):
51 - return MultiModalKwargs(data.data)
52 -
49 # PIL image
50 if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
51 image_processor = self._get_hf_image_processor(
vllm/multimodal/inputs.py new
+225
@@ -0,0 +1,225 @@
1 +from collections import UserDict, defaultdict
2 +from typing import (Any, Dict, List, Literal, Mapping, Sequence, Tuple,
3 + TypedDict, TypeVar, Union, cast, final)
4 +
5 +import numpy as np
6 +import torch
7 +import torch.types
8 +from PIL.Image import Image
9 +from typing_extensions import TypeAlias
10 +
11 +from vllm.utils import JSONTree, is_list_of, json_map_leaves
12 +
13 +_T = TypeVar("_T")
14 +
15 +# yapf: disable
16 +ImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
17 +"""
18 +A :class:`transformers.image_utils.ImageInput` representing a single image,
19 +which can be passed to a HuggingFace :code:`ImageProcessor`.
20 +"""
21 +
22 +VideoItem: TypeAlias = Union[
23 + List[Image],
24 + np.ndarray,
25 + torch.Tensor,
26 + List[np.ndarray],
27 + List[torch.Tensor],
28 +]
29 +"""
30 +
31 +A :class:`transformers.image_utils.VideoInput` representing a single video,
32 +which can be passed to a HuggingFace :code:`VideoProcessor`.
33 +"""
34 +
35 +AudioItem: TypeAlias = Union[
36 + np.ndarray,
37 + List[float],
38 + Tuple[np.ndarray, float], # DEPRECATED: Use mm_processor_kwargs instead
39 +]
40 +"""
41 +Represents a single audio that can be inputted to a HuggingFace
42 +:code:`AudioProcessor`.
43 +"""
44 +# yapf: enable
45 +
46 +MultiModalData: TypeAlias = Union[_T, List[_T]]
47 +"""
48 +Either a single data item, or a list of data items.
49 +
50 +The number of data items allowed per modality is restricted by
51 +:code:`--limit-mm-per-prompt`.
52 +"""
53 +
54 +
55 +@final
56 +class MultiModalDataBuiltins(TypedDict, total=False):
57 + """Type annotations for modality types predefined by vLLM."""
58 +
59 + image: MultiModalData[ImageItem]
60 + """The input image(s)."""
61 +
62 + video: MultiModalData[VideoItem]
63 + """The input video(s)."""
64 +
65 + audio: MultiModalData[AudioItem]
66 + """The input audio(s)."""
67 +
68 +
69 +MultiModalDataDict: TypeAlias = Mapping[str, MultiModalData[Any]]
70 +"""
71 +A dictionary containing an entry for each modality type to input.
72 +
73 +Note:
74 + This dictionary also accepts modality keys defined outside
75 + :class:`MultiModalDataBuiltins` as long as a customized plugin
76 + is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
77 + Read more on that :ref:`here <adding_multimodal_plugin>`.
78 +"""
79 +
80 +
81 +class PlaceholderRange(TypedDict):
82 + """
83 + Placeholder location information for multi-modal data.
84 +
85 + For example:
86 + Prompt: AAAA BBBB What is in these images?
87 + Images A and B will have:
88 + A: { "offset": 0, "length": 4 }
89 + B: { "offset": 5, "length": 4 }
90 + """
91 +
92 + offset: int
93 + """The start index of the placeholder in the prompt."""
94 +
95 + length: int
96 + """The length of the placeholder."""
97 +
98 +
99 +NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
100 +"""
101 +Uses a list instead of a tensor if the dimensions of each element do not match.
102 +"""
103 +
104 +BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
105 +"""
106 +A dictionary containing nested tensors which have been batched via
107 +:meth:`MultiModalKwargs.batch`.
108 +"""
109 +
110 +
111 +class MultiModalKwargs(UserDict[str, NestedTensors]):
112 + """
113 + A dictionary that represents the keyword arguments to
114 + :meth:`~torch.nn.Module.forward`.
115 + """
116 +
117 + @staticmethod
118 + def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
119 + """
120 + Stack the inner dimensions that have the same shape in
121 + a nested list of tensors.
122 +
123 + Thus, a dimension represented by a list means that the inner
124 + dimensions are different for each element along that dimension.
125 + """
126 + if isinstance(nested_tensors, torch.Tensor):
127 + return nested_tensors
128 +
129 + # TODO: Remove these once all models have been migrated
130 + if isinstance(nested_tensors, np.ndarray):
131 + return torch.from_numpy(nested_tensors)
132 + if isinstance(nested_tensors, (int, float)):
133 + return torch.tensor(nested_tensors)
134 +
135 + stacked = [MultiModalKwargs._try_stack(t) for t in nested_tensors]
136 + if not is_list_of(stacked, torch.Tensor, check="all"):
137 + # Only tensors (not lists) can be stacked.
138 + return stacked
139 +
140 + tensors_ = cast(List[torch.Tensor], stacked)
141 + if any(t.shape != tensors_[0].shape for t in tensors_):
142 + # The tensors have incompatible shapes and can't be stacked.
143 + return tensors_
144 +
145 + return torch.stack(tensors_)
146 +
147 + @staticmethod
148 + def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
149 + """
150 + Batch multiple inputs together into a dictionary.
151 +
152 + The resulting dictionary has the same keys as the inputs.
153 + If the corresponding value from each input is a tensor and they all
154 + share the same shape, the output value is a single batched tensor;
155 + otherwise, the output value is a list containing the original value
156 + from each input.
157 + """
158 + if len(inputs_list) == 0:
159 + return {}
160 +
161 + # We need to consider the case where each item in the batch
162 + # contains different modalities (i.e. different keys).
163 + item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
164 +
165 + for inputs in inputs_list:
166 + for k, v in inputs.items():
167 + item_lists[k].append(v)
168 +
169 + return {
170 + k: MultiModalKwargs._try_stack(item_list)
171 + for k, item_list in item_lists.items()
172 + }
173 +
174 + @staticmethod
175 + def as_kwargs(
176 + batched_inputs: BatchedTensorInputs,
177 + *,
178 + device: torch.types.Device,
179 + ) -> BatchedTensorInputs:
180 + json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
181 +
182 + json_mapped = json_map_leaves(
183 + lambda x: x.to(device, non_blocking=True),
184 + json_inputs,
185 + )
186 +
187 + return cast(BatchedTensorInputs, json_mapped)
188 +
189 +
190 +MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]
191 +"""
192 +A dictionary containing placeholder ranges.
193 +"""
194 +
195 +
196 +class MultiModalInputsV2(TypedDict):
197 + """
198 + Represents the outputs of :class:`vllm.multimodal.MultiModalProcessor`,
199 + ready to be passed to vLLM internals.
200 + """
201 +
202 + type: Literal["multimodal"]
203 + """The type of inputs."""
204 +
205 + prompt: str
206 + """
207 + The original, unprocessed prompt text.
208 +
209 + Note:
210 + Since prompt text is not required by vLLM internals, we leave this
211 + unprocessed to save CPU computation. You can still call
212 + :code:`tokenizer.decode(prompt_token_ids)` to get the processed text.
213 + """
214 +
215 + prompt_token_ids: List[int]
216 + """The processed token IDs which includes placeholder tokens."""
217 +
218 + mm_kwargs: MultiModalKwargs
219 + """Keyword arguments to be directly passed to the model after batching."""
220 +
221 + mm_placeholders: MultiModalPlaceholderDict
222 + """
223 + For each modality, information about the placeholder tokens in
224 + :code:`prompt_token_ids`.
225 + """
vllm/multimodal/processing.py new
+273
@@ -0,0 +1,273 @@
1 +from dataclasses import dataclass
2 +from functools import lru_cache, partial
3 +from typing import (Any, Callable, Collection, Generic, List, Mapping,
4 + Optional, TypedDict, TypeVar, final)
5 +
6 +from transformers import BatchFeature
7 +from typing_extensions import TypeAlias
8 +
9 +from vllm.inputs import InputProcessingContext
10 +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
11 +from vllm.utils import is_list_of
12 +
13 +from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
14 + MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
15 + VideoItem)
16 +
17 +_T = TypeVar("_T")
18 +
19 +ReplacementFunc: TypeAlias = Callable[[_T, BatchFeature, int], List[int]]
20 +"""
21 +Given the original data item, HF-processed data, and index of the processed
22 +item, output the replacement token IDs to be allocated in vLLM.
23 +"""
24 +
25 +
26 +@dataclass
27 +class ModalityProcessingMetadata(Generic[_T]):
28 + placeholder_replacements: Mapping[str, ReplacementFunc]
29 + """
30 + A dictionary where each item represents the original placeholder in the
31 + prompt text and the corresponding replacement.
32 + """
33 +
34 +
35 +class MultiModalProcessingMetadataBuiltins(TypedDict, total=False):
36 + """Type annotations for modality types predefined by vLLM."""
37 +
38 + image: ModalityProcessingMetadata[ImageItem]
39 + video: ModalityProcessingMetadata[VideoItem]
40 + audio: ModalityProcessingMetadata[AudioItem]
41 +
42 +
43 +MultiModalProcessingMetadata: TypeAlias = \
44 + Mapping[str, ModalityProcessingMetadata[Any]]
45 +"""
46 +A dictionary containing an entry for each modality type to process.
47 +
48 +Note:
49 + This dictionary also accepts modality keys defined outside
50 + :class:`MultiModalProcessingMetadataBuiltins` as long as a customized plugin
51 + is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
52 + Read more on that :ref:`here <adding_multimodal_plugin>`.
53 +"""
54 +
55 +MultiModalMultiData: TypeAlias = List[_T]
56 +"""
57 +A list of data items, where the number of data items allowed
58 +per modality is restricted by :code:`--limit-mm-per-prompt`.
59 +"""
60 +
61 +
62 +@final
63 +class MultiModalMultiDataBuiltins(TypedDict, total=False):
64 + """Type annotations for modality types predefined by vLLM."""
65 +
66 + image: MultiModalMultiData[ImageItem]
67 + """The input images."""
68 +
69 + video: MultiModalMultiData[VideoItem]
70 + """The input videos."""
71 +
72 + audio: MultiModalMultiData[AudioItem]
73 + """The input audios."""
74 +
75 +
76 +MultiModalMultiDataDict: TypeAlias = Mapping[str, MultiModalMultiData[Any]]
77 +"""
78 +A dictionary containing an entry for each modality type to input.
79 +
80 +Note:
81 + This dictionary also accepts modality keys defined outside
82 + :class:`MultiModalMultiDataBuiltins` as long as a customized plugin
83 + is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
84 + Read more on that :ref:`here <adding_multimodal_plugin>`.
85 +"""
86 +
87 +
88 +def to_multi_format(data: MultiModalDataDict) -> MultiModalMultiDataDict:
89 + """
90 + Convert a :class:`MultiModalDataDict` containing single data items
91 + to a :class:`MultiModalMultiDataDict` containing multiple data items
92 + per entry.
93 + """
94 + multi_data: Mapping[str, MultiModalMultiData[Any]] = {}
95 +
96 + for k, v in data.items():
97 + # yapf: disable
98 + if k == "video":
99 + # Special case since even a single item can be a list
100 + multi_data[k] = v if is_list_of(v, list) else [v] # type: ignore[index]
101 + elif k in ("image", "audio"):
102 + multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
103 + else:
104 + multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
105 + # yapf: enable
106 +
107 + return multi_data
108 +
109 +
110 +def encode_no_special_tokens(
111 + tokenizer: AnyTokenizer,
112 + text: str,
113 +) -> List[int]:
114 + """
115 + Backend-agnostic equivalent of HF's
116 + :code:`tokenizer.encode(text, add_special_tokens=False)`.
117 + """
118 + if isinstance(tokenizer, MistralTokenizer):
119 + return tokenizer.tokenizer.encode(text, bos=False, eos=False)
120 +
121 + return tokenizer.encode(text, add_special_tokens=False)
122 +
123 +
124 +@lru_cache
125 +def candidate_placeholders(
126 + tokenizer: AnyTokenizer,
127 + placeholder_text: str,
128 +) -> Collection[List[int]]:
129 + """Generate token ID sequences that may represent a placeholder text."""
130 + # When the placeholder text is not mapped to a special token ID,
131 + # it may be tokenized differently based on whether it is at the start/end
132 + # of the string. So, we go through each combination of whether the text
133 + # is at the start and end boundaries of the string
134 +
135 + # Matches the placeholder when it is in the middle of the string
136 + start_id, = encode_no_special_tokens(tokenizer, "a")
137 + end_id, = encode_no_special_tokens(tokenizer, "b")
138 +
139 + candidate_basic = encode_no_special_tokens(tokenizer, placeholder_text)
140 +
141 + start_id_, *candidate_a = encode_no_special_tokens(
142 + tokenizer,
143 + f"a{placeholder_text}",
144 + )
145 + assert start_id == start_id_
146 +
147 + start_id_, *candidate_ab, end_id_ = encode_no_special_tokens(
148 + tokenizer,
149 + f"a{placeholder_text}b",
150 + )
151 + assert start_id == start_id_ and end_id == end_id_
152 +
153 + *candidate_b, end_id_ = encode_no_special_tokens(
154 + tokenizer,
155 + f"{placeholder_text}b",
156 + )
157 + assert end_id == end_id_
158 +
159 + # Remove duplicates (need to convert to tuple to be hashable)
160 + unique_candidates = {
161 + tuple(c)
162 + for c in [candidate_basic, candidate_a, candidate_ab, candidate_b]
163 + }
164 +
165 + # Convert back to list
166 + return [list(c) for c in unique_candidates]
167 +
168 +
169 +def apply_placeholders(
170 + token_ids: List[int],
171 + placeholder_ids: List[int],
172 + get_replacement_ids: Callable[[], List[int]],
173 +) -> Optional[PlaceholderRange]:
174 + """
175 + Find the first occurrence of :code:`placeholder_ids`,
176 + and replace it with the output of :code:`get_replacement_ids`.
177 +
178 + This function updates :code:`token_ids` in place.
179 + """
180 + placeholder_length = len(placeholder_ids)
181 +
182 + for start_idx in range(len(token_ids) - placeholder_length + 1):
183 + if token_ids[start_idx:placeholder_length] == placeholder_ids:
184 + token_ids[start_idx:placeholder_length] = get_replacement_ids()
185 +
186 + return PlaceholderRange(offset=start_idx,
187 + length=placeholder_length)
188 +
189 + return None
190 +
191 +
192 +class MultiModalProcessor:
193 + """
194 + Helper class to process multi-modal inputs to be used in vLLM.
195 + """
196 +
197 + def __init__(
198 + self,
199 + ctx: InputProcessingContext,
200 + metadata: MultiModalProcessingMetadata,
201 + ) -> None:
202 + super().__init__()
203 +
204 + self.ctx = ctx
205 + self.metadata = metadata
206 +
207 + def __call__(
208 + self,
209 + prompt: str,
210 + mm_data: MultiModalDataDict,
211 + mm_processor_kwargs: Mapping[str, object],
212 + ) -> MultiModalInputsV2:
213 + return self.apply(prompt, mm_data, mm_processor_kwargs)
214 +
215 + def apply(
216 + self,
217 + prompt: str,
218 + mm_data: MultiModalDataDict,
219 + mm_processor_kwargs: Mapping[str, object],
220 + ) -> MultiModalInputsV2:
221 + tokenizer = self.ctx.tokenizer
222 + hf_processor = self.ctx.get_hf_processor()
223 +
224 + processed_inputs = hf_processor(
225 + text=prompt, # type: ignore
226 + **mm_data,
227 + **mm_processor_kwargs,
228 + )
229 + new_token_ids, = processed_inputs.pop("input_ids").tolist()
230 + mm_kwargs = MultiModalKwargs(processed_inputs)
231 +
232 + mm_placeholders: Mapping[str, List[PlaceholderRange]] = {}
233 +
234 + for modality, orig_inputs in to_multi_format(mm_data).items():
235 + assert isinstance(orig_inputs, list)
236 +
237 + metadata = self.metadata[modality]
238 + placeholder_replacements = metadata.placeholder_replacements
239 +
240 + modality_placeholders: List[PlaceholderRange] = []
241 +
242 + for item_idx, orig_item in enumerate(orig_inputs):
243 + for match_text, replace_fn in placeholder_replacements.items():
244 + candidates = candidate_placeholders(tokenizer, match_text)
245 + get_replacement_ids = partial(
246 + replace_fn,
247 + orig_item,
248 + processed_inputs,
249 + item_idx,
250 + )
251 +
252 + for match_ids in candidates:
253 + # TODO(youkaichao): Don't update new_token_ids
254 + placeholders = apply_placeholders(
255 + new_token_ids,
256 + match_ids,
257 + get_replacement_ids,
258 + )
259 +
260 + if placeholders is not None:
261 + modality_placeholders.append(placeholders)
262 +
263 + # yapf: disable
264 + mm_placeholders[modality] = modality_placeholders # type: ignore[index]
265 + # yapf: enable
266 +
267 + return MultiModalInputsV2(
268 + type="multimodal",
269 + prompt=prompt,
270 + prompt_token_ids=new_token_ids,
271 + mm_kwargs=mm_kwargs,
272 + mm_placeholders=mm_placeholders,
273 + )
vllm/multimodal/registry.py
+80 -4
@@ -1,13 +1,20 @@
1 import functools
2 from collections import UserDict
3 -from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Sequence
3 +from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
4 + Sequence, Type, TypeVar)
5
6 +import torch.nn as nn
7 +from typing_extensions import TypeAlias
8 +
9 +from vllm.inputs import InputProcessingContext
10 from vllm.logger import init_logger
11 +from vllm.transformers_utils.tokenizer import AnyTokenizer
12
13 from .audio import AudioPlugin
8 -from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalKwargs,
9 - MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
14 +from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
15 from .image import ImagePlugin
16 +from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
17 +from .processing import MultiModalProcessor
18 from .video import VideoPlugin
19
20 if TYPE_CHECKING:
@@ -15,8 +22,18 @@ if TYPE_CHECKING:
22
23 logger = init_logger(__name__)
24
25 +N = TypeVar("N", bound=Type[nn.Module])
26 +
27 +MultiModalProcessorFactory: TypeAlias = Callable[[InputProcessingContext],
28 + MultiModalProcessor]
29 +"""
30 +Constructs a :class:`MultiModalProcessor` instance from the context.
31 +
32 +The processing metadata should be derived from the context.
33 +"""
34 +
35
19 -class _MultiModalLimits(UserDict):
36 +class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
37 """
38 Wraps `_limits_by_model` for a more informative error message
39 when attempting to access a model that does not exist.
@@ -45,6 +62,9 @@ class MultiModalRegistry:
62 plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
63 self._plugins = {p.get_data_key(): p for p in plugins}
64
65 + self._processor_factories: Dict[Type[nn.Module],
66 + MultiModalProcessorFactory] = {}
67 +
68 # This is used for non-multimodal models
69 self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
70
@@ -243,3 +263,59 @@ class MultiModalRegistry:
263 This should be called after :meth:`init_mm_limits_per_prompt`.
264 """
265 return self._limits_by_model[model_config]
266 +
267 + def register_processor(
268 + self,
269 + factory: MultiModalProcessorFactory,
270 + ):
271 + """
272 + Register a multi-modal processor to a model class.
273 +
274 + When the model receives multi-modal data, the provided function is
275 + invoked to transform the data into a dictionary of model inputs.
276 +
277 + See also:
278 + - :ref:`input_processing_pipeline`
279 + - :ref:`enabling_multimodal_inputs`
280 + """
281 +
282 + def wrapper(model_cls: N) -> N:
283 + if model_cls in self._processor_factories:
284 + logger.warning(
285 + "Model class %s already has an input mapper "
286 + "registered to %s. It is overwritten by the new one.",
287 + model_cls, self)
288 +
289 + self._processor_factories[model_cls] = factory
290 +
291 + return model_cls
292 +
293 + return wrapper
294 +
295 + def has_processor(self, model_config: "ModelConfig") -> bool:
296 + """
297 + Test whether a multi-modal processor is defined for a specific model.
298 + """
299 + # Avoid circular import
300 + from vllm.model_executor.model_loader import get_model_architecture
301 +
302 + model_cls, _ = get_model_architecture(model_config)
303 + return model_cls in self._processor_factories
304 +
305 + def create_processor(
306 + self,
307 + model_config: "ModelConfig",
308 + tokenizer: AnyTokenizer,
309 + ) -> MultiModalProcessor:
310 + """
311 + Create a multi-modal processor for a specific model and tokenizer.
312 + """
313 +
314 + # Avoid circular import
315 + from vllm.model_executor.model_loader import get_model_architecture
316 +
317 + model_cls, _ = get_model_architecture(model_config)
318 + processor_factory = self._processor_factories[model_cls]
319 +
320 + ctx = InputProcessingContext(model_config, tokenizer)
321 + return processor_factory(ctx)
vllm/multimodal/utils.py
+2 -1
@@ -11,9 +11,10 @@ from PIL import Image
11 import vllm.envs as envs
12 from vllm.connections import global_http_connection
13 from vllm.logger import init_logger
14 -from vllm.multimodal.base import MultiModalDataDict, PlaceholderRange
14 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
15
16 +from .inputs import MultiModalDataDict, PlaceholderRange
17 +
18 logger = init_logger(__name__)
19
20 cached_get_tokenizer = lru_cache(get_tokenizer)
vllm/multimodal/video.py
+5 -15
@@ -1,5 +1,5 @@
1 from functools import lru_cache
2 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
2 +from typing import TYPE_CHECKING, Any, Dict, Optional
3
4 import numpy as np
5
@@ -9,8 +9,9 @@ from vllm.transformers_utils.processor import get_video_processor
9 from vllm.transformers_utils.tokenizer import get_tokenizer
10 from vllm.utils import is_list_of
11
12 -from .base import MultiModalData, MultiModalKwargs
12 +from .base import MultiModalData
13 from .image import ImagePlugin
14 +from .inputs import MultiModalKwargs, VideoItem
15
16 if TYPE_CHECKING:
17 from vllm.config import ModelConfig
@@ -20,17 +21,6 @@ logger = init_logger(__name__)
21 cached_get_video_processor = lru_cache(get_video_processor)
22 cached_get_tokenizer = lru_cache(get_tokenizer)
23
23 -VideoInput = Union[
24 - "np.ndarray", # single video input
25 - List["np.ndarray"],
26 - # TODO: support more types
27 - # List[Image.Image], List[List[Image.Image]],
28 - # "torch.Tensor",
29 - # List["torch.Tensor"],
30 - # List[List["np.ndarrray"]],
31 - # List[List["torch.Tensor"]],
32 -]
33 -
24
25 class VideoPlugin(ImagePlugin):
26 """Plugin for video data."""
@@ -53,13 +43,13 @@ class VideoPlugin(ImagePlugin):
43 def _default_input_mapper(
44 self,
45 ctx: InputContext,
56 - data: MultiModalData[object],
46 + data: MultiModalData[VideoItem],
47 **mm_processor_kwargs,
48 ) -> MultiModalKwargs:
49 model_config = ctx.model_config
50
51 if isinstance(data, list) and len(data) == 1:
62 - data = data[0]
52 + data = data[0] # type: ignore
53
54 if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
55 video_processor = self._get_hf_video_processor(
vllm/sequence.py
+17 -51
@@ -5,25 +5,21 @@ from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
7 from dataclasses import dataclass, field
8 -from functools import cached_property, reduce
9 -from typing import (TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List,
10 - Mapping, Optional)
8 +from functools import reduce
9 +from typing import Any, Callable, DefaultDict, Dict, List, Mapping, Optional
10 from typing import Sequence as GenericSequence
11 from typing import Set, Tuple, Union
12
13 import msgspec
14 import torch
16 -from typing_extensions import assert_never
15
16 +from vllm.inputs import SingletonInputs, SingletonInputsAdapter
17 from vllm.lora.request import LoRARequest
18 from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
19 from vllm.pooling_params import PoolingParams
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sampling_params import RequestOutputKind, SamplingParams
22
24 -if TYPE_CHECKING:
25 - from vllm.inputs import SingletonInputs
26 -
23 VLLM_TOKEN_ID_ARRAY_TYPE = "l"
24
25 VLLM_INVALID_TOKEN_ID = -1
@@ -407,14 +403,14 @@ class Sequence:
403 def __init__(
404 self,
405 seq_id: int,
410 - inputs: "SingletonInputs",
406 + inputs: SingletonInputs,
407 block_size: int,
408 eos_token_id: Optional[int] = None,
409 lora_request: Optional[LoRARequest] = None,
410 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
411 ) -> None:
412 self.seq_id = seq_id
417 - self.inputs = inputs
413 + self.inputs = SingletonInputsAdapter(inputs)
414 self.block_size = block_size
415 self.eos_token_id = eos_token_id
416 self.lora_request = lora_request
@@ -441,59 +437,29 @@ class Sequence:
437 def n_blocks(self) -> int:
438 return (self.get_len() + self.block_size - 1) // self.block_size
439
444 - @cached_property
440 + @property
441 def prompt(self) -> Optional[str]:
446 - inputs = self.inputs
447 -
448 - if inputs["type"] == "token":
449 - return inputs.get("prompt")
442 + return self.inputs.prompt
443
451 - assert_never(inputs)
452 -
453 - @cached_property
444 + @property
445 def prompt_token_ids(self) -> List[int]:
455 - inputs = self.inputs
456 -
457 - if inputs["type"] == "token":
458 - return inputs.get("prompt_token_ids", [])
446 + return self.inputs.prompt_token_ids
447
460 - assert_never(inputs)
461 -
462 - @cached_property
448 + @property
449 def prompt_embeds(self) -> Optional[torch.Tensor]:
464 - inputs = self.inputs
465 -
466 - if inputs["type"] == "token":
467 - return None
468 -
469 - assert_never(inputs)
450 + return self.inputs.prompt_embeds
451
471 - @cached_property
452 + @property
453 def multi_modal_data(self) -> "MultiModalDataDict":
473 - inputs = self.inputs
474 -
475 - if inputs["type"] == "token":
476 - return inputs.get("multi_modal_data", {})
477 -
478 - assert_never(inputs)
479 -
480 - @cached_property
481 - def mm_processor_kwargs(self) -> Dict[str, Any]:
482 - inputs = self.inputs
483 -
484 - if inputs["type"] == "token":
485 - return inputs.get("mm_processor_kwargs", {})
486 -
487 - assert_never(inputs)
454 + return self.inputs.multi_modal_data
455
456 @property
457 def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
491 - inputs = self.inputs
492 -
493 - if inputs["type"] == "token":
494 - return inputs.get("multi_modal_placeholders", {})
458 + return self.inputs.multi_modal_placeholders
459
496 - assert_never(inputs)
460 + @property
461 + def mm_processor_kwargs(self) -> Dict[str, Any]:
462 + return self.inputs.mm_processor_kwargs
463
464 @property
465 def lora_int_id(self) -> int:
vllm/v1/engine/async_llm.py
+4
@@ -6,6 +6,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
6 from vllm.engine.metrics_types import StatLoggerBase
7 from vllm.engine.protocol import EngineClient
8 from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9 +from vllm.inputs.preprocess import InputPreprocessor
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -321,6 +322,9 @@ class AsyncLLM(EngineClient):
322 async def get_decoding_config(self):
323 raise ValueError("Not Supported on V1 yet.")
324
325 + async def get_input_preprocessor(self) -> InputPreprocessor:
326 + return self.processor.input_preprocessor
327 +
328 async def get_tokenizer(
329 self,
330 lora_request: Optional[LoRARequest] = None,
vllm/v1/engine/llm_engine.py
+3 -1
@@ -7,6 +7,7 @@ from vllm.envs import VLLM_ENABLE_V1_MULTIPROCESSING
7 from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
8 from vllm.logger import init_logger
9 from vllm.lora.request import LoRARequest
10 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
11 from vllm.outputs import RequestOutput
12 from vllm.pooling_params import PoolingParams
13 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -32,6 +33,7 @@ class LLMEngine:
33 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
34 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
35 input_registry: InputRegistry = INPUT_REGISTRY,
36 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
37 use_cached_outputs: bool = False,
38 multiprocess_mode: bool = False,
39 ) -> None:
@@ -50,7 +52,7 @@ class LLMEngine:
52 # Processor (convert Inputs --> EngineCoreRequests)
53 self.processor = Processor(vllm_config.model_config,
54 vllm_config.lora_config, self.tokenizer,
53 - input_registry)
55 + input_registry, mm_registry)
56
57 # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
58 self.detokenizer = Detokenizer(
vllm/v1/engine/processor.py
+55 -18
@@ -2,15 +2,17 @@ import time
2 from typing import Any, Dict, Mapping, Optional, Tuple, Union
3
4 from vllm.config import LoRAConfig, ModelConfig
5 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
6 - EncoderDecoderLLMInputs, InputRegistry, PromptType)
5 +from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
6 + PromptType, SingletonInputsAdapter)
7 +from vllm.inputs.parse import is_encoder_decoder_inputs
8 from vllm.inputs.preprocess import InputPreprocessor
9 from vllm.lora.request import LoRARequest
10 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
11 from vllm.pooling_params import PoolingParams
12 from vllm.prompt_adapter.request import PromptAdapterRequest
13 from vllm.sampling_params import SamplingParams
14 from vllm.transformers_utils.config import try_get_generation_config
13 -from vllm.transformers_utils.tokenizer_group import AnyTokenizer
15 +from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16 from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
17
18
@@ -20,8 +22,9 @@ class Processor:
22 self,
23 model_config: ModelConfig,
24 lora_config: Optional[LoRAConfig],
23 - tokenizer: AnyTokenizer,
25 + tokenizer: BaseTokenizerGroup,
26 input_registry: InputRegistry = INPUT_REGISTRY,
27 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
28 ):
29
30 self.model_config = model_config
@@ -31,7 +34,8 @@ class Processor:
34 self.generation_config_fields = _load_generation_config_dict(
35 model_config)
36 self.input_preprocessor = InputPreprocessor(model_config,
34 - self.tokenizer)
37 + self.tokenizer,
38 + mm_registry)
39 self.input_processor = input_registry.create_input_processor(
40 model_config)
41
@@ -73,6 +77,19 @@ class Processor:
77 self._validate_model_inputs(processed_inputs)
78 eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
79
80 + if is_encoder_decoder_inputs(processed_inputs):
81 + decoder_inputs = SingletonInputsAdapter(
82 + processed_inputs["decoder"])
83 + encoder_inputs = SingletonInputsAdapter(
84 + processed_inputs["encoder"])
85 + else:
86 + decoder_inputs = SingletonInputsAdapter(processed_inputs)
87 + encoder_inputs = None
88 +
89 + # TODO: Impl encoder-decoder
90 + if encoder_inputs is not None:
91 + raise NotImplementedError
92 +
93 assert isinstance(params, SamplingParams)
94 # TODO: can we avoid cloning here in multiproc case
95 sampling_params = params.clone()
@@ -81,27 +98,43 @@ class Processor:
98
99 # Make Request for Detokenizer.
100 detokenizer_request = DetokenizerRequest(
84 - request_id, processed_inputs.get("prompt"),
85 - processed_inputs.get("prompt_token_ids"),
101 + request_id,
102 + decoder_inputs.prompt,
103 + decoder_inputs.prompt_token_ids,
104 sampling_params.skip_special_tokens,
105 sampling_params.spaces_between_special_tokens,
88 - sampling_params.output_kind, sampling_params.stop,
89 - sampling_params.include_stop_str_in_output)
106 + sampling_params.output_kind,
107 + sampling_params.stop,
108 + sampling_params.include_stop_str_in_output,
109 + )
110
111 # Make Request for EngineCore.
112 engine_core_request = EngineCoreRequest(
93 - request_id, processed_inputs.get("prompt"),
94 - processed_inputs.get("prompt_token_ids"),
95 - processed_inputs.get("multi_modal_data"),
96 - processed_inputs.get("multi_modal_placeholders"),
97 - processed_inputs.get("mm_processor_kwargs"), sampling_params,
98 - eos_token_id, arrival_time, lora_request)
113 + request_id,
114 + decoder_inputs.prompt,
115 + decoder_inputs.prompt_token_ids,
116 + decoder_inputs.multi_modal_data,
117 + decoder_inputs.multi_modal_placeholders,
118 + decoder_inputs.mm_processor_kwargs,
119 + sampling_params,
120 + eos_token_id,
121 + arrival_time,
122 + lora_request,
123 + )
124
125 return detokenizer_request, engine_core_request
126
102 - def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
103 - EncoderDecoderLLMInputs]):
104 - prompt_ids = inputs.get("prompt_token_ids")
127 + def _validate_model_inputs(self, inputs: ProcessorInputs):
128 + if is_encoder_decoder_inputs(inputs):
129 + # For encoder-decoder multimodal models, the max_prompt_len
130 + # restricts the decoder prompt length
131 + prompt_inputs = inputs["decoder" if self.model_config.
132 + is_multimodal_model else "encoder"]
133 + else:
134 + prompt_inputs = inputs
135 +
136 + prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids
137 +
138 if prompt_ids is None or len(prompt_ids) == 0:
139 raise ValueError("Prompt cannot be empty")
140
@@ -117,6 +150,10 @@ class Processor:
150 "inputs, the number of image tokens depends on the number "
151 "of images, and possibly their aspect ratios as well.")
152
153 + # TODO: Find out how many placeholder tokens are there so we can
154 + # check that chunked prefill does not truncate them
155 + # max_batch_len = self.scheduler_config.max_num_batched_tokens
156 +
157
158 def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
159 config = try_get_generation_config(
vllm/v1/request.py
+11 -15
@@ -1,7 +1,7 @@
1 import enum
2 -from typing import TYPE_CHECKING, List, Optional, Union
2 +from typing import List, Optional, Union
3
4 -from vllm.inputs.data import DecoderOnlyInputs
4 +from vllm.inputs import DecoderOnlyInputs, SingletonInputsAdapter, token_inputs
5 from vllm.lora.request import LoRARequest
6 from vllm.multimodal import MultiModalKwargs
7 from vllm.sampling_params import SamplingParams
@@ -9,23 +9,20 @@ from vllm.sequence import RequestMetrics
9 from vllm.v1.engine import EngineCoreRequest
10 from vllm.v1.utils import ConstantList
11
12 -if TYPE_CHECKING:
13 - from vllm.inputs import DecoderOnlyInputs
14 -
12
13 class Request:
14
15 def __init__(
16 self,
17 request_id: str,
21 - inputs: "DecoderOnlyInputs",
18 + inputs: DecoderOnlyInputs,
19 sampling_params: SamplingParams,
20 eos_token_id: Optional[int],
21 arrival_time: float,
22 lora_request: Optional[LoRARequest] = None,
23 ) -> None:
24 self.request_id = request_id
28 - self.inputs = inputs
25 + self.inputs = SingletonInputsAdapter(inputs)
26 self.sampling_params = sampling_params
27 # Because of LoRA, the eos token id can be different for each request.
28 self.eos_token_id = eos_token_id
@@ -41,17 +38,17 @@ class Request:
38 assert sampling_params.max_tokens is not None
39 self.max_tokens = sampling_params.max_tokens
40
44 - self.prompt = inputs.get("prompt")
45 - self.prompt_token_ids = inputs["prompt_token_ids"]
41 + self.prompt = self.inputs.prompt
42 + self.prompt_token_ids = self.inputs.prompt_token_ids
43 self.num_prompt_tokens = len(self.prompt_token_ids)
44 self._output_token_ids: List[int] = []
45 self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46 self.num_computed_tokens = 0
47
48 # Raw multimodal data before the mm input mapper (e.g., PIL images).
52 - self.mm_data = inputs.get("multi_modal_data")
53 - self.mm_processor_kwargs = inputs.get("mm_processor_kwargs")
54 - mm_positions = inputs.get("multi_modal_placeholders")
49 + self.mm_data = self.inputs.multi_modal_data
50 + self.mm_processor_kwargs = self.inputs.mm_processor_kwargs
51 + mm_positions = self.inputs.multi_modal_placeholders
52 if mm_positions:
53 # FIXME(woosuk): Support other modalities.
54 self.mm_positions = mm_positions.get("image", [])
@@ -64,8 +61,7 @@ class Request:
61 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
62 return cls(
63 request_id=request.request_id,
67 - inputs=DecoderOnlyInputs(
68 - type="token",
64 + inputs=token_inputs(
65 prompt_token_ids=request.prompt_token_ids,
66 prompt=request.prompt,
67 multi_modal_data=request.mm_data,
@@ -114,7 +110,7 @@ class Request:
110 return RequestStatus.get_finished_reason(self.status)
111
112 def has_encoder_inputs(self) -> bool:
117 - return self.mm_data is not None
113 + return len(self.mm_data) > 0
114
115 @property
116 def num_encoder_inputs(self) -> int:
vllm/v1/worker/gpu_model_runner.py
+1 -1
@@ -28,7 +28,7 @@ from vllm.v1.outputs import ModelRunnerOutput
28 from vllm.v1.sample.metadata import SamplingMetadata
29
30 if TYPE_CHECKING:
31 - from vllm.multimodal.base import PlaceholderRange
31 + from vllm.multimodal.inputs import PlaceholderRange
32 from vllm.v1.core.scheduler import SchedulerOutput
33
34 logger = init_logger(__name__)
vllm/worker/cpu_model_runner.py
+27 -14
@@ -148,19 +148,29 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
148 query_lens=seq_lens,
149 )
150
151 - def _compute_multi_modal_input(self, seq_group: SequenceGroupMetadata,
152 - seq_data: SequenceData, computed_len: int,
153 - mm_processor_kwargs: Dict[str, Any]):
154 -
151 + def _compute_multi_modal_input(
152 + self,
153 + seq_data: SequenceData,
154 + computed_len: int,
155 + seq_group_metadata: SequenceGroupMetadata,
156 + ):
157 # NOTE: mm_data only includes the subset of multi-modal items that
158 # intersect with the current prefill positions.
159 mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
158 - seq_group, range(computed_len, len(seq_data.get_token_ids())))
160 + seq_group_metadata,
161 + range(computed_len, len(seq_data.get_token_ids())),
162 + )
163
164 if not mm_data:
161 - return
165 + return None, None, None
166
163 - mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs)
167 + if self.runner.mm_registry.has_processor(self.runner.model_config):
168 + mm_kwargs = mm_data
169 + else:
170 + mm_kwargs = self.multi_modal_input_mapper(
171 + mm_data,
172 + seq_group_metadata.mm_processor_kwargs,
173 + )
174
175 # special processing for mrope position deltas.
176 mrope_positions = None
@@ -202,7 +212,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
212
213 slot_mapping: List[int] = []
214 seq_lens: List[int] = []
205 - multi_model_kwargs_list: List[MultiModalKwargs] = []
215 + multi_modal_kwargs_list: List[MultiModalKwargs] = []
216 multi_modal_placeholder_maps: Dict[
217 str,
218 MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -223,11 +233,14 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
233
234 mrope_positions = None
235 if seq_group_metadata.multi_modal_data:
226 - mm_kwargs, placeholder_maps, mrope_positions = self \
227 - ._compute_multi_modal_input(
228 - seq_group_metadata, seq_data, computed_len,
229 - seq_group_metadata.mm_processor_kwargs)
230 - multi_model_kwargs_list.append(mm_kwargs)
236 + (
237 + mm_kwargs,
238 + placeholder_maps,
239 + mrope_positions,
240 + ) = self._compute_multi_modal_input(seq_data, computed_len,
241 + seq_group_metadata)
242 +
243 + multi_modal_kwargs_list.append(mm_kwargs)
244 for modality, placeholder_map in placeholder_maps.items():
245 multi_modal_placeholder_maps[modality].extend(
246 placeholder_map)
@@ -302,7 +315,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
315 multi_modal_placeholder_index_maps=placeholder_index_maps,
316 )
317
305 - multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
318 + multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
319
320 return (input_tokens, input_positions, attn_metadata, seq_lens,
321 multi_modal_kwargs)
vllm/worker/hpu_model_runner.py
+3 -3
@@ -716,7 +716,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
716 context_lens: List[int] = []
717 query_lens: List[int] = []
718 prefix_block_tables: List[List[int]] = []
719 - multi_model_kwargs_list: List[MultiModalKwargs] = []
719 + multi_modal_kwargs_list: List[MultiModalKwargs] = []
720
721 if len(seq_group_metadata_list) == 0:
722 return PreparePromptMetadata.empty()
@@ -777,7 +777,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
777 mm_data = seq_group_metadata.multi_modal_data
778 if mm_data:
779 mm_kwargs = self.multi_modal_input_mapper(mm_data)
780 - multi_model_kwargs_list.append(mm_kwargs)
780 + multi_modal_kwargs_list.append(mm_kwargs)
781
782 if seq_group_metadata.block_tables is None:
783 # During memory profiling, the block tables are not initialized
@@ -876,7 +876,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
876 multi_modal_placeholder_index_maps=
877 None # FIXME(kzawora): mutli-modality will not work here
878 )
879 - multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
879 + multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
880
881 return PreparePromptMetadata(input_tokens=input_tokens,
882 input_positions=input_positions,
vllm/worker/model_runner.py
+15 -10
@@ -252,7 +252,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
252 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
253
254 # Multi-modal inputs.
255 - multi_model_kwargs: Optional[MultiModalKwargs] = None,
255 + multi_modal_kwargs: Optional[MultiModalKwargs] = None,
256 multi_modal_placeholder_maps: Optional[Dict[
257 str, MultiModalPlaceholderMap]] = None,
258
@@ -373,7 +373,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
373 prompt_adapter_prompt_mapping or [])
374
375 self.prompt_adapter_request = prompt_adapter_request
376 - self.multi_model_kwargs = multi_model_kwargs
376 + self.multi_modal_kwargs = multi_modal_kwargs
377 self.multi_modal_placeholder_maps = multi_modal_placeholder_maps
378 self.prefix_cache_hit = prefix_cache_hit
379
@@ -661,10 +661,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
661 if not mm_data:
662 return
663
664 - mm_kwargs = self.multi_modal_input_mapper(
665 - mm_data,
666 - mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
667 - inter_data.multi_model_kwargs = mm_kwargs
664 + if self.runner.mm_registry.has_processor(self.runner.model_config):
665 + mm_kwargs = mm_data
666 + else:
667 + mm_kwargs = self.multi_modal_input_mapper(
668 + mm_data,
669 + seq_group_metadata.mm_processor_kwargs,
670 + )
671 +
672 + inter_data.multi_modal_kwargs = mm_kwargs
673 inter_data.multi_modal_placeholder_maps = placeholder_maps
674
675 # special processing for mrope position deltas.
@@ -938,11 +943,11 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
943 )
944
945 # Multi-modal data.
941 - multi_model_kwargs_list = [
942 - data.multi_model_kwargs for data in self.inter_data_list
943 - if data.multi_model_kwargs is not None
946 + multi_modal_kwargs_list = [
947 + data.multi_modal_kwargs for data in self.inter_data_list
948 + if data.multi_modal_kwargs is not None
949 ]
945 - multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
950 + multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
951
952 return self.model_input_cls(
953 input_tokens=input_tokens_tensor,
vllm/worker/neuron_model_runner.py
+13 -9
@@ -67,7 +67,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
67 self.pin_memory = is_pin_memory_available()
68
69 # Multi-modal data support
70 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
70 + self.mm_registry = MULTIMODAL_REGISTRY
71 + self.multi_modal_input_mapper = self.mm_registry \
72 .create_input_mapper(self.model_config)
73
74 # Lazy initialization.
@@ -122,7 +123,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
123 input_block_ids: List[int] = []
124
125 seq_lens: List[int] = []
125 - multi_model_kwargs_list: List[MultiModalKwargs] = []
126 + multi_modal_kwargs_list: List[MultiModalKwargs] = []
127 for seq_group_metadata in seq_group_metadata_list:
128 assert seq_group_metadata.is_prompt
129 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -144,12 +145,15 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
145
146 mm_data = seq_group_metadata.multi_modal_data
147 if mm_data:
147 - # Process multi-modal data
148 - mm_kwargs = self.multi_modal_input_mapper(
149 - mm_data,
150 - mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs,
151 - )
152 - multi_model_kwargs_list.append(mm_kwargs)
148 + if self.mm_registry.has_processor(self.model_config):
149 + mm_kwargs = mm_data
150 + else:
151 + mm_kwargs = self.multi_modal_input_mapper(
152 + mm_data,
153 + seq_group_metadata.mm_processor_kwargs,
154 + )
155 +
156 + multi_modal_kwargs_list.append(mm_kwargs)
157
158 max_seq_len = max(seq_lens)
159 assert max_seq_len > 0
@@ -167,7 +171,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
171 dtype=torch.long,
172 device=self.device)
173
170 - multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
174 + multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
175
176 return (input_tokens, input_positions, input_block_ids, seq_lens,
177 multi_modal_kwargs)
vllm/worker/openvino_model_runner.py
+13 -8
@@ -70,7 +70,8 @@ class OpenVINOModelRunner(ModelRunnerBase):
70 )
71
72 # Multi-modal data support
73 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
73 + self.mm_registry = MULTIMODAL_REGISTRY
74 + self.multi_modal_input_mapper = self.mm_registry \
75 .create_input_mapper(self.model_config)
76
77 # Lazy initialization.
@@ -102,7 +103,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
103 seq_lens: List[int] = []
104 past_lens: List[int] = []
105 query_lens: List[int] = []
105 - multi_model_kwargs_list: List[MultiModalKwargs] = []
106 + multi_modal_kwargs_list: List[MultiModalKwargs] = []
107 multi_modal_placeholder_maps: Dict[
108 str,
109 MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -222,11 +223,15 @@ class OpenVINOModelRunner(ModelRunnerBase):
223 mm_data, placeholder_maps = MultiModalPlaceholderMap \
224 .from_seq_group(seq_group_metadata, positions_range)
225
225 - mm_kwargs = self.multi_modal_input_mapper(
226 - mm_data,
227 - mm_processor_kwargs=seq_group_metadata.
228 - mm_processor_kwargs)
229 - multi_model_kwargs_list.append(mm_kwargs)
226 + if self.mm_registry.has_processor(self.model_config):
227 + mm_kwargs = mm_data
228 + else:
229 + mm_kwargs = self.multi_modal_input_mapper(
230 + mm_data,
231 + seq_group_metadata.mm_processor_kwargs,
232 + )
233 +
234 + multi_modal_kwargs_list.append(mm_kwargs)
235
236 for modality, placeholder_map in placeholder_maps.items():
237 multi_modal_placeholder_maps[modality].extend(
@@ -275,7 +280,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
280 multi_modal_placeholder_index_maps=placeholder_index_maps,
281 )
282
278 - multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
283 + multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
284
285 return ModelInput(
286 input_tokens,
vllm/worker/xpu_model_runner.py
+12 -4
@@ -160,7 +160,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
160 input_positions: List[int] = []
161 slot_mapping: List[int] = []
162 seq_lens: List[int] = []
163 - multi_model_kwargs_list: List[MultiModalKwargs] = []
163 + multi_modal_kwargs_list: List[MultiModalKwargs] = []
164 multi_modal_placeholder_maps: Dict[
165 str,
166 MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -191,8 +191,16 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
191 mm_data, placeholder_maps = MultiModalPlaceholderMap \
192 .from_seq_group(seq_group_metadata, positions_range)
193
194 - mm_kwargs = self.runner.multi_modal_input_mapper(mm_data)
195 - multi_model_kwargs_list.append(mm_kwargs)
194 + if self.runner.mm_registry.has_processor(
195 + self.runner.model_config):
196 + mm_kwargs = mm_data
197 + else:
198 + mm_kwargs = self.runner.multi_modal_input_mapper(
199 + mm_data,
200 + seq_group_metadata.mm_processor_kwargs,
201 + )
202 +
203 + multi_modal_kwargs_list.append(mm_kwargs)
204
205 for modality, placeholder_map in placeholder_maps.items():
206 multi_modal_placeholder_maps[modality].extend(
@@ -264,7 +272,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
272 block_tables=torch.tensor([], device=self.device, dtype=torch.int),
273 )
274
267 - multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
275 + multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
276
277 return (input_tokens, input_positions, attn_metadata, seq_lens,
278 multi_modal_kwargs)