A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[1/N] Initial prototype for multi-modal processor (#10044)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 13, 2024 at 20:39 UTC
0b8bb86bf19d68950b4d92a99350e07a26ec0d2c
48 files changed
+1133
-437
docs/source/models/enabling_multimodal_inputs.rst
+1
-1
@@ -66,7 +66,7 @@ A default mapper is available for each modality in the core vLLM library. This i
66
3. Register maximum number of multi-modal tokens
67
------------------------------------------------
68
69
-For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data instance
69
+For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data item
70
and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
71
72
.. code-block:: diff
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen.py
+1
-1
@@ -6,7 +6,7 @@ import torch
6
from PIL.Image import Image
7
8
from vllm.inputs import InputContext, token_inputs
9
-from vllm.multimodal.base import MultiModalKwargs
9
+from vllm.multimodal import MultiModalKwargs
10
from vllm.multimodal.utils import cached_get_tokenizer
11
12
from .....conftest import IMAGE_ASSETS
tests/multimodal/test_inputs.py
renamed
+1
-1
@@ -1,6 +1,6 @@
1
import torch
2
3
-from vllm.multimodal.base import MultiModalKwargs, NestedTensors
3
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
4
5
6
def assert_nested_tensors_equal(expected: NestedTensors,
tests/multimodal/test_processor_kwargs.py
+23
-14
@@ -1,12 +1,12 @@
1
from array import array
2
-from typing import Mapping
2
+from typing import Callable, Dict, Mapping, Optional
3
from unittest.mock import patch
4
5
import pytest
6
import torch
7
8
from vllm.inputs import (DecoderOnlyInputs, DummyData, InputContext,
9
- InputRegistry, token_inputs)
9
+ InputRegistry, ProcessorInputs, token_inputs)
10
from vllm.multimodal import MultiModalRegistry
11
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
12
@@ -34,10 +34,9 @@ def use_processor_mock():
34
inputs: DecoderOnlyInputs,
35
*,
36
num_crops=DEFAULT_NUM_CROPS):
37
- # For testing purposes, we don't worry about the llm inputs / return
38
- # type validation, and just return the value of the kwarg that we
39
- # clobber.
40
- return num_crops
37
+ # For testing purposes, we don't worry about the prompt
38
+ return token_inputs(prompt_token_ids=[],
39
+ mm_processor_kwargs={"num_crops": num_crops})
40
41
with patch("vllm.inputs.registry.InputRegistry._get_model_input_processor",
42
return_value=custom_processor):
@@ -109,6 +108,21 @@ def _get_num_crops_info(init_num_crops: int, inference_num_crops: int):
108
return init_kwargs, inference_kwargs, expected_seq_count
109
110
111
+def _get_processed_num_crops(
112
+ processor: Callable[[ProcessorInputs], ProcessorInputs],
113
+ inference_kwargs: Optional[Dict[str, int]],
114
+) -> int:
115
+ processed_inputs = processor(
116
+ token_inputs(prompt_token_ids=[],
117
+ prompt="",
118
+ mm_processor_kwargs=inference_kwargs))
119
+
120
+ assert "type" in processed_inputs
121
+ assert processed_inputs["type"] == "token"
122
+ assert "mm_processor_kwargs" in processed_inputs
123
+ return processed_inputs["mm_processor_kwargs"]["num_crops"]
124
+
125
+
126
@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
127
(None, None),
128
(NUM_CROPS_OVERRIDE, None),
@@ -124,10 +138,8 @@ def test_input_processor_kwargs(use_processor_mock, init_num_crops,
138
139
ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
140
processor = dummy_registry.create_input_processor(ctx.model_config)
127
- num_crops_val = processor(
128
- token_inputs(prompt_token_ids=[],
129
- prompt="",
130
- mm_processor_kwargs=inference_kwargs))
141
+ num_crops_val = _get_processed_num_crops(processor, inference_kwargs)
142
+
143
assert num_crops_val == expected_seq_count
144
145
@@ -153,10 +165,7 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
165
166
processor = dummy_registry.create_input_processor(ctx.model_config)
167
# Should filter out the inference time kwargs
156
- num_crops_val = processor(
157
- token_inputs(prompt_token_ids=[],
158
- prompt="",
159
- mm_processor_kwargs=mm_processor_kwargs))
168
+ num_crops_val = _get_processed_num_crops(processor, mm_processor_kwargs)
169
assert num_crops_val == DEFAULT_NUM_CROPS
170
171
tests/v1/core/test_prefix_caching.py
+2
-2
@@ -1,5 +1,5 @@
1
"""Compare the with and without prefix caching."""
2
-from vllm.inputs import DecoderOnlyInputs
2
+from vllm.inputs import token_inputs
3
from vllm.sampling_params import SamplingParams
4
from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
5
from vllm.v1.core.kv_cache_utils import hash_block_tokens
@@ -8,7 +8,7 @@ from vllm.v1.core.kv_cache_utils import hash_block_tokens
8
def make_request(request_id, prompt_token_ids):
9
return Request(
10
request_id=request_id,
11
- inputs=DecoderOnlyInputs(prompt_token_ids=prompt_token_ids),
11
+ inputs=token_inputs(prompt_token_ids=prompt_token_ids),
12
sampling_params=SamplingParams(max_tokens=17),
13
eos_token_id=100,
14
arrival_time=0,
vllm/config.py
+1
-1
@@ -107,7 +107,7 @@ class ModelConfig:
107
matches the model name exposed via the APIs. If multiple model
108
names provided, the first name will be used. If not specified,
109
the model name will be the same as `model`.
110
- limit_mm_per_prompt: Maximum number of data instances per modality
110
+ limit_mm_per_prompt: Maximum number of data items per modality
111
per prompt. Only applicable for multimodal models.
112
override_neuron_config: Initialize non default neuron config or
113
override default neuron config that are specific to Neuron devices,
vllm/engine/async_llm_engine.py
+4
@@ -19,6 +19,7 @@ from vllm.executor.executor_base import ExecutorAsyncBase
19
from vllm.executor.gpu_executor import GPUExecutorAsync
20
from vllm.executor.ray_utils import initialize_ray_cluster
21
from vllm.inputs import PromptType
22
+from vllm.inputs.preprocess import InputPreprocessor
23
from vllm.logger import init_logger
24
from vllm.lora.request import LoRARequest
25
from vllm.model_executor.guided_decoding import (
@@ -729,6 +730,9 @@ class AsyncLLMEngine(EngineClient):
730
self.set_errored(exc)
731
self._request_tracker.propagate_exception(exc)
732
733
+ async def get_input_preprocessor(self) -> InputPreprocessor:
734
+ return self.engine.input_preprocessor
735
+
736
async def get_tokenizer(
737
self,
738
lora_request: Optional[LoRARequest] = None,
vllm/engine/llm_engine.py
+6
-10
@@ -30,7 +30,7 @@ from vllm.executor.executor_base import ExecutorBase
30
from vllm.executor.gpu_executor import GPUExecutor
31
from vllm.executor.ray_utils import initialize_ray_cluster
32
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
33
- PromptType)
33
+ PromptType, SingletonInputsAdapter)
34
from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
35
from vllm.inputs.preprocess import InputPreprocessor
36
from vllm.logger import init_logger
@@ -39,6 +39,7 @@ from vllm.lora.request import LoRARequest
39
from vllm.model_executor.guided_decoding import (
40
get_local_guided_decoding_logits_processor)
41
from vllm.model_executor.layers.sampler import SamplerOutput
42
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
43
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
44
RequestOutputFactory)
45
from vllm.pooling_params import PoolingParams
@@ -226,6 +227,7 @@ class LLMEngine:
227
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
228
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
229
input_registry: InputRegistry = INPUT_REGISTRY,
230
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
231
use_cached_outputs: bool = False,
232
) -> None:
233
@@ -335,7 +337,8 @@ class LLMEngine:
337
model_config)
338
339
self.input_preprocessor = InputPreprocessor(model_config,
338
- self.tokenizer)
340
+ self.tokenizer,
341
+ mm_registry)
342
343
self.input_registry = input_registry
344
self.input_processor = input_registry.create_input_processor(
@@ -851,13 +854,6 @@ class LLMEngine:
854
)
855
processed_inputs = self.input_processor(preprocessed_inputs)
856
854
- # This is a bit of a hack - copy the mm_processor_kwargs that were
855
- # used in the input processor to the processed output, since these
856
- # kwargs are presumed to be immutable and the values should be aligned
857
- # between the input processor (here) and the input mapper.
858
- processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
859
- "mm_processor_kwargs")
860
-
857
self._add_processed_request(
858
request_id=request_id,
859
processed_inputs=processed_inputs,
@@ -2019,7 +2015,7 @@ class LLMEngine:
2015
else:
2016
prompt_inputs = inputs
2017
2022
- prompt_ids = prompt_inputs.get("prompt_token_ids")
2018
+ prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids
2019
2020
if prompt_ids is None or len(prompt_ids) == 0:
2021
raise ValueError("Prompt cannot be empty")
vllm/engine/multiprocessing/client.py
+6
@@ -31,6 +31,7 @@ from vllm.engine.protocol import EngineClient
31
# yapf: enable
32
from vllm.envs import VLLM_RPC_TIMEOUT
33
from vllm.inputs import PromptType
34
+from vllm.inputs.preprocess import InputPreprocessor
35
from vllm.logger import init_logger
36
from vllm.lora.request import LoRARequest
37
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -94,6 +95,8 @@ class MQLLMEngineClient(EngineClient):
95
parallel_config=engine_config.parallel_config,
96
enable_lora=bool(engine_config.lora_config),
97
)
98
+ self.input_preprocessor = InputPreprocessor(self.model_config,
99
+ self.tokenizer)
100
101
# Send RPCGenerateRequest to the MQLLMEngine.
102
self.input_socket: Socket = self.context.socket(zmq.constants.PUSH)
@@ -345,6 +348,9 @@ class MQLLMEngineClient(EngineClient):
348
or response != VLLM_RPC_SUCCESS_STR):
349
raise ValueError(error_message)
350
351
+ async def get_input_preprocessor(self) -> InputPreprocessor:
352
+ return self.input_preprocessor
353
+
354
async def get_tokenizer(self, lora_request: Optional[LoRARequest] = None):
355
return await self.tokenizer.get_lora_tokenizer_async(lora_request)
356
vllm/engine/protocol.py
+11
-5
@@ -62,7 +62,6 @@ class EngineClient(ABC):
62
async def beam_search(
63
self,
64
prompt: PromptType,
65
- model_config: ModelConfig,
65
request_id: str,
66
params: BeamSearchParams,
67
) -> AsyncGenerator[RequestOutput, None]:
@@ -74,13 +73,14 @@ class EngineClient(ABC):
73
length_penalty = params.length_penalty
74
include_stop_str_in_output = params.include_stop_str_in_output
75
77
- tokenizer = await self.get_tokenizer()
78
- input_preprocessor = InputPreprocessor(model_config, tokenizer)
76
+ preprocessor = await self.get_input_preprocessor()
77
+ tokenizer_group = preprocessor.get_tokenizer_group()
78
+ tokenizer = await tokenizer_group.get_lora_tokenizer_async()
79
80
if is_explicit_encoder_decoder_prompt(prompt):
81
raise NotImplementedError
82
else:
83
- processed_inputs = input_preprocessor._prompt_to_llm_inputs(
83
+ processed_inputs = preprocessor._prompt_to_llm_inputs(
84
prompt,
85
request_id=request_id,
86
)
@@ -220,6 +220,7 @@ class EngineClient(ABC):
220
Args:
221
request_id: The unique id of the request.
222
"""
223
+ ...
224
225
@abstractmethod
226
async def get_model_config(self) -> ModelConfig:
@@ -228,8 +229,13 @@ class EngineClient(ABC):
229
230
@abstractmethod
231
async def get_decoding_config(self) -> DecodingConfig:
231
- ...
232
"""Get the decoding configuration of the vLLM engine."""
233
+ ...
234
+
235
+ @abstractmethod
236
+ async def get_input_preprocessor(self) -> InputPreprocessor:
237
+ """Get the input processor of the vLLM engine."""
238
+ ...
239
240
@abstractmethod
241
async def get_tokenizer(
vllm/entrypoints/openai/serving_chat.py
-1
@@ -190,7 +190,6 @@ class OpenAIServingChat(OpenAIServing):
190
if isinstance(sampling_params, BeamSearchParams):
191
generator = self.engine_client.beam_search(
192
prompt=engine_prompt,
193
- model_config=self.model_config,
193
request_id=request_id,
194
params=sampling_params,
195
)
vllm/entrypoints/openai/serving_completion.py
-1
@@ -140,7 +140,6 @@ class OpenAIServingCompletion(OpenAIServing):
140
if isinstance(sampling_params, BeamSearchParams):
141
generator = self.engine_client.beam_search(
142
prompt=engine_prompt,
143
- model_config=self.model_config,
143
request_id=request_id,
144
params=sampling_params,
145
)
vllm/inputs/__init__.py
+8
-4
@@ -1,9 +1,11 @@
1
from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
2
ExplicitEncoderDecoderPrompt, ProcessorInputs, PromptType,
3
- SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
4
- TokensPrompt, build_explicit_enc_dec_prompt,
5
- to_enc_dec_tuple_list, token_inputs, zip_enc_dec_prompts)
6
-from .registry import DummyData, InputContext, InputRegistry
3
+ SingletonInputs, SingletonInputsAdapter, SingletonPrompt,
4
+ TextPrompt, TokenInputs, TokensPrompt,
5
+ build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
6
+ token_inputs, zip_enc_dec_prompts)
7
+from .registry import (DummyData, InputContext, InputProcessingContext,
8
+ InputRegistry)
9
10
INPUT_REGISTRY = InputRegistry()
11
"""
@@ -26,12 +28,14 @@ __all__ = [
28
"EncoderDecoderInputs",
29
"ProcessorInputs",
30
"SingletonInputs",
31
+ "SingletonInputsAdapter",
32
"build_explicit_enc_dec_prompt",
33
"to_enc_dec_tuple_list",
34
"zip_enc_dec_prompts",
35
"INPUT_REGISTRY",
36
"DummyData",
37
"InputContext",
38
+ "InputProcessingContext",
39
"InputRegistry",
40
]
41
vllm/inputs/data.py
+88
-11
@@ -1,10 +1,14 @@
1
+from dataclasses import dataclass
2
+from functools import cached_property
3
from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List, Literal,
4
Optional, Tuple, Union, cast)
5
4
-from typing_extensions import NotRequired, TypedDict, TypeVar
6
+import torch
7
+from typing_extensions import NotRequired, TypedDict, TypeVar, assert_never
8
9
if TYPE_CHECKING:
10
from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
11
+ from vllm.multimodal.inputs import MultiModalInputsV2
12
13
14
class TextPrompt(TypedDict):
@@ -36,13 +40,13 @@ class TokensPrompt(TypedDict):
40
41
multi_modal_data: NotRequired["MultiModalDataDict"]
42
"""
39
- Optional multi-modal data to pass to the model,
43
+ DEPRECATED: Optional multi-modal data to pass to the model,
44
if the model supports it.
45
"""
46
47
mm_processor_kwargs: NotRequired[Dict[str, Any]]
48
"""
45
- Optional multi-modal processor kwargs to be forwarded to the
49
+ DEPRECATED: Optional multi-modal processor kwargs to be forwarded to the
50
multimodal input mapper & processor. Note that if multiple modalities
51
have registered mappers etc for the model being considered, we attempt
52
to pass the mm_processor_kwargs to each of them.
@@ -176,7 +180,7 @@ def token_inputs(
180
return inputs
181
182
179
-DecoderOnlyInputs = TokenInputs
183
+DecoderOnlyInputs = Union[TokenInputs, "MultiModalInputsV2"]
184
"""
185
The inputs in :class:`~vllm.LLMEngine` before they are
186
passed to the model executor.
@@ -191,19 +195,91 @@ class EncoderDecoderInputs(TypedDict):
195
196
This specifies the required data for encoder-decoder models.
197
"""
194
- encoder: TokenInputs
198
+ encoder: Union[TokenInputs, "MultiModalInputsV2"]
199
"""The inputs for the encoder portion."""
200
197
- decoder: TokenInputs
201
+ decoder: Union[TokenInputs, "MultiModalInputsV2"]
202
"""The inputs for the decoder portion."""
203
204
201
-SingletonInputs = TokenInputs
205
+SingletonInputs = Union[TokenInputs, "MultiModalInputsV2"]
206
"""
207
A processed :class:`SingletonPrompt` which can be passed to
208
:class:`vllm.sequence.Sequence`.
209
"""
210
211
+
212
+@dataclass
213
+class SingletonInputsAdapter:
214
+ """
215
+ Unified interface to access the components of :class:`SingletonInputs`.
216
+ """
217
+ inputs: SingletonInputs
218
+
219
+ @cached_property
220
+ def prompt(self) -> Optional[str]:
221
+ inputs = self.inputs
222
+
223
+ if inputs["type"] == "token" or inputs["type"] == "multimodal":
224
+ return inputs.get("prompt")
225
+
226
+ assert_never(inputs)
227
+
228
+ @cached_property
229
+ def prompt_token_ids(self) -> List[int]:
230
+ inputs = self.inputs
231
+
232
+ if inputs["type"] == "token" or inputs["type"] == "multimodal":
233
+ return inputs.get("prompt_token_ids", [])
234
+
235
+ assert_never(inputs)
236
+
237
+ @cached_property
238
+ def prompt_embeds(self) -> Optional[torch.Tensor]:
239
+ inputs = self.inputs
240
+
241
+ if inputs["type"] == "token" or inputs["type"] == "multimodal":
242
+ return None
243
+
244
+ assert_never(inputs)
245
+
246
+ @cached_property
247
+ def multi_modal_data(self) -> "MultiModalDataDict":
248
+ inputs = self.inputs
249
+
250
+ if inputs["type"] == "token":
251
+ return inputs.get("multi_modal_data", {})
252
+
253
+ if inputs["type"] == "multimodal":
254
+ return inputs.get("mm_kwargs", {})
255
+
256
+ assert_never(inputs)
257
+
258
+ @cached_property
259
+ def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
260
+ inputs = self.inputs
261
+
262
+ if inputs["type"] == "token":
263
+ return inputs.get("multi_modal_placeholders", {})
264
+
265
+ if inputs["type"] == "multimodal":
266
+ return inputs.get("mm_placeholders", {})
267
+
268
+ assert_never(inputs)
269
+
270
+ @cached_property
271
+ def mm_processor_kwargs(self) -> Dict[str, Any]:
272
+ inputs = self.inputs
273
+
274
+ if inputs["type"] == "token":
275
+ return inputs.get("mm_processor_kwargs", {})
276
+
277
+ if inputs["type"] == "multimodal":
278
+ return {}
279
+
280
+ assert_never(inputs)
281
+
282
+
283
ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
284
"""
285
The inputs to :data:`vllm.inputs.InputProcessor`.
@@ -234,10 +310,11 @@ def zip_enc_dec_prompts(
310
) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
311
"""
312
Zip encoder and decoder prompts together into a list of
237
- :class:`ExplicitEncoderDecoderPrompt` instances. mm_processor_kwargs
238
- may also be provided; if a dict is passed, the same dictionary will be
239
- used for every encoder/decoder prompt. If an iterable is provided, it will
240
- be zipped with the encoder/decoder prompts.
313
+ :class:`ExplicitEncoderDecoderPrompt` instances.
314
+
315
+ ``mm_processor_kwargs`` may also be provided; if a dict is passed, the same
316
+ dictionary will be used for every encoder/decoder prompt. If an iterable is
317
+ provided, it will be zipped with the encoder/decoder prompts.
318
"""
319
if mm_processor_kwargs is None:
320
mm_processor_kwargs = cast(Dict[str, Any], {})
vllm/inputs/preprocess.py
+122
-21
@@ -1,11 +1,13 @@
1
import asyncio
2
-from typing import List, Optional
2
+from typing import List, Mapping, Optional, Union
3
4
from typing_extensions import assert_never
5
6
from vllm.config import ModelConfig
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
9
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10
+from vllm.multimodal.processing import MultiModalDataDict, MultiModalInputsV2
11
from vllm.prompt_adapter.request import PromptAdapterRequest
12
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
13
from vllm.utils import print_warning_once
@@ -23,11 +25,13 @@ class InputPreprocessor:
25
self,
26
model_config: ModelConfig,
27
tokenizer: Optional[BaseTokenizerGroup],
28
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
29
) -> None:
30
super().__init__()
31
32
self.model_config = model_config
33
self.tokenizer = tokenizer
34
+ self.mm_registry = mm_registry
35
36
def get_tokenizer_group(self) -> BaseTokenizerGroup:
37
if self.tokenizer is None:
@@ -198,14 +202,79 @@ class InputPreprocessor:
202
prompt=prompt,
203
lora_request=lora_request)
204
205
+ def _can_process_multimodal(self) -> bool:
206
+ model_config = self.model_config
207
+
208
+ if not model_config.is_multimodal_model:
209
+ raise ValueError("Your model does not support multi-modal inputs")
210
+
211
+ # Interim measure so we can handle models that have yet to be
212
+ # updated to use the new multi-modal processor
213
+ can_process_multimodal = self.mm_registry.has_processor(model_config)
214
+ if not can_process_multimodal:
215
+ logger.info(
216
+ "Your model uses the legacy input pipeline instead of the new "
217
+ "multi-modal processor. Please note that the legacy pipeline "
218
+ "will be removed in a future release. For more details, see: "
219
+ "https://github.com/vllm-project/vllm/issues/10114")
220
+
221
+ return can_process_multimodal
222
+
223
+ def _process_multimodal(
224
+ self,
225
+ prompt: Union[str, List[int]],
226
+ mm_data: MultiModalDataDict,
227
+ mm_processor_kwargs: Optional[Mapping[str, object]],
228
+ lora_request: Optional[LoRARequest],
229
+ ) -> MultiModalInputsV2:
230
+ """
231
+ Apply the model's multi-modal processor to a multi-modal prompt,
232
+ returning the corresponding token IDs and metadata.
233
+ """
234
+ tokenizer_group = self.get_tokenizer_group()
235
+ tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
236
+
237
+ mm_processor = self.mm_registry.create_processor(
238
+ self.model_config, tokenizer)
239
+
240
+ if isinstance(prompt, list):
241
+ prompt = tokenizer.decode(prompt)
242
+ if mm_processor_kwargs is None:
243
+ mm_processor_kwargs = {}
244
+
245
+ return mm_processor.apply(prompt, mm_data, mm_processor_kwargs)
246
+
247
+ async def _process_multimodal_async(
248
+ self,
249
+ prompt: Union[str, List[int]],
250
+ mm_data: MultiModalDataDict,
251
+ mm_processor_kwargs: Optional[Mapping[str, object]],
252
+ lora_request: Optional[LoRARequest],
253
+ ) -> MultiModalInputsV2:
254
+ """Async version of :meth:`_process_multimodal`."""
255
+ tokenizer_group = self.get_tokenizer_group()
256
+ tokenizer = await tokenizer_group.get_lora_tokenizer_async(lora_request
257
+ )
258
+
259
+ mm_processor = self.mm_registry.create_processor(
260
+ self.model_config, tokenizer)
261
+ if isinstance(prompt, list):
262
+ logger.warning("Passing `multi_modal_data` in TokensPrompt is"
263
+ "deprecated and will be removed in a future update")
264
+ prompt = tokenizer.decode(prompt)
265
+ if mm_processor_kwargs is None:
266
+ mm_processor_kwargs = {}
267
+
268
+ return mm_processor.apply(prompt, mm_data, mm_processor_kwargs)
269
+
270
def _prompt_to_llm_inputs(
271
self,
272
prompt: SingletonPrompt,
273
request_id: str,
274
lora_request: Optional[LoRARequest] = None,
275
) -> SingletonInputs:
207
- '''
208
- Extract the components of any single encoder or decoder input prompt.
276
+ """
277
+ Extract the singleton inputs from a prompt.
278
279
Arguments:
280
@@ -215,12 +284,8 @@ class InputPreprocessor:
284
285
Returns:
286
218
- * prompt
219
- * prompt_token_ids
220
- * multi_modal_data
221
- * mm_processor_kwargs (request-level input processor/mapper overrides)
222
- '''
223
-
287
+ * :class:`SingletonInputs` instance
288
+ """
289
parsed = parse_singleton_prompt(prompt)
290
291
if parsed["type"] == "str":
@@ -243,6 +308,14 @@ class InputPreprocessor:
308
multi_modal_data = tokens_content.get("multi_modal_data")
309
mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
310
311
+ if multi_modal_data is not None and self._can_process_multimodal():
312
+ return self._process_multimodal(
313
+ prompt_token_ids,
314
+ multi_modal_data,
315
+ mm_processor_kwargs,
316
+ lora_request=lora_request,
317
+ )
318
+
319
return token_inputs(
320
prompt_token_ids=prompt_token_ids,
321
multi_modal_data=multi_modal_data,
@@ -253,13 +326,22 @@ class InputPreprocessor:
326
text_content = parsed["content"]
327
328
prompt_text = text_content["prompt"]
329
+ multi_modal_data = text_content.get("multi_modal_data")
330
+ mm_processor_kwargs = text_content.get("mm_processor_kwargs")
331
+
332
+ if multi_modal_data is not None and self._can_process_multimodal():
333
+ return self._process_multimodal(
334
+ prompt_text,
335
+ multi_modal_data,
336
+ mm_processor_kwargs,
337
+ lora_request=lora_request,
338
+ )
339
+
340
prompt_token_ids = self._tokenize_prompt(
341
prompt_text,
342
request_id=request_id,
343
lora_request=lora_request,
344
)
261
- multi_modal_data = text_content.get("multi_modal_data")
262
- mm_processor_kwargs = text_content.get("mm_processor_kwargs")
345
346
return token_inputs(
347
prompt=prompt_text,
@@ -299,6 +381,14 @@ class InputPreprocessor:
381
multi_modal_data = tokens_content.get("multi_modal_data")
382
mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
383
384
+ if multi_modal_data is not None and self._can_process_multimodal():
385
+ return await self._process_multimodal_async(
386
+ prompt_token_ids,
387
+ multi_modal_data,
388
+ mm_processor_kwargs,
389
+ lora_request=lora_request,
390
+ )
391
+
392
return token_inputs(
393
prompt_token_ids=prompt_token_ids,
394
multi_modal_data=multi_modal_data,
@@ -309,13 +399,22 @@ class InputPreprocessor:
399
text_content = parsed["content"]
400
401
prompt_text = text_content["prompt"]
402
+ multi_modal_data = text_content.get("multi_modal_data")
403
+ mm_processor_kwargs = text_content.get("mm_processor_kwargs")
404
+
405
+ if multi_modal_data is not None and self._can_process_multimodal():
406
+ return await self._process_multimodal_async(
407
+ prompt_text,
408
+ multi_modal_data,
409
+ mm_processor_kwargs,
410
+ lora_request=lora_request,
411
+ )
412
+
413
prompt_token_ids = await self._tokenize_prompt_async(
414
prompt_text,
415
request_id=request_id,
416
lora_request=lora_request,
417
)
317
- multi_modal_data = text_content.get("multi_modal_data")
318
- mm_processor_kwargs = text_content.get("mm_processor_kwargs")
418
419
return token_inputs(
420
prompt=prompt_text,
@@ -331,7 +430,8 @@ class InputPreprocessor:
430
encoder_inputs: SingletonInputs,
431
decoder_inputs: Optional[SingletonInputs],
432
) -> EncoderDecoderInputs:
334
- if encoder_inputs["type"] == "token":
433
+ if (encoder_inputs["type"] == "token"
434
+ or encoder_inputs["type"] == "multimodal"):
435
pass
436
else:
437
assert_never(encoder_inputs)
@@ -340,7 +440,8 @@ class InputPreprocessor:
440
dec_token_ids = self._prepare_decoder_input_ids_for_generation(
441
None)
442
decoder_inputs = token_inputs(dec_token_ids)
343
- elif decoder_inputs["type"] == "token":
443
+ elif (decoder_inputs["type"] == "token"
444
+ or decoder_inputs["type"] == "multimodal"):
445
dec_token_ids = self._prepare_decoder_input_ids_for_generation(
446
decoder_inputs["prompt_token_ids"])
447
decoder_inputs["prompt_token_ids"] = dec_token_ids
@@ -361,7 +462,7 @@ class InputPreprocessor:
462
prompt: PromptType,
463
request_id: str,
464
) -> EncoderDecoderInputs:
364
- '''
465
+ """
466
For encoder/decoder models only:
467
Process an input prompt into an :class:`EncoderDecoderInputs` instance.
468
@@ -391,8 +492,7 @@ class InputPreprocessor:
492
Returns:
493
494
* :class:`EncoderDecoderInputs` instance
394
- '''
395
-
495
+ """
496
encoder_inputs: SingletonInputs
497
decoder_inputs: Optional[SingletonInputs]
498
@@ -460,7 +560,8 @@ class InputPreprocessor:
560
prompt_inputs: DecoderOnlyInputs,
561
prompt_adapter_request: Optional[PromptAdapterRequest],
562
) -> DecoderOnlyInputs:
463
- if prompt_inputs["type"] == "token":
563
+ if (prompt_inputs["type"] == "token"
564
+ or prompt_inputs["type"] == "multimodal"):
565
prompt_inputs["prompt_token_ids"] = self._apply_prompt_adapter(
566
prompt_inputs["prompt_token_ids"],
567
prompt_adapter_request=prompt_adapter_request,
@@ -477,7 +578,7 @@ class InputPreprocessor:
578
lora_request: Optional[LoRARequest] = None,
579
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
580
) -> DecoderOnlyInputs:
480
- '''
581
+ """
582
For decoder-only models:
583
Process an input prompt into an :class:`DecoderOnlyInputs` instance.
584
@@ -491,7 +592,7 @@ class InputPreprocessor:
592
Returns:
593
594
* :class:`DecoderOnlyInputs` instance
494
- '''
595
+ """
596
597
prompt_comps = self._prompt_to_llm_inputs(
598
prompt,
vllm/inputs/registry.py
+50
-6
@@ -5,14 +5,17 @@ from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5
Optional, Protocol, Type, cast)
6
7
from torch import nn
8
-from transformers import PretrainedConfig
9
-from typing_extensions import TypeVar
8
+from transformers import PretrainedConfig, ProcessorMixin
9
+from typing_extensions import TypeVar, assert_never
10
11
from vllm.logger import init_logger
12
+from vllm.transformers_utils.processor import cached_get_processor
13
+from vllm.transformers_utils.tokenizer import AnyTokenizer
14
from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
15
resolve_mm_processor_kwargs)
16
15
-from .data import ProcessorInputs
17
+from .data import ProcessorInputs, SingletonInputs
18
+from .parse import is_encoder_decoder_inputs
19
20
if TYPE_CHECKING:
21
from vllm.config import ModelConfig
@@ -61,6 +64,19 @@ class InputContext:
64
return self.model_config.hf_image_processor_config
65
66
67
+@dataclass(frozen=True)
68
+class InputProcessingContext(InputContext):
69
+ tokenizer: AnyTokenizer
70
+ """The tokenizer used to tokenize the inputs."""
71
+
72
+ def get_hf_processor(self) -> ProcessorMixin:
73
+ return cached_get_processor(
74
+ self.model_config.tokenizer,
75
+ tokenizer=self.tokenizer, # Override the tokenizer with ours
76
+ trust_remote_code=self.model_config.trust_remote_code,
77
+ )
78
+
79
+
80
N = TypeVar("N", bound=Type[nn.Module])
81
82
@@ -94,7 +110,7 @@ class DummyDataFactory(Protocol):
110
...
111
112
97
-class _MultiModalCounts(UserDict):
113
+class _MultiModalCounts(UserDict[str, int]):
114
"""
115
Wraps `mm_counts` for a more informative error message
116
when attempting to access a plugin that does not exist.
@@ -287,6 +303,21 @@ class InputRegistry:
303
return self._input_processors_by_model_type \
304
.get(model_cls, self._default_input_processor)
305
306
+ def _ensure_mm_kwargs(
307
+ self,
308
+ inputs: SingletonInputs,
309
+ mm_processor_kwargs: Dict[str, Any],
310
+ ):
311
+ if inputs["type"] == "token":
312
+ # In case the input processor for that model fails to set it
313
+ if "mm_processor_kwargs" not in inputs:
314
+ inputs["mm_processor_kwargs"] = mm_processor_kwargs
315
+ elif inputs["type"] == "multimodal":
316
+ # Be more strict in V2
317
+ assert "mm_kwargs" in inputs
318
+ else:
319
+ assert_never(inputs["type"])
320
+
321
def process_input(self, model_config: "ModelConfig",
322
inputs: ProcessorInputs) -> ProcessorInputs:
323
"""
@@ -312,8 +343,21 @@ class InputRegistry:
343
processor,
344
)
345
315
- return processor(InputContext(model_config), inputs,
316
- **mm_processor_kwargs)
346
+ processed_inputs = processor(
347
+ InputContext(model_config),
348
+ inputs,
349
+ **mm_processor_kwargs,
350
+ )
351
+
352
+ if is_encoder_decoder_inputs(processed_inputs):
353
+ self._ensure_mm_kwargs(processed_inputs["encoder"],
354
+ mm_processor_kwargs)
355
+ self._ensure_mm_kwargs(processed_inputs["decoder"],
356
+ mm_processor_kwargs)
357
+ else:
358
+ self._ensure_mm_kwargs(processed_inputs, mm_processor_kwargs)
359
+
360
+ return processed_inputs
361
362
def create_input_processor(self, model_config: "ModelConfig"):
363
"""
vllm/model_executor/models/chatglm.py
+2
-2
@@ -30,8 +30,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31
from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
32
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
34
-from vllm.multimodal.base import MultiModalData
33
+from vllm.multimodal import MULTIMODAL_REGISTRY
34
+from vllm.multimodal.inputs import MultiModalData, MultiModalKwargs
35
from vllm.multimodal.utils import cached_get_tokenizer
36
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
37
SequenceData)
vllm/model_executor/models/fuyu.py
+1
-2
@@ -32,8 +32,7 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear
32
from vllm.model_executor.layers.sampler import SamplerOutput
33
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
-from vllm.multimodal import MULTIMODAL_REGISTRY
36
-from vllm.multimodal.base import MultiModalKwargs
35
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
36
from vllm.multimodal.image import cached_get_image_processor
37
from vllm.multimodal.utils import (cached_get_tokenizer,
38
consecutive_placeholder_ranges)
vllm/model_executor/models/h2ovl.py
+1
-2
@@ -15,8 +15,7 @@ from transformers import PretrainedConfig
15
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
16
token_inputs)
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
-from vllm.multimodal import MULTIMODAL_REGISTRY
19
-from vllm.multimodal.base import MultiModalKwargs
18
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
19
from vllm.multimodal.utils import cached_get_tokenizer
20
from vllm.utils import is_list_of
21
vllm/model_executor/models/internvl.py
+1
-2
@@ -25,8 +25,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
-from vllm.multimodal import MULTIMODAL_REGISTRY
29
-from vllm.multimodal.base import MultiModalKwargs
28
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
29
from vllm.multimodal.utils import cached_get_tokenizer
30
from vllm.sequence import IntermediateTensors
31
from vllm.utils import is_list_of
vllm/model_executor/models/llava.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.multimodal.base import NestedTensors
20
+from vllm.multimodal.inputs import NestedTensors
21
from vllm.sequence import IntermediateTensors
22
from vllm.utils import is_list_of
23
vllm/model_executor/models/minicpmv.py
+1
-2
@@ -51,8 +51,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
51
from vllm.model_executor.models.qwen2 import Qwen2Model
52
from vllm.model_executor.models.utils import LLMWrapper
53
from vllm.model_executor.sampling_metadata import SamplingMetadata
54
-from vllm.multimodal import MULTIMODAL_REGISTRY
55
-from vllm.multimodal.base import MultiModalKwargs
54
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
55
from vllm.multimodal.image import cached_get_image_processor
56
from vllm.multimodal.utils import cached_get_tokenizer
57
from vllm.sequence import IntermediateTensors, SequenceData
vllm/model_executor/models/phi3v.py
+1
-1
@@ -39,7 +39,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
39
from vllm.model_executor.pooling_metadata import PoolingMetadata
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
from vllm.multimodal import MULTIMODAL_REGISTRY
42
-from vllm.multimodal.base import NestedTensors, PlaceholderRange
42
+from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
43
from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
44
from vllm.sequence import IntermediateTensors, PoolerOutput
45
from vllm.utils import is_list_of
vllm/model_executor/models/pixtral.py
+1
-2
@@ -29,8 +29,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.models.utils import merge_multimodal_embeddings
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
-from vllm.multimodal import MULTIMODAL_REGISTRY
33
-from vllm.multimodal.base import MultiModalKwargs
32
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
33
from vllm.multimodal.utils import (cached_get_tokenizer,
34
consecutive_placeholder_ranges)
35
from vllm.sequence import IntermediateTensors, SequenceData
vllm/model_executor/models/qwen.py
+1
-2
@@ -42,8 +42,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43
from vllm.model_executor.models.module_mapping import MultiModelKeys
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.multimodal import MULTIMODAL_REGISTRY
46
-from vllm.multimodal.base import MultiModalKwargs
45
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
46
from vllm.multimodal.utils import cached_get_tokenizer
47
from vllm.sequence import IntermediateTensors, SequenceData
48
from vllm.utils import is_list_of
vllm/model_executor/models/qwen2_vl.py
+3
-3
@@ -60,10 +60,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
60
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
61
from vllm.model_executor.models.qwen2 import Qwen2Model
62
from vllm.model_executor.pooling_metadata import PoolingMetadata
63
-from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
64
- MultiModalKwargs)
65
-from vllm.multimodal.base import MultiModalData
63
+from vllm.multimodal import MULTIMODAL_REGISTRY
64
from vllm.multimodal.image import cached_get_image_processor
65
+from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
66
+ MultiModalKwargs)
67
from vllm.multimodal.utils import cached_get_tokenizer
68
from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
69
from vllm.transformers_utils.config import uses_mrope
vllm/model_executor/models/utils.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.config import VllmConfig
15
from vllm.logger import init_logger
16
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.models import ModelRegistry
18
-from vllm.multimodal.base import MultiModalPlaceholderMap, NestedTensors
18
+from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
19
from vllm.platforms import current_platform
20
from vllm.sequence import IntermediateTensors
21
from vllm.utils import is_pin_memory_available
vllm/multimodal/__init__.py
+6
-4
@@ -1,7 +1,8 @@
1
-from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2
- MultiModalDataDict, MultiModalKwargs,
3
- MultiModalPlaceholderDict, MultiModalPlaceholderMap,
4
- MultiModalPlugin, NestedTensors)
1
+from .base import MultiModalPlaceholderMap, MultiModalPlugin
2
+from .inputs import (BatchedTensorInputs, MultiModalData,
3
+ MultiModalDataBuiltins, MultiModalDataDict,
4
+ MultiModalKwargs, MultiModalPlaceholderDict,
5
+ NestedTensors)
6
from .registry import MultiModalRegistry
7
8
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -15,6 +16,7 @@ See also:
16
17
__all__ = [
18
"BatchedTensorInputs",
19
+ "MultiModalData",
20
"MultiModalDataBuiltins",
21
"MultiModalDataDict",
22
"MultiModalKwargs",
vllm/multimodal/audio.py
+9
-3
@@ -1,5 +1,7 @@
1
from vllm.inputs.registry import InputContext
2
-from vllm.multimodal.base import MultiModalKwargs, MultiModalPlugin
2
+
3
+from .base import MultiModalPlugin
4
+from .inputs import AudioItem, MultiModalData, MultiModalKwargs
5
6
7
class AudioPlugin(MultiModalPlugin):
@@ -8,8 +10,12 @@ class AudioPlugin(MultiModalPlugin):
10
def get_data_key(self) -> str:
11
return "audio"
12
11
- def _default_input_mapper(self, ctx: InputContext, data: object,
12
- **mm_processor_kwargs) -> MultiModalKwargs:
13
+ def _default_input_mapper(
14
+ self,
15
+ ctx: InputContext,
16
+ data: MultiModalData[AudioItem],
17
+ **mm_processor_kwargs,
18
+ ) -> MultiModalKwargs:
19
raise NotImplementedError("There is no default audio input mapper")
20
21
def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
vllm/multimodal/base.py
+18
-170
@@ -1,180 +1,23 @@
1
from abc import ABC, abstractmethod
2
-from collections import UserDict, defaultdict
3
-from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
4
- NamedTuple, Optional, Tuple, Type, TypedDict, TypeVar,
5
- Union, cast, final)
6
-
7
-import numpy as np
8
-import torch
9
-import torch.types
10
-from PIL import Image
2
+from collections import defaultdict
3
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
4
+ Optional, Sequence, Tuple, Type, TypeVar, Union)
5
+
6
from torch import nn
12
-from typing_extensions import TypeAlias
7
8
from vllm.inputs import InputContext
9
from vllm.logger import init_logger
16
-from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
17
- json_map_leaves, resolve_mm_processor_kwargs)
10
+from vllm.utils import (get_allowed_kwarg_only_overrides,
11
+ resolve_mm_processor_kwargs)
12
13
if TYPE_CHECKING:
14
from vllm.config import ModelConfig
15
from vllm.sequence import SequenceGroupMetadata
16
23
-logger = init_logger(__name__)
24
-
25
-NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
26
-"""
27
-Uses a list instead of a tensor if the dimensions of each element do not match.
28
-"""
29
-
30
-BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
31
-"""
32
-A dictionary containing nested tensors which have been batched via
33
-:meth:`MultiModalKwargs.batch`.
34
-"""
35
-
36
-
37
-class _MultiModalKwargsBase(UserDict[str, NestedTensors]):
38
- pass
39
-
40
-
41
-class MultiModalKwargs(_MultiModalKwargsBase):
42
- """
43
- A dictionary that represents the keyword arguments to
44
- :meth:`~torch.nn.Module.forward`.
45
- """
46
-
47
- @staticmethod
48
- def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
49
- """
50
- Recursively stacks lists of tensors when they all have the same shape.
51
- """
52
- if isinstance(nested_tensors, torch.Tensor):
53
- return nested_tensors
54
-
55
- if isinstance(nested_tensors, np.ndarray):
56
- return torch.from_numpy(nested_tensors)
57
-
58
- if isinstance(nested_tensors, (int, float)):
59
- return torch.tensor(nested_tensors)
17
+from .inputs import (MultiModalData, MultiModalDataDict, MultiModalKwargs,
18
+ PlaceholderRange)
19
61
- stacked = [MultiModalKwargs._try_stack(t) for t in nested_tensors]
62
- if not is_list_of(stacked, torch.Tensor, check="all"):
63
- # Only tensors (not lists) can be stacked.
64
- return stacked
65
-
66
- tensors_ = cast(List[torch.Tensor], stacked)
67
- if any(t.shape != tensors_[0].shape for t in tensors_):
68
- # The tensors have incompatible shapes and can't be stacked.
69
- return tensors_
70
-
71
- return torch.stack(tensors_)
72
-
73
- @staticmethod
74
- def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
75
- """
76
- Batch multiple inputs together into a dictionary.
77
-
78
- The resulting dictionary has the same keys as the inputs.
79
- If the corresponding value from each input is a tensor and they all
80
- share the same shape, the output value is a single batched tensor;
81
- otherwise, the output value is a list containing the original value
82
- from each input.
83
- """
84
- if len(inputs_list) == 0:
85
- return {}
86
-
87
- item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
88
-
89
- for inputs in inputs_list:
90
- # For models that supports multiple modalities (e.g. Qwen2-VL),
91
- # different modalities will return different data keys,
92
- # so batch() should skip the same key check.
93
-
94
- for k, v in inputs.items():
95
- item_lists[k].append(v)
96
-
97
- return {
98
- k: MultiModalKwargs._try_stack(item_list)
99
- for k, item_list in item_lists.items()
100
- }
101
-
102
- @staticmethod
103
- def as_kwargs(
104
- batched_inputs: BatchedTensorInputs,
105
- *,
106
- device: torch.types.Device,
107
- ) -> BatchedTensorInputs:
108
- json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
109
-
110
- json_mapped = json_map_leaves(
111
- lambda x: x.to(device, non_blocking=True),
112
- json_inputs,
113
- )
114
-
115
- return cast(BatchedTensorInputs, json_mapped)
116
-
117
-
118
-_T = TypeVar("_T")
119
-
120
-MultiModalData: TypeAlias = Union[_T, List[_T]]
121
-"""
122
-Either a single data instance, or a list of data instances.
123
-
124
-The number of data instances allowed per modality is restricted by
125
-`--limit-mm-per-prompt`.
126
-"""
127
-
128
-
129
-@final
130
-class MultiModalDataBuiltins(TypedDict, total=False):
131
- """Modality types that are predefined by vLLM."""
132
-
133
- image: MultiModalData[Image.Image]
134
- """The input image(s)."""
135
-
136
- audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]]
137
- """The input audio item(s) and corresponding sampling rate(s)."""
138
-
139
- video: MultiModalData[Tuple[np.ndarray]]
140
- """The input video(s)."""
141
-
142
-
143
-MultiModalDataDict = Union[MultiModalDataBuiltins,
144
- Mapping[str, MultiModalData[object]]]
145
-"""
146
-A dictionary containing an item for each modality type to input.
147
-
148
-Note:
149
- This dictionary also accepts modality keys defined outside
150
- :class:`MultiModalDataBuiltins` as long as a customized plugin is registered
151
- through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
152
- Read more on that :ref:`here <adding_multimodal_plugin>`.
153
-"""
154
-
155
-
156
-class PlaceholderRange(TypedDict):
157
- """
158
- Placeholder location information for multi-modal data.
159
-
160
- For example:
161
- Prompt: AAAA BBBB What is in these images?
162
- Images A and B will have:
163
- A: { "offset": 0, "length": 4 }
164
- B: { "offset": 5, "length": 4 }
165
- """
166
-
167
- offset: int
168
- """The start index of the placeholder in the prompt."""
169
-
170
- length: int
171
- """The length of the placeholder."""
172
-
173
-
174
-MultiModalPlaceholderDict = Mapping[str, List[PlaceholderRange]]
175
-"""
176
-A dictionary containing placeholder ranges.
177
-"""
20
+logger = init_logger(__name__)
21
22
MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
23
MultiModalKwargs]
@@ -192,6 +35,7 @@ Calculate the maximum number of multimodal tokens input to the language
35
model. This does not include tokens that correspond to the input text.
36
"""
37
38
+_T = TypeVar("_T")
39
N = TypeVar("N", bound=Type[nn.Module])
40
41
@@ -224,7 +68,7 @@ class MultiModalPlugin(ABC):
68
def _default_input_mapper(
69
self,
70
ctx: InputContext,
227
- data: MultiModalData[object],
71
+ data: MultiModalData[Any],
72
**mm_processor_kwargs,
73
) -> MultiModalKwargs:
74
"""
@@ -273,8 +117,8 @@ class MultiModalPlugin(ABC):
117
def map_input(
118
self,
119
model_config: "ModelConfig",
276
- data: MultiModalData[object],
277
- mm_processor_kwargs: Dict[str, Any],
120
+ data: MultiModalData[Any],
121
+ mm_processor_kwargs: Optional[Dict[str, Any]],
122
) -> MultiModalKwargs:
123
"""
124
Transform the data into a dictionary of model inputs using the
@@ -289,6 +133,7 @@ class MultiModalPlugin(ABC):
133
- :ref:`input_processing_pipeline`
134
- :ref:`enabling_multimodal_inputs`
135
"""
136
+
137
# Avoid circular import
138
from vllm.model_executor.model_loader import get_model_architecture
139
@@ -300,6 +145,9 @@ class MultiModalPlugin(ABC):
145
raise KeyError(f"No input mapper in {self} is registered for "
146
f"model class {model_cls.__name__}.")
147
148
+ if mm_processor_kwargs is None:
149
+ mm_processor_kwargs = {}
150
+
151
# In the case of the default mapper, we have to get resource
152
# processor through its HuggingFace autoclass; since this goes
153
# through **kwargs, we can't inspect it the same way, so we allow
@@ -508,7 +356,7 @@ class MultiModalPlaceholderMap:
356
self,
357
positions: range,
358
multi_modal_items: List[_T],
511
- multi_modal_placeholders: List[PlaceholderRange],
359
+ multi_modal_placeholders: Sequence[PlaceholderRange],
360
) -> List[_T]:
361
"""
362
Adds the multi-modal items that intersect ```positions`` to this
vllm/multimodal/image.py
+3
-7
@@ -3,14 +3,14 @@ from typing import TYPE_CHECKING, Any, Dict, Optional
3
4
import torch
5
from PIL import Image
6
-from transformers.image_processing_base import BatchFeature
6
7
from vllm.inputs.registry import InputContext
8
from vllm.logger import init_logger
9
from vllm.transformers_utils.processor import get_image_processor
10
from vllm.utils import is_list_of
11
13
-from .base import MultiModalData, MultiModalKwargs, MultiModalPlugin
12
+from .base import MultiModalPlugin
13
+from .inputs import ImageItem, MultiModalData, MultiModalKwargs
14
15
if TYPE_CHECKING:
16
from vllm.config import ModelConfig
@@ -41,15 +41,11 @@ class ImagePlugin(MultiModalPlugin):
41
def _default_input_mapper(
42
self,
43
ctx: InputContext,
44
- data: MultiModalData[object],
44
+ data: MultiModalData[ImageItem],
45
**mm_processor_kwargs,
46
) -> MultiModalKwargs:
47
model_config = ctx.model_config
48
49
- # Processed by input processor
50
- if isinstance(data, BatchFeature):
51
- return MultiModalKwargs(data.data)
52
-
49
# PIL image
50
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
51
image_processor = self._get_hf_image_processor(
vllm/multimodal/inputs.py
new
+225
@@ -0,0 +1,225 @@
1
+from collections import UserDict, defaultdict
2
+from typing import (Any, Dict, List, Literal, Mapping, Sequence, Tuple,
3
+ TypedDict, TypeVar, Union, cast, final)
4
+
5
+import numpy as np
6
+import torch
7
+import torch.types
8
+from PIL.Image import Image
9
+from typing_extensions import TypeAlias
10
+
11
+from vllm.utils import JSONTree, is_list_of, json_map_leaves
12
+
13
+_T = TypeVar("_T")
14
+
15
+# yapf: disable
16
+ImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
17
+"""
18
+A :class:`transformers.image_utils.ImageInput` representing a single image,
19
+which can be passed to a HuggingFace :code:`ImageProcessor`.
20
+"""
21
+
22
+VideoItem: TypeAlias = Union[
23
+ List[Image],
24
+ np.ndarray,
25
+ torch.Tensor,
26
+ List[np.ndarray],
27
+ List[torch.Tensor],
28
+]
29
+"""
30
+
31
+A :class:`transformers.image_utils.VideoInput` representing a single video,
32
+which can be passed to a HuggingFace :code:`VideoProcessor`.
33
+"""
34
+
35
+AudioItem: TypeAlias = Union[
36
+ np.ndarray,
37
+ List[float],
38
+ Tuple[np.ndarray, float], # DEPRECATED: Use mm_processor_kwargs instead
39
+]
40
+"""
41
+Represents a single audio that can be inputted to a HuggingFace
42
+:code:`AudioProcessor`.
43
+"""
44
+# yapf: enable
45
+
46
+MultiModalData: TypeAlias = Union[_T, List[_T]]
47
+"""
48
+Either a single data item, or a list of data items.
49
+
50
+The number of data items allowed per modality is restricted by
51
+:code:`--limit-mm-per-prompt`.
52
+"""
53
+
54
+
55
+@final
56
+class MultiModalDataBuiltins(TypedDict, total=False):
57
+ """Type annotations for modality types predefined by vLLM."""
58
+
59
+ image: MultiModalData[ImageItem]
60
+ """The input image(s)."""
61
+
62
+ video: MultiModalData[VideoItem]
63
+ """The input video(s)."""
64
+
65
+ audio: MultiModalData[AudioItem]
66
+ """The input audio(s)."""
67
+
68
+
69
+MultiModalDataDict: TypeAlias = Mapping[str, MultiModalData[Any]]
70
+"""
71
+A dictionary containing an entry for each modality type to input.
72
+
73
+Note:
74
+ This dictionary also accepts modality keys defined outside
75
+ :class:`MultiModalDataBuiltins` as long as a customized plugin
76
+ is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
77
+ Read more on that :ref:`here <adding_multimodal_plugin>`.
78
+"""
79
+
80
+
81
+class PlaceholderRange(TypedDict):
82
+ """
83
+ Placeholder location information for multi-modal data.
84
+
85
+ For example:
86
+ Prompt: AAAA BBBB What is in these images?
87
+ Images A and B will have:
88
+ A: { "offset": 0, "length": 4 }
89
+ B: { "offset": 5, "length": 4 }
90
+ """
91
+
92
+ offset: int
93
+ """The start index of the placeholder in the prompt."""
94
+
95
+ length: int
96
+ """The length of the placeholder."""
97
+
98
+
99
+NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
100
+"""
101
+Uses a list instead of a tensor if the dimensions of each element do not match.
102
+"""
103
+
104
+BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
105
+"""
106
+A dictionary containing nested tensors which have been batched via
107
+:meth:`MultiModalKwargs.batch`.
108
+"""
109
+
110
+
111
+class MultiModalKwargs(UserDict[str, NestedTensors]):
112
+ """
113
+ A dictionary that represents the keyword arguments to
114
+ :meth:`~torch.nn.Module.forward`.
115
+ """
116
+
117
+ @staticmethod
118
+ def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
119
+ """
120
+ Stack the inner dimensions that have the same shape in
121
+ a nested list of tensors.
122
+
123
+ Thus, a dimension represented by a list means that the inner
124
+ dimensions are different for each element along that dimension.
125
+ """
126
+ if isinstance(nested_tensors, torch.Tensor):
127
+ return nested_tensors
128
+
129
+ # TODO: Remove these once all models have been migrated
130
+ if isinstance(nested_tensors, np.ndarray):
131
+ return torch.from_numpy(nested_tensors)
132
+ if isinstance(nested_tensors, (int, float)):
133
+ return torch.tensor(nested_tensors)
134
+
135
+ stacked = [MultiModalKwargs._try_stack(t) for t in nested_tensors]
136
+ if not is_list_of(stacked, torch.Tensor, check="all"):
137
+ # Only tensors (not lists) can be stacked.
138
+ return stacked
139
+
140
+ tensors_ = cast(List[torch.Tensor], stacked)
141
+ if any(t.shape != tensors_[0].shape for t in tensors_):
142
+ # The tensors have incompatible shapes and can't be stacked.
143
+ return tensors_
144
+
145
+ return torch.stack(tensors_)
146
+
147
+ @staticmethod
148
+ def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
149
+ """
150
+ Batch multiple inputs together into a dictionary.
151
+
152
+ The resulting dictionary has the same keys as the inputs.
153
+ If the corresponding value from each input is a tensor and they all
154
+ share the same shape, the output value is a single batched tensor;
155
+ otherwise, the output value is a list containing the original value
156
+ from each input.
157
+ """
158
+ if len(inputs_list) == 0:
159
+ return {}
160
+
161
+ # We need to consider the case where each item in the batch
162
+ # contains different modalities (i.e. different keys).
163
+ item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
164
+
165
+ for inputs in inputs_list:
166
+ for k, v in inputs.items():
167
+ item_lists[k].append(v)
168
+
169
+ return {
170
+ k: MultiModalKwargs._try_stack(item_list)
171
+ for k, item_list in item_lists.items()
172
+ }
173
+
174
+ @staticmethod
175
+ def as_kwargs(
176
+ batched_inputs: BatchedTensorInputs,
177
+ *,
178
+ device: torch.types.Device,
179
+ ) -> BatchedTensorInputs:
180
+ json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
181
+
182
+ json_mapped = json_map_leaves(
183
+ lambda x: x.to(device, non_blocking=True),
184
+ json_inputs,
185
+ )
186
+
187
+ return cast(BatchedTensorInputs, json_mapped)
188
+
189
+
190
+MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]
191
+"""
192
+A dictionary containing placeholder ranges.
193
+"""
194
+
195
+
196
+class MultiModalInputsV2(TypedDict):
197
+ """
198
+ Represents the outputs of :class:`vllm.multimodal.MultiModalProcessor`,
199
+ ready to be passed to vLLM internals.
200
+ """
201
+
202
+ type: Literal["multimodal"]
203
+ """The type of inputs."""
204
+
205
+ prompt: str
206
+ """
207
+ The original, unprocessed prompt text.
208
+
209
+ Note:
210
+ Since prompt text is not required by vLLM internals, we leave this
211
+ unprocessed to save CPU computation. You can still call
212
+ :code:`tokenizer.decode(prompt_token_ids)` to get the processed text.
213
+ """
214
+
215
+ prompt_token_ids: List[int]
216
+ """The processed token IDs which includes placeholder tokens."""
217
+
218
+ mm_kwargs: MultiModalKwargs
219
+ """Keyword arguments to be directly passed to the model after batching."""
220
+
221
+ mm_placeholders: MultiModalPlaceholderDict
222
+ """
223
+ For each modality, information about the placeholder tokens in
224
+ :code:`prompt_token_ids`.
225
+ """
vllm/multimodal/processing.py
new
+273
@@ -0,0 +1,273 @@
1
+from dataclasses import dataclass
2
+from functools import lru_cache, partial
3
+from typing import (Any, Callable, Collection, Generic, List, Mapping,
4
+ Optional, TypedDict, TypeVar, final)
5
+
6
+from transformers import BatchFeature
7
+from typing_extensions import TypeAlias
8
+
9
+from vllm.inputs import InputProcessingContext
10
+from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
11
+from vllm.utils import is_list_of
12
+
13
+from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
14
+ MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
15
+ VideoItem)
16
+
17
+_T = TypeVar("_T")
18
+
19
+ReplacementFunc: TypeAlias = Callable[[_T, BatchFeature, int], List[int]]
20
+"""
21
+Given the original data item, HF-processed data, and index of the processed
22
+item, output the replacement token IDs to be allocated in vLLM.
23
+"""
24
+
25
+
26
+@dataclass
27
+class ModalityProcessingMetadata(Generic[_T]):
28
+ placeholder_replacements: Mapping[str, ReplacementFunc]
29
+ """
30
+ A dictionary where each item represents the original placeholder in the
31
+ prompt text and the corresponding replacement.
32
+ """
33
+
34
+
35
+class MultiModalProcessingMetadataBuiltins(TypedDict, total=False):
36
+ """Type annotations for modality types predefined by vLLM."""
37
+
38
+ image: ModalityProcessingMetadata[ImageItem]
39
+ video: ModalityProcessingMetadata[VideoItem]
40
+ audio: ModalityProcessingMetadata[AudioItem]
41
+
42
+
43
+MultiModalProcessingMetadata: TypeAlias = \
44
+ Mapping[str, ModalityProcessingMetadata[Any]]
45
+"""
46
+A dictionary containing an entry for each modality type to process.
47
+
48
+Note:
49
+ This dictionary also accepts modality keys defined outside
50
+ :class:`MultiModalProcessingMetadataBuiltins` as long as a customized plugin
51
+ is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
52
+ Read more on that :ref:`here <adding_multimodal_plugin>`.
53
+"""
54
+
55
+MultiModalMultiData: TypeAlias = List[_T]
56
+"""
57
+A list of data items, where the number of data items allowed
58
+per modality is restricted by :code:`--limit-mm-per-prompt`.
59
+"""
60
+
61
+
62
+@final
63
+class MultiModalMultiDataBuiltins(TypedDict, total=False):
64
+ """Type annotations for modality types predefined by vLLM."""
65
+
66
+ image: MultiModalMultiData[ImageItem]
67
+ """The input images."""
68
+
69
+ video: MultiModalMultiData[VideoItem]
70
+ """The input videos."""
71
+
72
+ audio: MultiModalMultiData[AudioItem]
73
+ """The input audios."""
74
+
75
+
76
+MultiModalMultiDataDict: TypeAlias = Mapping[str, MultiModalMultiData[Any]]
77
+"""
78
+A dictionary containing an entry for each modality type to input.
79
+
80
+Note:
81
+ This dictionary also accepts modality keys defined outside
82
+ :class:`MultiModalMultiDataBuiltins` as long as a customized plugin
83
+ is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
84
+ Read more on that :ref:`here <adding_multimodal_plugin>`.
85
+"""
86
+
87
+
88
+def to_multi_format(data: MultiModalDataDict) -> MultiModalMultiDataDict:
89
+ """
90
+ Convert a :class:`MultiModalDataDict` containing single data items
91
+ to a :class:`MultiModalMultiDataDict` containing multiple data items
92
+ per entry.
93
+ """
94
+ multi_data: Mapping[str, MultiModalMultiData[Any]] = {}
95
+
96
+ for k, v in data.items():
97
+ # yapf: disable
98
+ if k == "video":
99
+ # Special case since even a single item can be a list
100
+ multi_data[k] = v if is_list_of(v, list) else [v] # type: ignore[index]
101
+ elif k in ("image", "audio"):
102
+ multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
103
+ else:
104
+ multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
105
+ # yapf: enable
106
+
107
+ return multi_data
108
+
109
+
110
+def encode_no_special_tokens(
111
+ tokenizer: AnyTokenizer,
112
+ text: str,
113
+) -> List[int]:
114
+ """
115
+ Backend-agnostic equivalent of HF's
116
+ :code:`tokenizer.encode(text, add_special_tokens=False)`.
117
+ """
118
+ if isinstance(tokenizer, MistralTokenizer):
119
+ return tokenizer.tokenizer.encode(text, bos=False, eos=False)
120
+
121
+ return tokenizer.encode(text, add_special_tokens=False)
122
+
123
+
124
+@lru_cache
125
+def candidate_placeholders(
126
+ tokenizer: AnyTokenizer,
127
+ placeholder_text: str,
128
+) -> Collection[List[int]]:
129
+ """Generate token ID sequences that may represent a placeholder text."""
130
+ # When the placeholder text is not mapped to a special token ID,
131
+ # it may be tokenized differently based on whether it is at the start/end
132
+ # of the string. So, we go through each combination of whether the text
133
+ # is at the start and end boundaries of the string
134
+
135
+ # Matches the placeholder when it is in the middle of the string
136
+ start_id, = encode_no_special_tokens(tokenizer, "a")
137
+ end_id, = encode_no_special_tokens(tokenizer, "b")
138
+
139
+ candidate_basic = encode_no_special_tokens(tokenizer, placeholder_text)
140
+
141
+ start_id_, *candidate_a = encode_no_special_tokens(
142
+ tokenizer,
143
+ f"a{placeholder_text}",
144
+ )
145
+ assert start_id == start_id_
146
+
147
+ start_id_, *candidate_ab, end_id_ = encode_no_special_tokens(
148
+ tokenizer,
149
+ f"a{placeholder_text}b",
150
+ )
151
+ assert start_id == start_id_ and end_id == end_id_
152
+
153
+ *candidate_b, end_id_ = encode_no_special_tokens(
154
+ tokenizer,
155
+ f"{placeholder_text}b",
156
+ )
157
+ assert end_id == end_id_
158
+
159
+ # Remove duplicates (need to convert to tuple to be hashable)
160
+ unique_candidates = {
161
+ tuple(c)
162
+ for c in [candidate_basic, candidate_a, candidate_ab, candidate_b]
163
+ }
164
+
165
+ # Convert back to list
166
+ return [list(c) for c in unique_candidates]
167
+
168
+
169
+def apply_placeholders(
170
+ token_ids: List[int],
171
+ placeholder_ids: List[int],
172
+ get_replacement_ids: Callable[[], List[int]],
173
+) -> Optional[PlaceholderRange]:
174
+ """
175
+ Find the first occurrence of :code:`placeholder_ids`,
176
+ and replace it with the output of :code:`get_replacement_ids`.
177
+
178
+ This function updates :code:`token_ids` in place.
179
+ """
180
+ placeholder_length = len(placeholder_ids)
181
+
182
+ for start_idx in range(len(token_ids) - placeholder_length + 1):
183
+ if token_ids[start_idx:placeholder_length] == placeholder_ids:
184
+ token_ids[start_idx:placeholder_length] = get_replacement_ids()
185
+
186
+ return PlaceholderRange(offset=start_idx,
187
+ length=placeholder_length)
188
+
189
+ return None
190
+
191
+
192
+class MultiModalProcessor:
193
+ """
194
+ Helper class to process multi-modal inputs to be used in vLLM.
195
+ """
196
+
197
+ def __init__(
198
+ self,
199
+ ctx: InputProcessingContext,
200
+ metadata: MultiModalProcessingMetadata,
201
+ ) -> None:
202
+ super().__init__()
203
+
204
+ self.ctx = ctx
205
+ self.metadata = metadata
206
+
207
+ def __call__(
208
+ self,
209
+ prompt: str,
210
+ mm_data: MultiModalDataDict,
211
+ mm_processor_kwargs: Mapping[str, object],
212
+ ) -> MultiModalInputsV2:
213
+ return self.apply(prompt, mm_data, mm_processor_kwargs)
214
+
215
+ def apply(
216
+ self,
217
+ prompt: str,
218
+ mm_data: MultiModalDataDict,
219
+ mm_processor_kwargs: Mapping[str, object],
220
+ ) -> MultiModalInputsV2:
221
+ tokenizer = self.ctx.tokenizer
222
+ hf_processor = self.ctx.get_hf_processor()
223
+
224
+ processed_inputs = hf_processor(
225
+ text=prompt, # type: ignore
226
+ **mm_data,
227
+ **mm_processor_kwargs,
228
+ )
229
+ new_token_ids, = processed_inputs.pop("input_ids").tolist()
230
+ mm_kwargs = MultiModalKwargs(processed_inputs)
231
+
232
+ mm_placeholders: Mapping[str, List[PlaceholderRange]] = {}
233
+
234
+ for modality, orig_inputs in to_multi_format(mm_data).items():
235
+ assert isinstance(orig_inputs, list)
236
+
237
+ metadata = self.metadata[modality]
238
+ placeholder_replacements = metadata.placeholder_replacements
239
+
240
+ modality_placeholders: List[PlaceholderRange] = []
241
+
242
+ for item_idx, orig_item in enumerate(orig_inputs):
243
+ for match_text, replace_fn in placeholder_replacements.items():
244
+ candidates = candidate_placeholders(tokenizer, match_text)
245
+ get_replacement_ids = partial(
246
+ replace_fn,
247
+ orig_item,
248
+ processed_inputs,
249
+ item_idx,
250
+ )
251
+
252
+ for match_ids in candidates:
253
+ # TODO(youkaichao): Don't update new_token_ids
254
+ placeholders = apply_placeholders(
255
+ new_token_ids,
256
+ match_ids,
257
+ get_replacement_ids,
258
+ )
259
+
260
+ if placeholders is not None:
261
+ modality_placeholders.append(placeholders)
262
+
263
+ # yapf: disable
264
+ mm_placeholders[modality] = modality_placeholders # type: ignore[index]
265
+ # yapf: enable
266
+
267
+ return MultiModalInputsV2(
268
+ type="multimodal",
269
+ prompt=prompt,
270
+ prompt_token_ids=new_token_ids,
271
+ mm_kwargs=mm_kwargs,
272
+ mm_placeholders=mm_placeholders,
273
+ )
vllm/multimodal/registry.py
+80
-4
@@ -1,13 +1,20 @@
1
import functools
2
from collections import UserDict
3
-from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Sequence
3
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
4
+ Sequence, Type, TypeVar)
5
6
+import torch.nn as nn
7
+from typing_extensions import TypeAlias
8
+
9
+from vllm.inputs import InputProcessingContext
10
from vllm.logger import init_logger
11
+from vllm.transformers_utils.tokenizer import AnyTokenizer
12
13
from .audio import AudioPlugin
8
-from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalKwargs,
9
- MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
14
+from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
15
from .image import ImagePlugin
16
+from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
17
+from .processing import MultiModalProcessor
18
from .video import VideoPlugin
19
20
if TYPE_CHECKING:
@@ -15,8 +22,18 @@ if TYPE_CHECKING:
22
23
logger = init_logger(__name__)
24
25
+N = TypeVar("N", bound=Type[nn.Module])
26
+
27
+MultiModalProcessorFactory: TypeAlias = Callable[[InputProcessingContext],
28
+ MultiModalProcessor]
29
+"""
30
+Constructs a :class:`MultiModalProcessor` instance from the context.
31
+
32
+The processing metadata should be derived from the context.
33
+"""
34
+
35
19
-class _MultiModalLimits(UserDict):
36
+class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
37
"""
38
Wraps `_limits_by_model` for a more informative error message
39
when attempting to access a model that does not exist.
@@ -45,6 +62,9 @@ class MultiModalRegistry:
62
plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
63
self._plugins = {p.get_data_key(): p for p in plugins}
64
65
+ self._processor_factories: Dict[Type[nn.Module],
66
+ MultiModalProcessorFactory] = {}
67
+
68
# This is used for non-multimodal models
69
self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
70
@@ -243,3 +263,59 @@ class MultiModalRegistry:
263
This should be called after :meth:`init_mm_limits_per_prompt`.
264
"""
265
return self._limits_by_model[model_config]
266
+
267
+ def register_processor(
268
+ self,
269
+ factory: MultiModalProcessorFactory,
270
+ ):
271
+ """
272
+ Register a multi-modal processor to a model class.
273
+
274
+ When the model receives multi-modal data, the provided function is
275
+ invoked to transform the data into a dictionary of model inputs.
276
+
277
+ See also:
278
+ - :ref:`input_processing_pipeline`
279
+ - :ref:`enabling_multimodal_inputs`
280
+ """
281
+
282
+ def wrapper(model_cls: N) -> N:
283
+ if model_cls in self._processor_factories:
284
+ logger.warning(
285
+ "Model class %s already has an input mapper "
286
+ "registered to %s. It is overwritten by the new one.",
287
+ model_cls, self)
288
+
289
+ self._processor_factories[model_cls] = factory
290
+
291
+ return model_cls
292
+
293
+ return wrapper
294
+
295
+ def has_processor(self, model_config: "ModelConfig") -> bool:
296
+ """
297
+ Test whether a multi-modal processor is defined for a specific model.
298
+ """
299
+ # Avoid circular import
300
+ from vllm.model_executor.model_loader import get_model_architecture
301
+
302
+ model_cls, _ = get_model_architecture(model_config)
303
+ return model_cls in self._processor_factories
304
+
305
+ def create_processor(
306
+ self,
307
+ model_config: "ModelConfig",
308
+ tokenizer: AnyTokenizer,
309
+ ) -> MultiModalProcessor:
310
+ """
311
+ Create a multi-modal processor for a specific model and tokenizer.
312
+ """
313
+
314
+ # Avoid circular import
315
+ from vllm.model_executor.model_loader import get_model_architecture
316
+
317
+ model_cls, _ = get_model_architecture(model_config)
318
+ processor_factory = self._processor_factories[model_cls]
319
+
320
+ ctx = InputProcessingContext(model_config, tokenizer)
321
+ return processor_factory(ctx)
vllm/multimodal/utils.py
+2
-1
@@ -11,9 +11,10 @@ from PIL import Image
11
import vllm.envs as envs
12
from vllm.connections import global_http_connection
13
from vllm.logger import init_logger
14
-from vllm.multimodal.base import MultiModalDataDict, PlaceholderRange
14
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
15
16
+from .inputs import MultiModalDataDict, PlaceholderRange
17
+
18
logger = init_logger(__name__)
19
20
cached_get_tokenizer = lru_cache(get_tokenizer)
vllm/multimodal/video.py
+5
-15
@@ -1,5 +1,5 @@
1
from functools import lru_cache
2
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
2
+from typing import TYPE_CHECKING, Any, Dict, Optional
3
4
import numpy as np
5
@@ -9,8 +9,9 @@ from vllm.transformers_utils.processor import get_video_processor
9
from vllm.transformers_utils.tokenizer import get_tokenizer
10
from vllm.utils import is_list_of
11
12
-from .base import MultiModalData, MultiModalKwargs
12
+from .base import MultiModalData
13
from .image import ImagePlugin
14
+from .inputs import MultiModalKwargs, VideoItem
15
16
if TYPE_CHECKING:
17
from vllm.config import ModelConfig
@@ -20,17 +21,6 @@ logger = init_logger(__name__)
21
cached_get_video_processor = lru_cache(get_video_processor)
22
cached_get_tokenizer = lru_cache(get_tokenizer)
23
23
-VideoInput = Union[
24
- "np.ndarray", # single video input
25
- List["np.ndarray"],
26
- # TODO: support more types
27
- # List[Image.Image], List[List[Image.Image]],
28
- # "torch.Tensor",
29
- # List["torch.Tensor"],
30
- # List[List["np.ndarrray"]],
31
- # List[List["torch.Tensor"]],
32
-]
33
-
24
25
class VideoPlugin(ImagePlugin):
26
"""Plugin for video data."""
@@ -53,13 +43,13 @@ class VideoPlugin(ImagePlugin):
43
def _default_input_mapper(
44
self,
45
ctx: InputContext,
56
- data: MultiModalData[object],
46
+ data: MultiModalData[VideoItem],
47
**mm_processor_kwargs,
48
) -> MultiModalKwargs:
49
model_config = ctx.model_config
50
51
if isinstance(data, list) and len(data) == 1:
62
- data = data[0]
52
+ data = data[0] # type: ignore
53
54
if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
55
video_processor = self._get_hf_video_processor(
vllm/sequence.py
+17
-51
@@ -5,25 +5,21 @@ from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
7
from dataclasses import dataclass, field
8
-from functools import cached_property, reduce
9
-from typing import (TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List,
10
- Mapping, Optional)
8
+from functools import reduce
9
+from typing import Any, Callable, DefaultDict, Dict, List, Mapping, Optional
10
from typing import Sequence as GenericSequence
11
from typing import Set, Tuple, Union
12
13
import msgspec
14
import torch
16
-from typing_extensions import assert_never
15
16
+from vllm.inputs import SingletonInputs, SingletonInputsAdapter
17
from vllm.lora.request import LoRARequest
18
from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
19
from vllm.pooling_params import PoolingParams
20
from vllm.prompt_adapter.request import PromptAdapterRequest
21
from vllm.sampling_params import RequestOutputKind, SamplingParams
22
24
-if TYPE_CHECKING:
25
- from vllm.inputs import SingletonInputs
26
-
23
VLLM_TOKEN_ID_ARRAY_TYPE = "l"
24
25
VLLM_INVALID_TOKEN_ID = -1
@@ -407,14 +403,14 @@ class Sequence:
403
def __init__(
404
self,
405
seq_id: int,
410
- inputs: "SingletonInputs",
406
+ inputs: SingletonInputs,
407
block_size: int,
408
eos_token_id: Optional[int] = None,
409
lora_request: Optional[LoRARequest] = None,
410
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
411
) -> None:
412
self.seq_id = seq_id
417
- self.inputs = inputs
413
+ self.inputs = SingletonInputsAdapter(inputs)
414
self.block_size = block_size
415
self.eos_token_id = eos_token_id
416
self.lora_request = lora_request
@@ -441,59 +437,29 @@ class Sequence:
437
def n_blocks(self) -> int:
438
return (self.get_len() + self.block_size - 1) // self.block_size
439
444
- @cached_property
440
+ @property
441
def prompt(self) -> Optional[str]:
446
- inputs = self.inputs
447
-
448
- if inputs["type"] == "token":
449
- return inputs.get("prompt")
442
+ return self.inputs.prompt
443
451
- assert_never(inputs)
452
-
453
- @cached_property
444
+ @property
445
def prompt_token_ids(self) -> List[int]:
455
- inputs = self.inputs
456
-
457
- if inputs["type"] == "token":
458
- return inputs.get("prompt_token_ids", [])
446
+ return self.inputs.prompt_token_ids
447
460
- assert_never(inputs)
461
-
462
- @cached_property
448
+ @property
449
def prompt_embeds(self) -> Optional[torch.Tensor]:
464
- inputs = self.inputs
465
-
466
- if inputs["type"] == "token":
467
- return None
468
-
469
- assert_never(inputs)
450
+ return self.inputs.prompt_embeds
451
471
- @cached_property
452
+ @property
453
def multi_modal_data(self) -> "MultiModalDataDict":
473
- inputs = self.inputs
474
-
475
- if inputs["type"] == "token":
476
- return inputs.get("multi_modal_data", {})
477
-
478
- assert_never(inputs)
479
-
480
- @cached_property
481
- def mm_processor_kwargs(self) -> Dict[str, Any]:
482
- inputs = self.inputs
483
-
484
- if inputs["type"] == "token":
485
- return inputs.get("mm_processor_kwargs", {})
486
-
487
- assert_never(inputs)
454
+ return self.inputs.multi_modal_data
455
456
@property
457
def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
491
- inputs = self.inputs
492
-
493
- if inputs["type"] == "token":
494
- return inputs.get("multi_modal_placeholders", {})
458
+ return self.inputs.multi_modal_placeholders
459
496
- assert_never(inputs)
460
+ @property
461
+ def mm_processor_kwargs(self) -> Dict[str, Any]:
462
+ return self.inputs.mm_processor_kwargs
463
464
@property
465
def lora_int_id(self) -> int:
vllm/v1/engine/async_llm.py
+4
@@ -6,6 +6,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
6
from vllm.engine.metrics_types import StatLoggerBase
7
from vllm.engine.protocol import EngineClient
8
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9
+from vllm.inputs.preprocess import InputPreprocessor
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -321,6 +322,9 @@ class AsyncLLM(EngineClient):
322
async def get_decoding_config(self):
323
raise ValueError("Not Supported on V1 yet.")
324
325
+ async def get_input_preprocessor(self) -> InputPreprocessor:
326
+ return self.processor.input_preprocessor
327
+
328
async def get_tokenizer(
329
self,
330
lora_request: Optional[LoRARequest] = None,
vllm/v1/engine/llm_engine.py
+3
-1
@@ -7,6 +7,7 @@ from vllm.envs import VLLM_ENABLE_V1_MULTIPROCESSING
7
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
10
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
11
from vllm.outputs import RequestOutput
12
from vllm.pooling_params import PoolingParams
13
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -32,6 +33,7 @@ class LLMEngine:
33
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
34
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
35
input_registry: InputRegistry = INPUT_REGISTRY,
36
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
37
use_cached_outputs: bool = False,
38
multiprocess_mode: bool = False,
39
) -> None:
@@ -50,7 +52,7 @@ class LLMEngine:
52
# Processor (convert Inputs --> EngineCoreRequests)
53
self.processor = Processor(vllm_config.model_config,
54
vllm_config.lora_config, self.tokenizer,
53
- input_registry)
55
+ input_registry, mm_registry)
56
57
# Detokenizer (converts EngineCoreOutputs --> RequestOutput)
58
self.detokenizer = Detokenizer(
vllm/v1/engine/processor.py
+55
-18
@@ -2,15 +2,17 @@ import time
2
from typing import Any, Dict, Mapping, Optional, Tuple, Union
3
4
from vllm.config import LoRAConfig, ModelConfig
5
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
6
- EncoderDecoderLLMInputs, InputRegistry, PromptType)
5
+from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
6
+ PromptType, SingletonInputsAdapter)
7
+from vllm.inputs.parse import is_encoder_decoder_inputs
8
from vllm.inputs.preprocess import InputPreprocessor
9
from vllm.lora.request import LoRARequest
10
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
11
from vllm.pooling_params import PoolingParams
12
from vllm.prompt_adapter.request import PromptAdapterRequest
13
from vllm.sampling_params import SamplingParams
14
from vllm.transformers_utils.config import try_get_generation_config
13
-from vllm.transformers_utils.tokenizer_group import AnyTokenizer
15
+from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16
from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
17
18
@@ -20,8 +22,9 @@ class Processor:
22
self,
23
model_config: ModelConfig,
24
lora_config: Optional[LoRAConfig],
23
- tokenizer: AnyTokenizer,
25
+ tokenizer: BaseTokenizerGroup,
26
input_registry: InputRegistry = INPUT_REGISTRY,
27
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
28
):
29
30
self.model_config = model_config
@@ -31,7 +34,8 @@ class Processor:
34
self.generation_config_fields = _load_generation_config_dict(
35
model_config)
36
self.input_preprocessor = InputPreprocessor(model_config,
34
- self.tokenizer)
37
+ self.tokenizer,
38
+ mm_registry)
39
self.input_processor = input_registry.create_input_processor(
40
model_config)
41
@@ -73,6 +77,19 @@ class Processor:
77
self._validate_model_inputs(processed_inputs)
78
eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
79
80
+ if is_encoder_decoder_inputs(processed_inputs):
81
+ decoder_inputs = SingletonInputsAdapter(
82
+ processed_inputs["decoder"])
83
+ encoder_inputs = SingletonInputsAdapter(
84
+ processed_inputs["encoder"])
85
+ else:
86
+ decoder_inputs = SingletonInputsAdapter(processed_inputs)
87
+ encoder_inputs = None
88
+
89
+ # TODO: Impl encoder-decoder
90
+ if encoder_inputs is not None:
91
+ raise NotImplementedError
92
+
93
assert isinstance(params, SamplingParams)
94
# TODO: can we avoid cloning here in multiproc case
95
sampling_params = params.clone()
@@ -81,27 +98,43 @@ class Processor:
98
99
# Make Request for Detokenizer.
100
detokenizer_request = DetokenizerRequest(
84
- request_id, processed_inputs.get("prompt"),
85
- processed_inputs.get("prompt_token_ids"),
101
+ request_id,
102
+ decoder_inputs.prompt,
103
+ decoder_inputs.prompt_token_ids,
104
sampling_params.skip_special_tokens,
105
sampling_params.spaces_between_special_tokens,
88
- sampling_params.output_kind, sampling_params.stop,
89
- sampling_params.include_stop_str_in_output)
106
+ sampling_params.output_kind,
107
+ sampling_params.stop,
108
+ sampling_params.include_stop_str_in_output,
109
+ )
110
111
# Make Request for EngineCore.
112
engine_core_request = EngineCoreRequest(
93
- request_id, processed_inputs.get("prompt"),
94
- processed_inputs.get("prompt_token_ids"),
95
- processed_inputs.get("multi_modal_data"),
96
- processed_inputs.get("multi_modal_placeholders"),
97
- processed_inputs.get("mm_processor_kwargs"), sampling_params,
98
- eos_token_id, arrival_time, lora_request)
113
+ request_id,
114
+ decoder_inputs.prompt,
115
+ decoder_inputs.prompt_token_ids,
116
+ decoder_inputs.multi_modal_data,
117
+ decoder_inputs.multi_modal_placeholders,
118
+ decoder_inputs.mm_processor_kwargs,
119
+ sampling_params,
120
+ eos_token_id,
121
+ arrival_time,
122
+ lora_request,
123
+ )
124
125
return detokenizer_request, engine_core_request
126
102
- def _validate_model_inputs(self, inputs: Union[DecoderOnlyInputs,
103
- EncoderDecoderLLMInputs]):
104
- prompt_ids = inputs.get("prompt_token_ids")
127
+ def _validate_model_inputs(self, inputs: ProcessorInputs):
128
+ if is_encoder_decoder_inputs(inputs):
129
+ # For encoder-decoder multimodal models, the max_prompt_len
130
+ # restricts the decoder prompt length
131
+ prompt_inputs = inputs["decoder" if self.model_config.
132
+ is_multimodal_model else "encoder"]
133
+ else:
134
+ prompt_inputs = inputs
135
+
136
+ prompt_ids = SingletonInputsAdapter(prompt_inputs).prompt_token_ids
137
+
138
if prompt_ids is None or len(prompt_ids) == 0:
139
raise ValueError("Prompt cannot be empty")
140
@@ -117,6 +150,10 @@ class Processor:
150
"inputs, the number of image tokens depends on the number "
151
"of images, and possibly their aspect ratios as well.")
152
153
+ # TODO: Find out how many placeholder tokens are there so we can
154
+ # check that chunked prefill does not truncate them
155
+ # max_batch_len = self.scheduler_config.max_num_batched_tokens
156
+
157
158
def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
159
config = try_get_generation_config(
vllm/v1/request.py
+11
-15
@@ -1,7 +1,7 @@
1
import enum
2
-from typing import TYPE_CHECKING, List, Optional, Union
2
+from typing import List, Optional, Union
3
4
-from vllm.inputs.data import DecoderOnlyInputs
4
+from vllm.inputs import DecoderOnlyInputs, SingletonInputsAdapter, token_inputs
5
from vllm.lora.request import LoRARequest
6
from vllm.multimodal import MultiModalKwargs
7
from vllm.sampling_params import SamplingParams
@@ -9,23 +9,20 @@ from vllm.sequence import RequestMetrics
9
from vllm.v1.engine import EngineCoreRequest
10
from vllm.v1.utils import ConstantList
11
12
-if TYPE_CHECKING:
13
- from vllm.inputs import DecoderOnlyInputs
14
-
12
13
class Request:
14
15
def __init__(
16
self,
17
request_id: str,
21
- inputs: "DecoderOnlyInputs",
18
+ inputs: DecoderOnlyInputs,
19
sampling_params: SamplingParams,
20
eos_token_id: Optional[int],
21
arrival_time: float,
22
lora_request: Optional[LoRARequest] = None,
23
) -> None:
24
self.request_id = request_id
28
- self.inputs = inputs
25
+ self.inputs = SingletonInputsAdapter(inputs)
26
self.sampling_params = sampling_params
27
# Because of LoRA, the eos token id can be different for each request.
28
self.eos_token_id = eos_token_id
@@ -41,17 +38,17 @@ class Request:
38
assert sampling_params.max_tokens is not None
39
self.max_tokens = sampling_params.max_tokens
40
44
- self.prompt = inputs.get("prompt")
45
- self.prompt_token_ids = inputs["prompt_token_ids"]
41
+ self.prompt = self.inputs.prompt
42
+ self.prompt_token_ids = self.inputs.prompt_token_ids
43
self.num_prompt_tokens = len(self.prompt_token_ids)
44
self._output_token_ids: List[int] = []
45
self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46
self.num_computed_tokens = 0
47
48
# Raw multimodal data before the mm input mapper (e.g., PIL images).
52
- self.mm_data = inputs.get("multi_modal_data")
53
- self.mm_processor_kwargs = inputs.get("mm_processor_kwargs")
54
- mm_positions = inputs.get("multi_modal_placeholders")
49
+ self.mm_data = self.inputs.multi_modal_data
50
+ self.mm_processor_kwargs = self.inputs.mm_processor_kwargs
51
+ mm_positions = self.inputs.multi_modal_placeholders
52
if mm_positions:
53
# FIXME(woosuk): Support other modalities.
54
self.mm_positions = mm_positions.get("image", [])
@@ -64,8 +61,7 @@ class Request:
61
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
62
return cls(
63
request_id=request.request_id,
67
- inputs=DecoderOnlyInputs(
68
- type="token",
64
+ inputs=token_inputs(
65
prompt_token_ids=request.prompt_token_ids,
66
prompt=request.prompt,
67
multi_modal_data=request.mm_data,
@@ -114,7 +110,7 @@ class Request:
110
return RequestStatus.get_finished_reason(self.status)
111
112
def has_encoder_inputs(self) -> bool:
117
- return self.mm_data is not None
113
+ return len(self.mm_data) > 0
114
115
@property
116
def num_encoder_inputs(self) -> int:
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -28,7 +28,7 @@ from vllm.v1.outputs import ModelRunnerOutput
28
from vllm.v1.sample.metadata import SamplingMetadata
29
30
if TYPE_CHECKING:
31
- from vllm.multimodal.base import PlaceholderRange
31
+ from vllm.multimodal.inputs import PlaceholderRange
32
from vllm.v1.core.scheduler import SchedulerOutput
33
34
logger = init_logger(__name__)
vllm/worker/cpu_model_runner.py
+27
-14
@@ -148,19 +148,29 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
148
query_lens=seq_lens,
149
)
150
151
- def _compute_multi_modal_input(self, seq_group: SequenceGroupMetadata,
152
- seq_data: SequenceData, computed_len: int,
153
- mm_processor_kwargs: Dict[str, Any]):
154
-
151
+ def _compute_multi_modal_input(
152
+ self,
153
+ seq_data: SequenceData,
154
+ computed_len: int,
155
+ seq_group_metadata: SequenceGroupMetadata,
156
+ ):
157
# NOTE: mm_data only includes the subset of multi-modal items that
158
# intersect with the current prefill positions.
159
mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
158
- seq_group, range(computed_len, len(seq_data.get_token_ids())))
160
+ seq_group_metadata,
161
+ range(computed_len, len(seq_data.get_token_ids())),
162
+ )
163
164
if not mm_data:
161
- return
165
+ return None, None, None
166
163
- mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs)
167
+ if self.runner.mm_registry.has_processor(self.runner.model_config):
168
+ mm_kwargs = mm_data
169
+ else:
170
+ mm_kwargs = self.multi_modal_input_mapper(
171
+ mm_data,
172
+ seq_group_metadata.mm_processor_kwargs,
173
+ )
174
175
# special processing for mrope position deltas.
176
mrope_positions = None
@@ -202,7 +212,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
212
213
slot_mapping: List[int] = []
214
seq_lens: List[int] = []
205
- multi_model_kwargs_list: List[MultiModalKwargs] = []
215
+ multi_modal_kwargs_list: List[MultiModalKwargs] = []
216
multi_modal_placeholder_maps: Dict[
217
str,
218
MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -223,11 +233,14 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
233
234
mrope_positions = None
235
if seq_group_metadata.multi_modal_data:
226
- mm_kwargs, placeholder_maps, mrope_positions = self \
227
- ._compute_multi_modal_input(
228
- seq_group_metadata, seq_data, computed_len,
229
- seq_group_metadata.mm_processor_kwargs)
230
- multi_model_kwargs_list.append(mm_kwargs)
236
+ (
237
+ mm_kwargs,
238
+ placeholder_maps,
239
+ mrope_positions,
240
+ ) = self._compute_multi_modal_input(seq_data, computed_len,
241
+ seq_group_metadata)
242
+
243
+ multi_modal_kwargs_list.append(mm_kwargs)
244
for modality, placeholder_map in placeholder_maps.items():
245
multi_modal_placeholder_maps[modality].extend(
246
placeholder_map)
@@ -302,7 +315,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
315
multi_modal_placeholder_index_maps=placeholder_index_maps,
316
)
317
305
- multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
318
+ multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
319
320
return (input_tokens, input_positions, attn_metadata, seq_lens,
321
multi_modal_kwargs)
vllm/worker/hpu_model_runner.py
+3
-3
@@ -716,7 +716,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
716
context_lens: List[int] = []
717
query_lens: List[int] = []
718
prefix_block_tables: List[List[int]] = []
719
- multi_model_kwargs_list: List[MultiModalKwargs] = []
719
+ multi_modal_kwargs_list: List[MultiModalKwargs] = []
720
721
if len(seq_group_metadata_list) == 0:
722
return PreparePromptMetadata.empty()
@@ -777,7 +777,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
777
mm_data = seq_group_metadata.multi_modal_data
778
if mm_data:
779
mm_kwargs = self.multi_modal_input_mapper(mm_data)
780
- multi_model_kwargs_list.append(mm_kwargs)
780
+ multi_modal_kwargs_list.append(mm_kwargs)
781
782
if seq_group_metadata.block_tables is None:
783
# During memory profiling, the block tables are not initialized
@@ -876,7 +876,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
876
multi_modal_placeholder_index_maps=
877
None # FIXME(kzawora): mutli-modality will not work here
878
)
879
- multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
879
+ multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
880
881
return PreparePromptMetadata(input_tokens=input_tokens,
882
input_positions=input_positions,
vllm/worker/model_runner.py
+15
-10
@@ -252,7 +252,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
252
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
253
254
# Multi-modal inputs.
255
- multi_model_kwargs: Optional[MultiModalKwargs] = None,
255
+ multi_modal_kwargs: Optional[MultiModalKwargs] = None,
256
multi_modal_placeholder_maps: Optional[Dict[
257
str, MultiModalPlaceholderMap]] = None,
258
@@ -373,7 +373,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
373
prompt_adapter_prompt_mapping or [])
374
375
self.prompt_adapter_request = prompt_adapter_request
376
- self.multi_model_kwargs = multi_model_kwargs
376
+ self.multi_modal_kwargs = multi_modal_kwargs
377
self.multi_modal_placeholder_maps = multi_modal_placeholder_maps
378
self.prefix_cache_hit = prefix_cache_hit
379
@@ -661,10 +661,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
661
if not mm_data:
662
return
663
664
- mm_kwargs = self.multi_modal_input_mapper(
665
- mm_data,
666
- mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
667
- inter_data.multi_model_kwargs = mm_kwargs
664
+ if self.runner.mm_registry.has_processor(self.runner.model_config):
665
+ mm_kwargs = mm_data
666
+ else:
667
+ mm_kwargs = self.multi_modal_input_mapper(
668
+ mm_data,
669
+ seq_group_metadata.mm_processor_kwargs,
670
+ )
671
+
672
+ inter_data.multi_modal_kwargs = mm_kwargs
673
inter_data.multi_modal_placeholder_maps = placeholder_maps
674
675
# special processing for mrope position deltas.
@@ -938,11 +943,11 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
943
)
944
945
# Multi-modal data.
941
- multi_model_kwargs_list = [
942
- data.multi_model_kwargs for data in self.inter_data_list
943
- if data.multi_model_kwargs is not None
946
+ multi_modal_kwargs_list = [
947
+ data.multi_modal_kwargs for data in self.inter_data_list
948
+ if data.multi_modal_kwargs is not None
949
]
945
- multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
950
+ multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
951
952
return self.model_input_cls(
953
input_tokens=input_tokens_tensor,
vllm/worker/neuron_model_runner.py
+13
-9
@@ -67,7 +67,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
67
self.pin_memory = is_pin_memory_available()
68
69
# Multi-modal data support
70
- self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
70
+ self.mm_registry = MULTIMODAL_REGISTRY
71
+ self.multi_modal_input_mapper = self.mm_registry \
72
.create_input_mapper(self.model_config)
73
74
# Lazy initialization.
@@ -122,7 +123,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
123
input_block_ids: List[int] = []
124
125
seq_lens: List[int] = []
125
- multi_model_kwargs_list: List[MultiModalKwargs] = []
126
+ multi_modal_kwargs_list: List[MultiModalKwargs] = []
127
for seq_group_metadata in seq_group_metadata_list:
128
assert seq_group_metadata.is_prompt
129
seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -144,12 +145,15 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
145
146
mm_data = seq_group_metadata.multi_modal_data
147
if mm_data:
147
- # Process multi-modal data
148
- mm_kwargs = self.multi_modal_input_mapper(
149
- mm_data,
150
- mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs,
151
- )
152
- multi_model_kwargs_list.append(mm_kwargs)
148
+ if self.mm_registry.has_processor(self.model_config):
149
+ mm_kwargs = mm_data
150
+ else:
151
+ mm_kwargs = self.multi_modal_input_mapper(
152
+ mm_data,
153
+ seq_group_metadata.mm_processor_kwargs,
154
+ )
155
+
156
+ multi_modal_kwargs_list.append(mm_kwargs)
157
158
max_seq_len = max(seq_lens)
159
assert max_seq_len > 0
@@ -167,7 +171,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
171
dtype=torch.long,
172
device=self.device)
173
170
- multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
174
+ multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
175
176
return (input_tokens, input_positions, input_block_ids, seq_lens,
177
multi_modal_kwargs)
vllm/worker/openvino_model_runner.py
+13
-8
@@ -70,7 +70,8 @@ class OpenVINOModelRunner(ModelRunnerBase):
70
)
71
72
# Multi-modal data support
73
- self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
73
+ self.mm_registry = MULTIMODAL_REGISTRY
74
+ self.multi_modal_input_mapper = self.mm_registry \
75
.create_input_mapper(self.model_config)
76
77
# Lazy initialization.
@@ -102,7 +103,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
103
seq_lens: List[int] = []
104
past_lens: List[int] = []
105
query_lens: List[int] = []
105
- multi_model_kwargs_list: List[MultiModalKwargs] = []
106
+ multi_modal_kwargs_list: List[MultiModalKwargs] = []
107
multi_modal_placeholder_maps: Dict[
108
str,
109
MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -222,11 +223,15 @@ class OpenVINOModelRunner(ModelRunnerBase):
223
mm_data, placeholder_maps = MultiModalPlaceholderMap \
224
.from_seq_group(seq_group_metadata, positions_range)
225
225
- mm_kwargs = self.multi_modal_input_mapper(
226
- mm_data,
227
- mm_processor_kwargs=seq_group_metadata.
228
- mm_processor_kwargs)
229
- multi_model_kwargs_list.append(mm_kwargs)
226
+ if self.mm_registry.has_processor(self.model_config):
227
+ mm_kwargs = mm_data
228
+ else:
229
+ mm_kwargs = self.multi_modal_input_mapper(
230
+ mm_data,
231
+ seq_group_metadata.mm_processor_kwargs,
232
+ )
233
+
234
+ multi_modal_kwargs_list.append(mm_kwargs)
235
236
for modality, placeholder_map in placeholder_maps.items():
237
multi_modal_placeholder_maps[modality].extend(
@@ -275,7 +280,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
280
multi_modal_placeholder_index_maps=placeholder_index_maps,
281
)
282
278
- multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
283
+ multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
284
285
return ModelInput(
286
input_tokens,
vllm/worker/xpu_model_runner.py
+12
-4
@@ -160,7 +160,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
160
input_positions: List[int] = []
161
slot_mapping: List[int] = []
162
seq_lens: List[int] = []
163
- multi_model_kwargs_list: List[MultiModalKwargs] = []
163
+ multi_modal_kwargs_list: List[MultiModalKwargs] = []
164
multi_modal_placeholder_maps: Dict[
165
str,
166
MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -191,8 +191,16 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
191
mm_data, placeholder_maps = MultiModalPlaceholderMap \
192
.from_seq_group(seq_group_metadata, positions_range)
193
194
- mm_kwargs = self.runner.multi_modal_input_mapper(mm_data)
195
- multi_model_kwargs_list.append(mm_kwargs)
194
+ if self.runner.mm_registry.has_processor(
195
+ self.runner.model_config):
196
+ mm_kwargs = mm_data
197
+ else:
198
+ mm_kwargs = self.runner.multi_modal_input_mapper(
199
+ mm_data,
200
+ seq_group_metadata.mm_processor_kwargs,
201
+ )
202
+
203
+ multi_modal_kwargs_list.append(mm_kwargs)
204
205
for modality, placeholder_map in placeholder_maps.items():
206
multi_modal_placeholder_maps[modality].extend(
@@ -264,7 +272,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
272
block_tables=torch.tensor([], device=self.device, dtype=torch.int),
273
)
274
267
- multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
275
+ multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
276
277
return (input_tokens, input_positions, attn_metadata, seq_lens,
278
multi_modal_kwargs)