A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Misc] Clean up input processing (#17582)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 2, 2025 at 23:11 UTC
cb234955dfd4cbad552f4bfe1de6c5a3981766a7
9 files changed
+357
-283
tests/models/multimodal/pooling/test_intern_vit.py
+8
-6
@@ -6,6 +6,7 @@ from huggingface_hub import snapshot_download
6
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
7
8
from vllm.distributed import cleanup_dist_env_and_memory
9
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10
11
from ....conftest import ImageTestAssets
12
@@ -14,6 +15,7 @@ from ....conftest import ImageTestAssets
15
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
16
17
18
+@torch.inference_mode()
19
def run_intern_vit_test(
20
image_assets: ImageTestAssets,
21
model_id: str,
@@ -21,11 +23,12 @@ def run_intern_vit_test(
23
dtype: str,
24
):
25
model = snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
26
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
27
28
img_processor = CLIPImageProcessor.from_pretrained(model)
29
images = [asset.pil_image for asset in image_assets]
30
pixel_values = [
28
- img_processor(images, return_tensors='pt').pixel_values.to(dtype)
31
+ img_processor(images, return_tensors='pt').pixel_values.to(torch_dtype)
32
for images in images
33
]
34
@@ -34,7 +37,7 @@ def run_intern_vit_test(
37
config.norm_type = "rms_norm"
38
39
hf_model = AutoModel.from_pretrained(model,
37
- torch_dtype=dtype,
40
+ torch_dtype=torch_dtype,
41
trust_remote_code=True).to("cuda")
42
hf_outputs_per_image = [
43
hf_model(pixel_value.to("cuda")).last_hidden_state
@@ -48,7 +51,7 @@ def run_intern_vit_test(
51
del hf_model
52
cleanup_dist_env_and_memory()
53
51
- vllm_model = vllm_model.to("cuda", dtype)
54
+ vllm_model = vllm_model.to("cuda", torch_dtype)
55
vllm_outputs_per_image = [
56
vllm_model(pixel_values=pixel_value.to("cuda"))
57
for pixel_value in pixel_values
@@ -66,9 +69,8 @@ def run_intern_vit_test(
69
"OpenGVLab/InternViT-300M-448px",
70
"OpenGVLab/InternViT-6B-448px-V1-5",
71
])
69
-@pytest.mark.parametrize("dtype", [torch.half])
70
-@torch.inference_mode()
71
-def test_models(image_assets, model_id, dtype: str) -> None:
72
+@pytest.mark.parametrize("dtype", ["half"])
73
+def test_models(dist_init, image_assets, model_id, dtype: str) -> None:
74
run_intern_vit_test(
75
image_assets,
76
model_id,
vllm/engine/async_llm_engine.py
-4
@@ -497,10 +497,6 @@ class _AsyncLLMEngine(LLMEngine):
497
prompt["prompt_token_ids"] = [0
498
] * prompt["prompt_embeds"].shape[-2]
499
500
- if self.tokenizer is not None:
501
- tokenizer = await self.get_tokenizer_async(lora_request)
502
- self._validate_token_prompt(prompt, tokenizer=tokenizer)
503
-
500
processed_inputs = await self.input_preprocessor.preprocess_async(
501
prompt,
502
lora_request=lora_request,
vllm/engine/llm_engine.py
+7
-27
@@ -30,7 +30,7 @@ from vllm.entrypoints.openai.logits_processors import (
30
get_logits_processors as get_openai_logits_processors)
31
from vllm.executor.executor_base import ExecutorBase
32
from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
33
-from vllm.inputs.parse import is_token_prompt, split_enc_dec_inputs
33
+from vllm.inputs.parse import split_enc_dec_inputs
34
from vllm.inputs.preprocess import InputPreprocessor
35
from vllm.logger import init_logger
36
from vllm.logits_process import get_bad_words_logits_processors
@@ -759,11 +759,6 @@ class LLMEngine:
759
seq_len = prompt["prompt_embeds"].shape[0]
760
prompt["prompt_token_ids"] = [0] * seq_len
761
762
- if self.tokenizer is not None:
763
- self._validate_token_prompt(
764
- prompt,
765
- tokenizer=self.get_tokenizer(lora_request=lora_request))
766
-
762
processed_inputs = self.input_preprocessor.preprocess(
763
prompt,
764
tokenization_kwargs=tokenization_kwargs,
@@ -782,27 +777,6 @@ class LLMEngine:
777
priority=priority,
778
)
779
785
- def _validate_token_prompt(self, prompt: PromptType,
786
- tokenizer: AnyTokenizer):
787
- # Guard against out-of-vocab tokens.
788
- # For some tokenizers, tokenizer.decode will happily return empty text
789
- # for token ids that are out of vocab, and we don't detect token ids
790
- # that are greater than the max token id before running the model.
791
- # However, these token ids will later crash a cuda kernel at runtime
792
- # with an index out of bounds error. This will crash the entire engine.
793
- # This needs to happen before multimodal input pre-processing, which
794
- # may add dummy <image> tokens that aren't part of the tokenizer's
795
- # vocabulary.
796
- if is_token_prompt(prompt):
797
- prompt_ids = prompt["prompt_token_ids"]
798
- if len(prompt_ids) == 0:
799
- # Empty prompt check is handled later
800
- return
801
- max_input_id = max(prompt_ids)
802
- if max_input_id > tokenizer.max_token_id:
803
- raise ValueError(
804
- "Token id {} is out of vocabulary".format(max_input_id))
805
-
780
def _create_sequence_group_with_sampling(
781
self,
782
request_id: str,
@@ -2049,6 +2023,12 @@ class LLMEngine:
2023
else:
2024
raise ValueError(f"The {prompt_type} prompt cannot be empty")
2025
2026
+ if tokenizer is not None:
2027
+ max_input_id = max(prompt_ids, default=0)
2028
+ if max_input_id > tokenizer.max_token_id:
2029
+ raise ValueError(
2030
+ f"Token id {max_input_id} is out of vocabulary")
2031
+
2032
max_prompt_len = self.model_config.max_model_len
2033
if len(prompt_ids) > max_prompt_len:
2034
if prompt_type == "encoder" and model_config.is_multimodal_model:
vllm/engine/protocol.py
+3
@@ -83,6 +83,9 @@ class EngineClient(ABC):
83
else:
84
processed_inputs = preprocessor._prompt_to_llm_inputs(prompt)
85
86
+ if processed_inputs["type"] == "embeds":
87
+ raise NotImplementedError
88
+
89
prompt_token_ids = processed_inputs["prompt_token_ids"]
90
prompt_text = processed_inputs.get("prompt")
91
multi_modal_data = processed_inputs.get("multi_modal_data")
vllm/entrypoints/llm.py
+4
-2
@@ -27,7 +27,7 @@ from vllm.entrypoints.score_utils import (_cosine_similarity,
27
_validate_score_input_lens)
28
from vllm.entrypoints.utils import _validate_truncation_size
29
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
30
-from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
30
+from vllm.inputs.parse import parse_and_batch_prompt
31
from vllm.logger import init_logger
32
from vllm.lora.request import LoRARequest
33
from vllm.model_executor.guided_decoding.guided_fields import (
@@ -567,10 +567,12 @@ class LLM:
567
mm_kwargs["mm_processor_kwargs"] = prompt[
568
"mm_processor_kwargs"]
569
570
- if is_token_prompt(prompt):
570
+ if "prompt_token_ids" in prompt:
571
+ prompt = cast(TokensPrompt, prompt) # Needed for mypy
572
prompt_tokens = prompt["prompt_token_ids"]
573
else:
574
prompt_tokens = tokenizer.encode(prompt["prompt"])
575
+
576
instances.append(
577
BeamSearchInstance(prompt_tokens, logprobs=None, **mm_kwargs))
578
vllm/inputs/data.py
+18
-5
@@ -70,6 +70,11 @@ class EmbedsPrompt(TypedDict):
70
prompt_embeds: torch.Tensor
71
"""The embeddings of the prompt."""
72
73
+ cache_salt: NotRequired[str]
74
+ """
75
+ Optional cache salt to be used for prefix caching.
76
+ """
77
+
78
79
SingletonPrompt = Union[str, TextPrompt, TokensPrompt, EmbedsPrompt]
80
"""
@@ -195,13 +200,21 @@ class EmbedsInputs(TypedDict):
200
prompt_embeds: torch.Tensor
201
"""The embeddings of the prompt."""
202
203
+ cache_salt: NotRequired[str]
204
+ """
205
+ Optional cache salt to be used for prefix caching.
206
+ """
207
+
208
199
-def embeds_inputs(prompt_embeds: torch.Tensor) -> EmbedsInputs:
209
+def embeds_inputs(
210
+ prompt_embeds: torch.Tensor,
211
+ cache_salt: Optional[str] = None,
212
+) -> EmbedsInputs:
213
"""Construct :class:`EmbedsInputs` from optional values."""
201
- inputs = EmbedsInputs(
202
- type="embeds",
203
- prompt_embeds=prompt_embeds,
204
- )
214
+ inputs = EmbedsInputs(type="embeds", prompt_embeds=prompt_embeds)
215
+
216
+ if cache_salt is not None:
217
+ inputs["cache_salt"] = cache_salt
218
219
return inputs
220
vllm/inputs/parse.py
+8
-19
@@ -6,9 +6,9 @@ from typing_extensions import TypeIs
6
7
from vllm.utils import is_list_of
8
9
-from .data import (EmbedsInputs, EmbedsPrompt, ExplicitEncoderDecoderPrompt,
10
- ProcessorInputs, PromptType, SingletonInputs,
11
- SingletonPrompt, TextPrompt, TokensPrompt)
9
+from .data import (EmbedsPrompt, ExplicitEncoderDecoderPrompt, ProcessorInputs,
10
+ PromptType, SingletonInputs, SingletonPrompt, TextPrompt,
11
+ TokensPrompt)
12
13
14
class ParsedText(TypedDict):
@@ -90,6 +90,10 @@ class ParsedEmbedsPrompt(TypedDict):
90
content: EmbedsPrompt
91
92
93
+ParsedSingletonPrompt = Union[ParsedStrPrompt, ParsedTextPrompt,
94
+ ParsedTokensPrompt, ParsedEmbedsPrompt]
95
+
96
+
97
@overload
98
def parse_singleton_prompt(prompt: str) -> ParsedStrPrompt:
99
...
@@ -110,10 +114,7 @@ def parse_singleton_prompt(prompt: EmbedsPrompt) -> ParsedEmbedsPrompt:
114
...
115
116
113
-def parse_singleton_prompt(
114
- prompt: SingletonPrompt,
115
-) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt,
116
- ParsedEmbedsPrompt]:
117
+def parse_singleton_prompt(prompt: SingletonPrompt) -> ParsedSingletonPrompt:
118
if isinstance(prompt, str):
119
return ParsedStrPrompt(type="str", content=prompt)
120
elif isinstance(prompt, dict):
@@ -131,23 +132,11 @@ def parse_singleton_prompt(
132
"inputs must be a string, TextPrompt, TokensPrompt, or EmbedsPrompt")
133
134
134
-def is_token_prompt(prompt: PromptType) -> TypeIs[TokensPrompt]:
135
- return isinstance(prompt, dict) and "prompt_token_ids" in prompt
136
-
137
-
138
-def is_embeds_prompt(prompt: PromptType) -> TypeIs[EmbedsPrompt]:
139
- return isinstance(prompt, dict) and "prompt_embeds" in prompt
140
-
141
-
135
def is_explicit_encoder_decoder_prompt(
136
prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
137
return isinstance(prompt, dict) and "encoder_prompt" in prompt
138
139
147
-def is_embeds_inputs(inputs: SingletonInputs) -> TypeIs[EmbedsInputs]:
148
- return isinstance(inputs, dict) and inputs["type"] == "embeds"
149
-
150
-
140
def split_enc_dec_inputs(
141
inputs: ProcessorInputs,
142
) -> tuple[Optional[SingletonInputs], SingletonInputs]:
vllm/inputs/preprocess.py
+302
-215
@@ -14,14 +14,14 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
15
MultiModalInputs)
16
from vllm.prompt_adapter.request import PromptAdapterRequest
17
+from vllm.transformers_utils.tokenizer import AnyTokenizer
18
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
19
19
-from .data import (DecoderOnlyInputs, EmbedsInputs, EncoderDecoderInputs,
20
- ProcessorInputs, PromptType, SingletonInputs,
21
- SingletonPrompt, TokenInputs, embeds_inputs, token_inputs)
22
-from .parse import (ParsedEmbedsPrompt, ParsedStrPrompt, ParsedTextPrompt,
23
- ParsedTokensPrompt, is_embeds_inputs,
24
- is_explicit_encoder_decoder_prompt, parse_singleton_prompt)
20
+from .data import (DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,
21
+ EncoderDecoderInputs, ProcessorInputs, PromptType,
22
+ SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
23
+ TokensPrompt, embeds_inputs, token_inputs)
24
+from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
25
26
logger = init_logger(__name__)
27
@@ -140,13 +140,10 @@ class InputPreprocessor:
140
"""
141
Prepares `decoder_input_ids` for generation with encoder-decoder models.
142
143
- Based on
144
-
145
- https://github.com/huggingface/transformers/blob/
146
- 4037a2b5b1278736e566aec12e169100275545ea/
147
- src/transformers/generation/utils.py
148
-
149
- specifically GenerationMixin._prepare_decoder_input_ids_for_generation()
143
+ Based on:
144
+ https://github.com/huggingface/transformers/blob/4037a2b5b1278736e566aec12e169100275545ea/src/transformers/generation/utils.py
145
+ specifically,
146
+ `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
147
148
Arguments:
149
@@ -183,6 +180,23 @@ class InputPreprocessor:
180
181
return prompt_token_ids
182
183
+ def _get_tokenization_kw(
184
+ self,
185
+ overrides: Optional[dict[str, Any]] = None,
186
+ ) -> dict[str, Any]:
187
+ kwargs = dict[str, Any]()
188
+
189
+ if self.model_config.hf_config.model_type == "whisper":
190
+ # For Whisper, special tokens should be provided by the user based
191
+ # on the task and language of their request. Also needed to avoid
192
+ # appending an EOS token to the prompt which disrupts generation.
193
+ kwargs["add_special_tokens"] = False
194
+
195
+ if overrides:
196
+ kwargs.update(overrides)
197
+
198
+ return kwargs
199
+
200
def _tokenize_prompt(
201
self,
202
prompt: str,
@@ -194,18 +208,11 @@ class InputPreprocessor:
208
corresponding token IDs.
209
"""
210
tokenizer = self.get_tokenizer_group()
197
- if tokenization_kwargs is None:
198
- tokenization_kwargs = {}
211
+ tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
212
200
- if self.model_config.hf_config.model_type == "whisper":
201
- # For Whisper, special tokens should be provided by the user based
202
- # on the task and language of their request. Also needed to avoid
203
- # appending an EOS token to the prompt which disrupts generation.
204
- tokenization_kwargs["add_special_tokens"] = False
213
+ encoder_config = self.model_config.encoder_config
214
206
- if (self.model_config.encoder_config is not None
207
- and self.model_config.encoder_config.get(
208
- "do_lower_case", False)):
215
+ if encoder_config and encoder_config.get("do_lower_case", False):
216
prompt = prompt.lower()
217
218
return tokenizer.encode(prompt=prompt,
@@ -220,18 +227,36 @@ class InputPreprocessor:
227
) -> list[int]:
228
"""Async version of :meth:`_tokenize_prompt`."""
229
tokenizer = self.get_tokenizer_group()
223
- if tokenization_kwargs is None:
224
- tokenization_kwargs = {}
230
+ tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
231
226
- if self.model_config.hf_config.model_type == "whisper":
227
- # For Whisper, special tokens should be provided by the user based
228
- # on the task and language of their request. Also needed to avoid
229
- # appending an EOS token to the prompt which disrupts generation.
230
- tokenization_kwargs["add_special_tokens"] = False
232
return await tokenizer.encode_async(prompt=prompt,
233
lora_request=lora_request,
234
**tokenization_kwargs)
235
236
+ def _get_mm_tokenizer(
237
+ self,
238
+ lora_request: Optional[LoRARequest],
239
+ ) -> AnyTokenizer:
240
+ # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
241
+ # while using also multi-modal input
242
+ if not self.tokenizer:
243
+ return cast(AnyTokenizer, object()) # Dummy
244
+
245
+ tokenizer_group = self.get_tokenizer_group()
246
+ return tokenizer_group.get_lora_tokenizer(lora_request)
247
+
248
+ async def _get_mm_tokenizer_async(
249
+ self,
250
+ lora_request: Optional[LoRARequest],
251
+ ) -> AnyTokenizer:
252
+ # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
253
+ # while using also multi-modal input
254
+ if not self.tokenizer:
255
+ return cast(AnyTokenizer, object()) # Dummy
256
+
257
+ tokenizer_group = self.get_tokenizer_group()
258
+ return await tokenizer_group.get_lora_tokenizer_async(lora_request)
259
+
260
def _process_multimodal(
261
self,
262
prompt: Union[str, list[int]],
@@ -244,13 +269,7 @@ class InputPreprocessor:
269
Apply the model's multi-modal processor to a multi-modal prompt,
270
returning the corresponding token IDs and metadata.
271
"""
247
- # At the moment on model (PrithviGeoSpatialMAE) requires to be
248
- # initialized without a tokenizer while using also multi-modal input
249
- if not self.tokenizer:
250
- tokenizer = object() # Dummy
251
- else:
252
- tokenizer_group = self.get_tokenizer_group()
253
- tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
272
+ tokenizer = self._get_mm_tokenizer(lora_request)
273
274
mm_processor = self.mm_registry.create_processor(self.model_config,
275
tokenizer=tokenizer)
@@ -270,14 +289,7 @@ class InputPreprocessor:
289
return_mm_hashes: bool = False,
290
) -> MultiModalInputs:
291
"""Async version of :meth:`_process_multimodal`."""
273
- # At the moment on model (PrithviGeoSpatialMAE) requires to be
274
- # initialized without a tokenizer while using also multi-modal input
275
- if not self.tokenizer:
276
- tokenizer = object() # Dummy
277
- else:
278
- tokenizer_group = self.get_tokenizer_group()
279
- tokenizer = await tokenizer_group.get_lora_tokenizer_async(
280
- lora_request)
292
+ tokenizer = await self._get_mm_tokenizer_async(lora_request)
293
294
mm_processor = self.mm_registry.create_processor(self.model_config,
295
tokenizer=tokenizer)
@@ -287,28 +299,160 @@ class InputPreprocessor:
299
return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,
300
return_mm_hashes)
301
290
- def _get_prompt_data(self, parsed_prompt: Union[ParsedStrPrompt,
291
- ParsedTextPrompt,
292
- ParsedTokensPrompt]):
293
- prompt_text = None
294
- prompt_token_ids = None
295
- token_type_ids = None
296
- cache_salt = None
302
+ def _process_embeds(
303
+ self,
304
+ parsed_content: EmbedsPrompt,
305
+ ) -> EmbedsInputs:
306
+ if envs.VLLM_USE_V1:
307
+ raise ValueError("prompt_embeds is only available in V0.")
308
+
309
+ prompt_embeds = parsed_content["prompt_embeds"]
310
298
- if parsed_prompt["type"] == "str":
299
- prompt_text = parsed_prompt["content"]
311
+ # prompt_embeds must be (seq_len, hidden_size), but if the user
312
+ # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),
313
+ # we can unambiguously process the intent by squeezing the batch
314
+ # dimension.
315
+ if prompt_embeds.ndim == 3:
316
+ prompt_embeds = prompt_embeds.squeeze(dim=0)
317
+
318
+ if prompt_embeds.ndim != 2:
319
+ raise ValueError(
320
+ "prompt_embeds must be of shape (seq_len, hidden_size).")
321
+
322
+ return embeds_inputs(prompt_embeds=prompt_embeds,
323
+ cache_salt=parsed_content.get("cache_salt"))
324
+
325
+ async def _process_embeds_async(
326
+ self,
327
+ parsed_content: EmbedsPrompt,
328
+ ) -> EmbedsInputs:
329
+ return self._process_embeds(parsed_content)
330
+
331
+ def _process_tokens(
332
+ self,
333
+ parsed_content: TokensPrompt,
334
+ lora_request: Optional[LoRARequest] = None,
335
+ return_mm_hashes: bool = False,
336
+ ) -> Union[TokenInputs, MultiModalInputs]:
337
+ prompt_token_ids = parsed_content["prompt_token_ids"]
338
+ token_type_ids = parsed_content.get("token_type_ids")
339
+
340
+ inputs: Union[TokenInputs, MultiModalInputs]
341
+ if multi_modal_data := parsed_content.get("multi_modal_data"):
342
+ inputs = self._process_multimodal(
343
+ prompt_token_ids,
344
+ multi_modal_data,
345
+ parsed_content.get("mm_processor_kwargs"),
346
+ lora_request=lora_request,
347
+ return_mm_hashes=return_mm_hashes,
348
+ )
349
else:
301
- cache_salt = parsed_prompt["content"].get("cache_salt")
302
- if parsed_prompt["type"] == "text":
303
- prompt_text = parsed_prompt["content"]["prompt"]
304
- elif parsed_prompt["type"] == "tokens":
305
- prompt_token_ids = parsed_prompt["content"].get(
306
- "prompt_token_ids")
307
- token_type_ids = parsed_prompt["content"].get("token_type_ids")
308
- else:
309
- assert_never(parsed_prompt)
350
+ inputs = token_inputs(
351
+ prompt_token_ids=prompt_token_ids,
352
+ token_type_ids=token_type_ids,
353
+ )
354
+
355
+ if cache_salt := parsed_content.get("cache_salt"):
356
+ inputs["cache_salt"] = cache_salt
357
+
358
+ return inputs
359
+
360
+ async def _process_tokens_async(
361
+ self,
362
+ parsed_content: TokensPrompt,
363
+ lora_request: Optional[LoRARequest] = None,
364
+ return_mm_hashes: bool = False,
365
+ ) -> Union[TokenInputs, MultiModalInputs]:
366
+ prompt_token_ids = parsed_content["prompt_token_ids"]
367
+ token_type_ids = parsed_content.get("token_type_ids")
368
+
369
+ inputs: Union[TokenInputs, MultiModalInputs]
370
+ if multi_modal_data := parsed_content.get("multi_modal_data"):
371
+ inputs = await self._process_multimodal_async(
372
+ prompt_token_ids,
373
+ multi_modal_data,
374
+ parsed_content.get("mm_processor_kwargs"),
375
+ lora_request=lora_request,
376
+ return_mm_hashes=return_mm_hashes,
377
+ )
378
+ else:
379
+ inputs = token_inputs(
380
+ prompt_token_ids=prompt_token_ids,
381
+ token_type_ids=token_type_ids,
382
+ )
383
+
384
+ if cache_salt := parsed_content.get("cache_salt"):
385
+ inputs["cache_salt"] = cache_salt
386
+
387
+ return inputs
388
+
389
+ def _process_text(
390
+ self,
391
+ parsed_content: TextPrompt,
392
+ tokenization_kwargs: Optional[dict[str, Any]] = None,
393
+ lora_request: Optional[LoRARequest] = None,
394
+ return_mm_hashes: bool = False,
395
+ ) -> Union[TokenInputs, MultiModalInputs]:
396
+ prompt_text = parsed_content["prompt"]
397
+
398
+ inputs: Union[TokenInputs, MultiModalInputs]
399
+ if multi_modal_data := parsed_content.get("multi_modal_data"):
400
+ inputs = self._process_multimodal(
401
+ prompt_text,
402
+ multi_modal_data,
403
+ parsed_content.get("mm_processor_kwargs"),
404
+ lora_request=lora_request,
405
+ return_mm_hashes=return_mm_hashes,
406
+ )
407
+ else:
408
+ prompt_token_ids = self._tokenize_prompt(
409
+ prompt_text,
410
+ lora_request=lora_request,
411
+ tokenization_kwargs=tokenization_kwargs,
412
+ )
413
+ inputs = token_inputs(
414
+ prompt=prompt_text,
415
+ prompt_token_ids=prompt_token_ids,
416
+ )
417
+
418
+ if cache_salt := parsed_content.get("cache_salt"):
419
+ inputs["cache_salt"] = cache_salt
420
+
421
+ return inputs
422
311
- return prompt_text, prompt_token_ids, token_type_ids, cache_salt
423
+ async def _process_text_async(
424
+ self,
425
+ parsed_content: TextPrompt,
426
+ tokenization_kwargs: Optional[dict[str, Any]] = None,
427
+ lora_request: Optional[LoRARequest] = None,
428
+ return_mm_hashes: bool = False,
429
+ ) -> Union[TokenInputs, MultiModalInputs]:
430
+ prompt_text = parsed_content["prompt"]
431
+
432
+ inputs: Union[TokenInputs, MultiModalInputs]
433
+ if multi_modal_data := parsed_content.get("multi_modal_data"):
434
+ inputs = await self._process_multimodal_async(
435
+ prompt_text,
436
+ multi_modal_data,
437
+ parsed_content.get("mm_processor_kwargs"),
438
+ lora_request=lora_request,
439
+ return_mm_hashes=return_mm_hashes,
440
+ )
441
+ else:
442
+ prompt_token_ids = await self._tokenize_prompt_async(
443
+ prompt_text,
444
+ lora_request=lora_request,
445
+ tokenization_kwargs=tokenization_kwargs,
446
+ )
447
+ inputs = token_inputs(
448
+ prompt=prompt_text,
449
+ prompt_token_ids=prompt_token_ids,
450
+ )
451
+
452
+ if cache_salt := parsed_content.get("cache_salt"):
453
+ inputs["cache_salt"] = cache_salt
454
+
455
+ return inputs
456
457
def _prompt_to_llm_inputs(
458
self,
@@ -333,39 +477,28 @@ class InputPreprocessor:
477
parsed = parse_singleton_prompt(prompt)
478
479
if parsed["type"] == "embeds":
336
- return self._process_prompt_embeds(parsed)
337
-
338
- prompt_text, prompt_token_ids, token_type_ids, cache_salt = \
339
- self._get_prompt_data(parsed)
340
-
341
- # If multimodal data is present, process and return immediately
342
- if parsed["type"] != "str" and parsed["content"].get(
343
- "multi_modal_data") is not None:
344
- inputs = self._process_multimodal(
345
- prompt_text if prompt_text is not None else prompt_token_ids,
346
- parsed["content"]["multi_modal_data"],
347
- parsed["content"].get("mm_processor_kwargs"),
480
+ return self._process_embeds(parsed["content"])
481
+ if parsed["type"] == "tokens":
482
+ return self._process_tokens(
483
+ parsed["content"],
484
lora_request=lora_request,
485
return_mm_hashes=return_mm_hashes,
486
)
351
- if cache_salt is not None:
352
- inputs["cache_salt"] = cache_salt
353
- return inputs
354
-
355
- if prompt_token_ids is None:
356
- prompt_token_ids = self._tokenize_prompt(
357
- prompt_text,
487
+ if parsed["type"] == "text":
488
+ return self._process_text(
489
+ parsed["content"],
490
+ tokenization_kwargs=tokenization_kwargs,
491
lora_request=lora_request,
492
+ return_mm_hashes=return_mm_hashes,
493
+ )
494
+ if parsed["type"] == "str":
495
+ return self._process_text(
496
+ TextPrompt(prompt=parsed["content"]),
497
tokenization_kwargs=tokenization_kwargs,
498
+ lora_request=lora_request,
499
+ return_mm_hashes=return_mm_hashes,
500
)
501
362
- return token_inputs(
363
- prompt=prompt_text,
364
- prompt_token_ids=prompt_token_ids,
365
- token_type_ids=token_type_ids,
366
- cache_salt=cache_salt,
367
- )
368
-
502
assert_never(parsed)
503
504
async def _prompt_to_llm_inputs_async(
@@ -375,79 +508,49 @@ class InputPreprocessor:
508
lora_request: Optional[LoRARequest] = None,
509
return_mm_hashes: bool = False,
510
) -> SingletonInputs:
378
- """Async version of :meth:`_extract_prompt_components`."""
511
+ """Async version of :meth:`_prompt_to_llm_inputs`."""
512
parsed = parse_singleton_prompt(prompt)
513
514
if parsed["type"] == "embeds":
382
- return self._process_prompt_embeds(parsed)
383
-
384
- prompt_text, prompt_token_ids, token_type_ids, cache_salt = \
385
- self._get_prompt_data(parsed)
386
-
387
- if parsed["type"] != "str" and parsed["content"].get(
388
- "multi_modal_data") is not None:
389
- inputs = await self._process_multimodal_async(
390
- prompt_token_ids if prompt_text is None else prompt_text,
391
- parsed["content"]["multi_modal_data"],
392
- parsed["content"].get("mm_processor_kwargs"),
515
+ return await self._process_embeds_async(parsed["content"])
516
+ if parsed["type"] == "tokens":
517
+ return await self._process_tokens_async(
518
+ parsed["content"],
519
lora_request=lora_request,
520
return_mm_hashes=return_mm_hashes,
521
)
396
- if cache_salt is not None:
397
- inputs["cache_salt"] = cache_salt
398
- return inputs
399
-
400
- if prompt_token_ids is None:
401
- prompt_token_ids = await self._tokenize_prompt_async(
402
- prompt_text,
522
+ if parsed["type"] == "text":
523
+ return await self._process_text_async(
524
+ parsed["content"],
525
+ tokenization_kwargs=tokenization_kwargs,
526
lora_request=lora_request,
527
+ return_mm_hashes=return_mm_hashes,
528
+ )
529
+ if parsed["type"] == "str":
530
+ return await self._process_text_async(
531
+ TextPrompt(prompt=parsed["content"]),
532
tokenization_kwargs=tokenization_kwargs,
533
+ lora_request=lora_request,
534
+ return_mm_hashes=return_mm_hashes,
535
)
536
407
- return token_inputs(
408
- prompt=prompt_text,
409
- prompt_token_ids=prompt_token_ids,
410
- token_type_ids=token_type_ids,
411
- cache_salt=cache_salt,
412
- )
413
-
414
- def _process_prompt_embeds(self,
415
- parsed: ParsedEmbedsPrompt) -> EmbedsInputs:
416
- if envs.VLLM_USE_V1:
417
- raise ValueError("prompt_embeds is only available in V0.")
418
-
419
- prompt_embeds_content = parsed["content"]
420
-
421
- prompt_embeds = prompt_embeds_content["prompt_embeds"]
422
-
423
- # prompt_embeds must be (seq_len, hidden_size), but if the user
424
- # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),
425
- # we can unambiguously process the intent by squeezing the batch
426
- # dimension.
427
- if prompt_embeds.ndim == 3 and prompt_embeds.shape[0] == 1:
428
- prompt_embeds = prompt_embeds.squeeze(dim=0)
429
-
430
- if prompt_embeds.ndim != 2:
431
- raise ValueError(
432
- "prompt_embeds must be of shape (seq_len, hidden_size).")
433
-
434
- return embeds_inputs(prompt_embeds=prompt_embeds)
435
-
537
assert_never(parsed)
538
539
def _build_enc_dec_llm_inputs(
540
self,
440
- encoder_inputs: Union[TokenInputs, MultiModalInputs],
441
- decoder_inputs: Optional[Union[TokenInputs, MultiModalInputs]],
541
+ encoder_inputs: SingletonInputs,
542
+ decoder_inputs: Optional[SingletonInputs],
543
) -> EncoderDecoderInputs:
443
- if (encoder_inputs["type"] == "token"
444
- or encoder_inputs["type"] == "multimodal"):
445
- pass
446
- else:
447
- assert_never(encoder_inputs) # type: ignore[arg-type]
544
+ if (encoder_inputs["type"] == "embeds"
545
+ or decoder_inputs and decoder_inputs["type"] == "embeds"):
546
+ raise ValueError("Embedding inputs are not supported for encoder-"
547
+ "decoder models")
548
449
- # Mypy does not correctly infer that EmbedsInputs is impossible
450
- assert "prompt_token_ids" in encoder_inputs
549
+ # Needed for mypy
550
+ encoder_inputs = cast(Union[TokenInputs, MultiModalInputs],
551
+ encoder_inputs)
552
+ decoder_inputs = cast(Optional[Union[TokenInputs, MultiModalInputs]],
553
+ decoder_inputs)
554
555
if decoder_inputs is None:
556
if self.model_config.hf_config.model_type == "whisper":
@@ -460,74 +563,78 @@ class InputPreprocessor:
563
dec_token_ids = self._prepare_decoder_input_ids_for_generation(
564
None)
565
decoder_inputs = token_inputs(dec_token_ids)
463
- elif (decoder_inputs["type"] == "token"
464
- or decoder_inputs["type"] == "multimodal"):
465
- dec_token_ids = self._prepare_decoder_input_ids_for_generation(
466
- decoder_inputs["prompt_token_ids"])
467
- decoder_inputs["prompt_token_ids"] = dec_token_ids
468
-
566
+ else:
567
if "multi_modal_data" in decoder_inputs:
568
raise ValueError("Multi-modal decoder inputs of encoder-"
569
"decoder models are not supported yet")
472
- else:
473
- assert_never(encoder_inputs) # type: ignore[arg-type]
570
+
571
+ dec_token_ids = self._prepare_decoder_input_ids_for_generation(
572
+ decoder_inputs["prompt_token_ids"])
573
+ decoder_inputs["prompt_token_ids"] = dec_token_ids
574
575
return EncoderDecoderInputs(
576
encoder=encoder_inputs,
577
decoder=decoder_inputs,
578
)
579
480
- def _separate_enc_dec_inputs_from_mm_processor_outputs(
580
+ def _split_enc_dec_mm_inputs(
581
self,
482
- inputs: SingletonInputs,
483
- decoder_inputs_to_override: Optional[Union[TokenInputs,
484
- MultiModalInputs]] = None,
582
+ inputs: Union[SingletonInputs, MultiModalEncDecInputs],
583
+ decoder_inputs_to_override: Optional[SingletonInputs] = None,
584
) -> tuple[SingletonInputs, SingletonInputs]:
585
"""
586
For encoder/decoder models only:
587
Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
588
"""
589
+ if (inputs["type"] == "embeds" or decoder_inputs_to_override
590
+ and decoder_inputs_to_override["type"] == "embeds"):
591
+ raise ValueError("Embedding inputs are not supported for encoder-"
592
+ "decoder models")
593
+
594
+ # Needed for mypy
595
+ inputs = cast(
596
+ Union[TokenInputs, MultiModalInputs, MultiModalEncDecInputs],
597
+ inputs,
598
+ )
599
+ decoder_inputs_to_override = cast(
600
+ Optional[Union[TokenInputs, MultiModalInputs]],
601
+ decoder_inputs_to_override,
602
+ )
603
+
604
encoder_inputs: SingletonInputs
605
decoder_inputs: SingletonInputs
492
- if inputs["type"] == "multimodal":
493
- # Multimodal data inputs
494
- assert ("encoder_prompt" in inputs
495
- and "encoder_prompt_token_ids" in inputs)
606
+
607
+ if inputs["type"] == "multimodal": # Multimodal data inputs
608
+ if not ("encoder_prompt" in inputs
609
+ and "encoder_prompt_token_ids" in inputs):
610
+ raise RuntimeError("You should register an encoder-decoder "
611
+ "multi-modal processor for encoder-decoder "
612
+ "models.")
613
inputs = cast(MultiModalEncDecInputs, inputs)
614
+
615
encoder_inputs = token_inputs(
616
prompt=inputs["encoder_prompt"],
617
prompt_token_ids=inputs["encoder_prompt_token_ids"],
618
)
501
- if decoder_inputs_to_override is not None:
502
- decoder_inputs = MultiModalInputs(
503
- type="multimodal",
504
- prompt=decoder_inputs_to_override.get("prompt", ""),
505
- prompt_token_ids=decoder_inputs_to_override[
506
- "prompt_token_ids"],
507
- mm_kwargs=inputs["mm_kwargs"],
508
- mm_hashes=inputs["mm_hashes"],
509
- mm_placeholders=inputs["mm_placeholders"],
510
- )
511
- else:
512
- decoder_inputs = MultiModalInputs(
513
- type="multimodal",
514
- prompt=inputs["prompt"],
515
- prompt_token_ids=inputs["prompt_token_ids"],
516
- mm_kwargs=inputs["mm_kwargs"],
517
- mm_hashes=inputs["mm_hashes"],
518
- mm_placeholders=inputs["mm_placeholders"],
519
- )
619
521
- cache_salt = inputs.get("cache_salt")
522
- if cache_salt is not None:
620
+ decoder_prompt_inputs = decoder_inputs_to_override or inputs
621
+ decoder_inputs = MultiModalInputs(
622
+ type="multimodal",
623
+ prompt=decoder_prompt_inputs.get("prompt", ""),
624
+ prompt_token_ids=decoder_prompt_inputs["prompt_token_ids"],
625
+ mm_kwargs=inputs["mm_kwargs"],
626
+ mm_hashes=inputs["mm_hashes"],
627
+ mm_placeholders=inputs["mm_placeholders"],
628
+ )
629
+ if cache_salt := inputs.get("cache_salt"):
630
decoder_inputs["cache_salt"] = cache_salt
631
525
- elif inputs["type"] == "token":
526
- # Text-only inputs
632
+ elif inputs["type"] == "token": # Text-only inputs
633
encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
634
decoder_inputs = decoder_inputs_to_override or inputs
635
else:
636
assert_never(inputs) # type: ignore[arg-type]
637
+
638
return encoder_inputs, decoder_inputs
639
640
def _process_encoder_decoder_prompt(
@@ -580,11 +687,9 @@ class InputPreprocessor:
687
# For multimodal model, override decoder prompt from processor
688
# with explicit decoder prompt.
689
if self.model_config.is_multimodal_model:
583
- assert decoder_inputs is None or not is_embeds_inputs(
584
- decoder_inputs)
690
encoder_inputs, decoder_inputs = (
586
- self._separate_enc_dec_inputs_from_mm_processor_outputs(
587
- encoder_inputs, decoder_inputs))
691
+ self._split_enc_dec_mm_inputs(encoder_inputs,
692
+ decoder_inputs))
693
else:
694
inputs = self._prompt_to_llm_inputs(
695
prompt,
@@ -593,16 +698,11 @@ class InputPreprocessor:
698
if self.model_config.is_multimodal_model:
699
# Encoder-Decoder Multimodal model
700
encoder_inputs, decoder_inputs = (
596
- self._separate_enc_dec_inputs_from_mm_processor_outputs(
597
- inputs))
701
+ self._split_enc_dec_mm_inputs(inputs))
702
else:
703
encoder_inputs = inputs
704
decoder_inputs = None
705
602
- # Mypy does not do type inference well with TypedDicts with Literal
603
- # values.
604
- assert not is_embeds_inputs(encoder_inputs)
605
- assert decoder_inputs is None or not is_embeds_inputs(decoder_inputs)
706
return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
707
708
async def _process_encoder_decoder_prompt_async(
@@ -635,11 +735,9 @@ class InputPreprocessor:
735
# For multimodal model, override decoder prompt from processor
736
# with explicit decoder prompt.
737
if self.model_config.is_multimodal_model:
638
- assert decoder_inputs is None or not is_embeds_inputs(
639
- decoder_inputs)
738
encoder_inputs, decoder_inputs = (
641
- self._separate_enc_dec_inputs_from_mm_processor_outputs(
642
- encoder_inputs, decoder_inputs))
739
+ self._split_enc_dec_mm_inputs(encoder_inputs,
740
+ decoder_inputs))
741
else:
742
inputs = await self._prompt_to_llm_inputs_async(
743
prompt,
@@ -648,16 +746,11 @@ class InputPreprocessor:
746
if self.model_config.is_multimodal_model:
747
# Encoder-Decoder Multimodal model
748
encoder_inputs, decoder_inputs = (
651
- self._separate_enc_dec_inputs_from_mm_processor_outputs(
652
- inputs))
749
+ self._split_enc_dec_mm_inputs(inputs))
750
else:
751
encoder_inputs = inputs
752
decoder_inputs = None
753
657
- # Mypy does not do type inference well with TypedDicts with Literal
658
- # values.
659
- assert not is_embeds_inputs(encoder_inputs)
660
- assert decoder_inputs is None or not is_embeds_inputs(decoder_inputs)
754
return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
755
756
def _build_decoder_only_llm_inputs(
@@ -665,19 +758,13 @@ class InputPreprocessor:
758
prompt_inputs: DecoderOnlyInputs,
759
prompt_adapter_request: Optional[PromptAdapterRequest],
760
) -> DecoderOnlyInputs:
668
- if (prompt_inputs["type"] == "token"
669
- or prompt_inputs["type"] == "multimodal"):
670
- # Mypy does not do type inference well with typedicts and Literal
671
- # values
672
- assert not is_embeds_inputs(prompt_inputs)
761
+ if "prompt_token_ids" in prompt_inputs:
762
+ prompt_inputs = cast(Union[TokenInputs, MultiModalInputs],
763
+ prompt_inputs) # Needed for mypy
764
prompt_inputs["prompt_token_ids"] = self._apply_prompt_adapter(
765
prompt_inputs["prompt_token_ids"],
766
prompt_adapter_request=prompt_adapter_request,
767
)
677
- elif (prompt_inputs["type"] == "embeds"):
678
- pass
679
- else:
680
- assert_never(prompt_inputs) # type: ignore[arg-type]
768
769
return prompt_inputs
770
vllm/multimodal/processing.py
+7
-5
@@ -1670,15 +1670,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1670
placeholders = mm_placeholders.get(modality, [])
1671
1672
if len(placeholders) != item_count:
1673
+ # NOTE: If you are a model developer, this can also arise from
1674
+ # an inconsistency between `_call_hf_processor` and
1675
+ # `_get_mm_fields_config` implementations
1676
raise RuntimeError(
1677
f"Expected there to be {item_count} prompt updates "
1678
f"corresponding to {item_count} {modality} items, but "
1679
f"instead found {len(placeholders)} prompt updates! "
1677
- "Either the prompt text has missing/incorrect tokens for "
1678
- "multi-modal inputs, or there is a problem with your "
1679
- "implementation of merged multi-modal processor for this "
1680
- "model (usually arising from an inconsistency between "
1681
- "`_call_hf_processor` and `_get_prompt_updates`).")
1680
+ "This is likely because you forgot to include input "
1681
+ "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
1682
+ "in the prompt. If the model has a chat template, make "
1683
+ "sure you have applied it before calling `LLM.generate`.")
1684
1685
def _maybe_apply_prompt_updates(
1686
self,