A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 20:15 UTC
[Refactor] Clean up input preprocessing (#33687)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 5, 2026 at 18:43 UTC
7bd42e609d24501f59a8b405229ed91f4ca8037c
4 files changed
+91
-204
vllm/inputs/data.py
+3
-1
@@ -12,11 +12,13 @@ from vllm.sampling_params import SamplingParams
12
if TYPE_CHECKING:
13
from vllm.multimodal.inputs import (
14
MultiModalDataDict,
15
+ MultiModalEncDecInputs,
16
MultiModalInputs,
17
MultiModalUUIDDict,
18
)
19
else:
20
MultiModalDataDict = object
21
+ MultiModalEncDecInputs = object
22
MultiModalInputs = object
23
MultiModalUUIDDict = object
24
@@ -241,7 +243,7 @@ class EncoderDecoderInputs(TypedDict):
243
This specifies the required data for encoder-decoder models.
244
"""
245
244
- encoder: TokenInputs | MultiModalInputs
246
+ encoder: TokenInputs | MultiModalEncDecInputs
247
"""The inputs for the encoder portion."""
248
249
decoder: TokenInputs | MultiModalInputs
vllm/inputs/parse.py
+16
@@ -69,6 +69,22 @@ def is_explicit_encoder_decoder_prompt(
69
return isinstance(prompt, dict) and "encoder_prompt" in prompt
70
71
72
+def split_enc_dec_prompt(
73
+ prompt: PromptType,
74
+) -> tuple[SingletonPrompt, SingletonPrompt | None]:
75
+ if isinstance(prompt, str):
76
+ return prompt, None
77
+
78
+ if "encoder_prompt" in prompt and "decoder_prompt" in prompt:
79
+ # NOTE: This passes pyright but not mypy
80
+ return (
81
+ prompt["encoder_prompt"], # type: ignore[typeddict-item]
82
+ prompt["decoder_prompt"], # type: ignore[typeddict-item]
83
+ )
84
+
85
+ return prompt, None
86
+
87
+
88
def split_enc_dec_inputs(
89
inputs: ProcessorInputs,
90
) -> tuple[SingletonInputs | None, SingletonInputs]:
vllm/inputs/preprocess.py
+68
-203
@@ -2,11 +2,12 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from collections.abc import Mapping
5
-from typing import Any, cast
5
+from typing import Any
6
7
from typing_extensions import assert_never
8
9
from vllm.config import ModelConfig, ObservabilityConfig
10
+from vllm.inputs.parse import split_enc_dec_prompt
11
from vllm.logger import init_logger
12
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
13
from vllm.multimodal.cache import BaseMultiModalProcessorCache
@@ -27,7 +28,6 @@ from .data import (
28
EmbedsInputs,
29
EmbedsPrompt,
30
EncoderDecoderInputs,
30
- ExplicitEncoderDecoderPrompt,
31
ProcessorInputs,
32
PromptType,
33
SingletonInputs,
@@ -86,30 +86,15 @@ class InputPreprocessor:
86
87
return self.tokenizer.eos_token_id
88
89
- def get_decoder_start_token_id(self) -> int | None:
89
+ def get_decoder_start_token_id(self) -> int:
90
"""
91
Obtain the decoder start token id employed by an encoder/decoder
92
- model. Returns None for non-encoder/decoder models or if the
93
- model config is unavailable.
92
+ model. Raises an error if it is not available.
93
"""
95
-
96
- if not self.model_config.is_encoder_decoder:
97
- logger.warning_once(
98
- "Using None for decoder start token id because "
99
- "this is not an encoder/decoder model."
100
- )
101
- return None
102
-
103
- if self.model_config is None or self.model_config.hf_config is None:
104
- logger.warning_once(
105
- "Using None for decoder start token id because "
106
- "model config is not available."
107
- )
108
- return None
109
-
94
dec_start_token_id = getattr(
95
self.model_config.hf_config, "decoder_start_token_id", None
96
)
97
+
98
if dec_start_token_id is None:
99
logger.warning_once(
100
"Falling back on <BOS> for decoder start token "
@@ -118,48 +103,12 @@ class InputPreprocessor:
103
)
104
dec_start_token_id = self.get_bos_token_id()
105
121
- return dec_start_token_id
122
-
123
- def _get_default_enc_dec_decoder_prompt(self) -> list[int]:
124
- """
125
- Specifically for encoder/decoder models:
126
- generate a default decoder prompt for when
127
- the user specifies only the encoder prompt.
128
-
129
- Encoder/decoder models utilize the decoder
130
- prompt in different ways; as new models are
131
- added, it is intended that this function
132
- will be extended to produce differing
133
- default decoder prompts, depending on the
134
- model variety.
135
-
136
- Absent a special case, the default behavior
137
- of this method is to mirror the behavior of
138
- the HuggingFace (HF) GenerationMixin for a None
139
- decoder prompt, which is to employ a logit processor
140
- setting to force the first decoded token to be <BOS>.
141
- Here, this behavior is approximated by having the
142
- "default" decoder prompt be <BOS>.
143
-
144
- However, it is possible that in the future
145
- other models may have different or more
146
- complex logic for the default decoder prompt.
147
- This motivates having a special helper method
148
- for default decoder prompts.
149
-
150
- Returns:
151
-
152
- * prompt_token_ids
153
- """
106
+ if dec_start_token_id is None:
107
+ raise RuntimeError("Cannot find decoder start token id or <BOS>")
108
155
- bos_token_id = self.get_bos_token_id()
156
- assert bos_token_id is not None
157
- return [bos_token_id]
109
+ return dec_start_token_id
110
159
- def _prepare_decoder_input_ids_for_generation(
160
- self,
161
- decoder_input_ids: list[int] | None,
162
- ) -> list[int]:
111
+ def _prepare_decoder_input_ids(self, decoder_input_ids: list[int]) -> list[int]:
112
"""
113
Prepares `decoder_input_ids` for generation with encoder-decoder models.
114
@@ -176,14 +125,7 @@ class InputPreprocessor:
125
126
* Processed token list
127
"""
179
-
128
decoder_start_token_id = self.get_decoder_start_token_id()
181
- assert decoder_start_token_id is not None
182
-
183
- if decoder_input_ids is None:
184
- # no decoder prompt input ->
185
- # use decoder_start_token_id as decoder_input_ids
186
- decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
129
130
if (
131
len(decoder_input_ids) == 0
@@ -428,111 +370,70 @@ class InputPreprocessor:
370
371
assert_never(parsed)
372
431
- def _build_enc_dec_llm_inputs(
373
+ def _validate_enc_inputs(
374
self,
433
- encoder_inputs: SingletonInputs,
434
- decoder_inputs: SingletonInputs | None,
435
- ) -> EncoderDecoderInputs:
436
- if (
437
- encoder_inputs["type"] == "embeds"
438
- or decoder_inputs
439
- and decoder_inputs["type"] == "embeds"
440
- ):
375
+ inputs: SingletonInputs,
376
+ ) -> TokenInputs | MultiModalEncDecInputs:
377
+ if inputs["type"] == "embeds":
378
raise ValueError(
379
"Embedding inputs are not supported for encoder-decoder models"
380
)
381
445
- # Needed for mypy
446
- encoder_inputs = cast(TokenInputs | MultiModalInputs, encoder_inputs)
447
- decoder_inputs = cast(TokenInputs | MultiModalInputs | None, decoder_inputs)
448
-
449
- if decoder_inputs is None:
450
- if self.model_config.hf_config.model_type == "whisper":
451
- # For Whisper models, the text prompt should go to the decoder.
452
- # If no explicit encoder/decoder inputs, then copy the prompt
453
- # from the encoder to the decoder. The encoder tokens are later
454
- # overridden by the audio features.
455
- dec_token_ids = encoder_inputs["prompt_token_ids"].copy()
456
- else:
457
- dec_token_ids = self._prepare_decoder_input_ids_for_generation(None)
458
- decoder_inputs = token_inputs(dec_token_ids)
459
- else:
460
- if "multi_modal_data" in decoder_inputs:
461
- raise ValueError(
462
- "Multi-modal decoder inputs of encoder-"
463
- "decoder models are not supported yet"
464
- )
465
-
466
- dec_token_ids = self._prepare_decoder_input_ids_for_generation(
467
- decoder_inputs["prompt_token_ids"]
382
+ if inputs["type"] == "multimodal" and "encoder_prompt_token_ids" not in inputs:
383
+ raise RuntimeError(
384
+ "You should register an encoder-decoder "
385
+ "multi-modal processor for encoder-decoder models."
386
)
469
- decoder_inputs["prompt_token_ids"] = dec_token_ids
387
471
- return EncoderDecoderInputs(
472
- encoder=encoder_inputs,
473
- decoder=decoder_inputs,
474
- )
388
+ return inputs # type: ignore[return-value]
389
476
- def _split_enc_dec_mm_inputs(
390
+ def _validate_dec_inputs(
391
self,
478
- inputs: SingletonInputs | MultiModalEncDecInputs,
479
- decoder_inputs_to_override: SingletonInputs | None = None,
480
- ) -> tuple[SingletonInputs, SingletonInputs]:
481
- """
482
- For encoder/decoder models only:
483
- Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
484
- """
485
- if (
486
- inputs["type"] == "embeds"
487
- or decoder_inputs_to_override
488
- and decoder_inputs_to_override["type"] == "embeds"
489
- ):
392
+ inputs: SingletonInputs,
393
+ ) -> TokenInputs | MultiModalInputs:
394
+ if inputs["type"] == "embeds":
395
raise ValueError(
396
"Embedding inputs are not supported for encoder-decoder models"
397
)
398
494
- # Needed for mypy
495
- inputs = cast(
496
- TokenInputs | MultiModalInputs | MultiModalEncDecInputs,
497
- inputs,
498
- )
499
- decoder_inputs_to_override = cast(
500
- TokenInputs | MultiModalInputs | None,
501
- decoder_inputs_to_override,
502
- )
399
+ return inputs
400
504
- encoder_inputs: SingletonInputs
505
- decoder_inputs: SingletonInputs
401
+ def _build_enc_dec_inputs(
402
+ self,
403
+ encoder_inputs: SingletonInputs,
404
+ decoder_inputs: SingletonInputs | None = None,
405
+ ) -> EncoderDecoderInputs:
406
+ if decoder_inputs is None:
407
+ decoder_inputs = encoder_inputs
408
507
- if inputs["type"] == "multimodal": # Multimodal data inputs
508
- if "encoder_prompt_token_ids" not in inputs:
509
- raise RuntimeError(
510
- "You should register an encoder-decoder "
511
- "multi-modal processor for encoder-decoder "
512
- "models."
513
- )
514
- inputs = cast(MultiModalEncDecInputs, inputs)
409
+ enc_inputs = self._validate_enc_inputs(encoder_inputs)
410
+ dec_inputs = self._validate_dec_inputs(decoder_inputs)
411
516
- encoder_inputs = token_inputs(inputs["encoder_prompt_token_ids"])
412
+ enc_inputs_new: TokenInputs | MultiModalEncDecInputs
413
+ dec_inputs_new: TokenInputs | MultiModalInputs
414
518
- decoder_prompt_inputs = decoder_inputs_to_override or inputs
519
- decoder_inputs = MultiModalInputs(
415
+ if enc_inputs["type"] == "multimodal":
416
+ enc_inputs_new = token_inputs(enc_inputs["encoder_prompt_token_ids"])
417
+ dec_inputs_new = MultiModalInputs(
418
type="multimodal",
521
- prompt_token_ids=decoder_prompt_inputs["prompt_token_ids"],
522
- mm_kwargs=inputs["mm_kwargs"],
523
- mm_hashes=inputs["mm_hashes"],
524
- mm_placeholders=inputs["mm_placeholders"],
419
+ prompt_token_ids=dec_inputs["prompt_token_ids"],
420
+ mm_kwargs=enc_inputs["mm_kwargs"],
421
+ mm_hashes=enc_inputs["mm_hashes"],
422
+ mm_placeholders=enc_inputs["mm_placeholders"],
423
)
526
- if cache_salt := inputs.get("cache_salt"):
527
- decoder_inputs["cache_salt"] = cache_salt
528
-
529
- elif inputs["type"] == "token": # Text-only inputs
530
- encoder_inputs = token_inputs(prompt_token_ids=[])
531
- decoder_inputs = decoder_inputs_to_override or inputs
424
+ elif enc_inputs["type"] == "token":
425
+ enc_inputs_new = token_inputs(prompt_token_ids=[])
426
+ dec_inputs_new = dec_inputs
427
else:
533
- assert_never(inputs) # type: ignore[arg-type]
428
+ assert_never(enc_inputs)
429
+
430
+ dec_inputs_new["prompt_token_ids"] = self._prepare_decoder_input_ids(
431
+ dec_inputs_new["prompt_token_ids"]
432
+ )
433
+ if cache_salt := enc_inputs.get("cache_salt"):
434
+ dec_inputs_new["cache_salt"] = cache_salt
435
535
- return encoder_inputs, decoder_inputs
436
+ return EncoderDecoderInputs(encoder=enc_inputs_new, decoder=dec_inputs_new)
437
438
def _process_encoder_decoder_prompt(
439
self,
@@ -574,54 +475,23 @@ class InputPreprocessor:
475
* [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
476
instance
477
"""
577
- encoder_inputs: SingletonInputs
578
- decoder_inputs: SingletonInputs | None
579
- if is_explicit_encoder_decoder_prompt(prompt):
580
- # `cast` is needed for mypy, but not pyright
581
- prompt_ = cast(ExplicitEncoderDecoderPrompt, prompt)
582
- encoder_inputs = self._prompt_to_llm_inputs(
583
- prompt_["encoder_prompt"],
478
+ encoder_prompt, decoder_prompt = split_enc_dec_prompt(prompt)
479
+
480
+ return self._build_enc_dec_inputs(
481
+ encoder_inputs=self._prompt_to_llm_inputs(
482
+ encoder_prompt,
483
tokenization_kwargs=tokenization_kwargs,
484
mm_uuids=mm_uuids,
586
- )
587
- if (decoder_input := prompt_["decoder_prompt"]) is None:
588
- decoder_inputs = None
589
- else:
590
- decoder_inputs = self._prompt_to_llm_inputs(
591
- decoder_input, tokenization_kwargs=tokenization_kwargs
485
+ ),
486
+ decoder_inputs=(
487
+ None
488
+ if decoder_prompt is None
489
+ else self._prompt_to_llm_inputs(
490
+ decoder_prompt,
491
+ tokenization_kwargs=tokenization_kwargs,
492
)
593
- # For multimodal model, override decoder prompt from processor
594
- # with explicit decoder prompt.
595
- if self.model_config.is_multimodal_model:
596
- encoder_inputs, decoder_inputs = self._split_enc_dec_mm_inputs(
597
- encoder_inputs, decoder_inputs
598
- )
599
- else:
600
- # `cast` is needed for mypy, but not pyright
601
- inputs = self._prompt_to_llm_inputs(
602
- cast(SingletonPrompt, prompt),
603
- tokenization_kwargs=tokenization_kwargs,
604
- mm_uuids=mm_uuids,
605
- )
606
- if self.model_config.is_multimodal_model:
607
- # Encoder-Decoder Multimodal model
608
- encoder_inputs, decoder_inputs = self._split_enc_dec_mm_inputs(inputs)
609
- else:
610
- encoder_inputs = inputs
611
- decoder_inputs = None
612
-
613
- return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
614
-
615
- def _build_decoder_only_llm_inputs(
616
- self,
617
- prompt_inputs: DecoderOnlyInputs,
618
- ) -> DecoderOnlyInputs:
619
- if "prompt_token_ids" in prompt_inputs:
620
- prompt_inputs = cast(
621
- TokenInputs | MultiModalInputs, prompt_inputs
622
- ) # Needed for mypy
623
-
624
- return prompt_inputs
493
+ ),
494
+ )
495
496
def _process_decoder_only_prompt(
497
self,
@@ -643,15 +513,12 @@ class InputPreprocessor:
513
514
* [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
515
"""
646
-
647
- prompt_comps = self._prompt_to_llm_inputs(
516
+ return self._prompt_to_llm_inputs(
517
prompt,
518
tokenization_kwargs=tokenization_kwargs,
519
mm_uuids=mm_uuids,
520
)
521
653
- return self._build_decoder_only_llm_inputs(prompt_comps)
654
-
522
def _preprocess(
523
self,
524
prompt: PromptType,
@@ -673,10 +540,8 @@ class InputPreprocessor:
540
"Cannot pass encoder-decoder prompt to decoder-only models"
541
)
542
676
- # Decoder-only operation
677
- # `cast` is needed for mypy, but not pyright
543
return self._process_decoder_only_prompt(
679
- cast(SingletonPrompt, prompt),
544
+ prompt,
545
tokenization_kwargs=tokenization_kwargs,
546
mm_uuids=mm_uuids,
547
)
vllm/multimodal/inputs.py
+4
@@ -1083,6 +1083,10 @@ class MultiModalEncDecInputs(MultiModalInputs):
1083
Represents the outputs of
1084
[`EncDecMultiModalProcessor`][vllm.multimodal.processing.EncDecMultiModalProcessor]
1085
ready to be passed to vLLM internals.
1086
+
1087
+ Note: Even text-only encoder-decoder models are currently implemented
1088
+ as multi-modal models for convenience.
1089
+ (Example: https://github.com/neuralmagic/bart-plugin)
1090
"""
1091
1092
encoder_prompt_token_ids: list[int]