A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
[Renderer] Move InputPreprocessor into Renderer (1.5/2) (#34598)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 16, 2026 at 15:46 UTC
ec17bdd8940798a5e74dc83ed489aacfbc32736d
11 files changed
+209
-136
tests/renderers/test_completions.py
+2
-2
@@ -93,14 +93,14 @@ def _build_renderer(
93
94
95
def _preprocess_prompt(
96
- mdoel_config: ModelConfig,
96
+ model_config: ModelConfig,
97
prompt_or_prompts: SingletonPrompt | bytes | Sequence[SingletonPrompt | bytes],
98
):
99
return [
100
(
101
prompt
102
if isinstance(prompt, bytes)
103
- else parse_model_prompt(mdoel_config, prompt)
103
+ else parse_model_prompt(model_config, prompt)
104
)
105
for prompt in prompt_to_seq(prompt_or_prompts)
106
]
vllm/inputs/data.py
+110
-3
@@ -3,7 +3,7 @@
3
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
4
5
import torch
6
-from typing_extensions import NotRequired, TypedDict
6
+from typing_extensions import NotRequired, TypedDict, assert_never
7
8
if TYPE_CHECKING:
9
from vllm.multimodal.inputs import (
@@ -200,15 +200,22 @@ class TokenInputs(_InputOptions):
200
prompt_token_ids: list[int]
201
"""The token IDs of the prompt."""
202
203
+ prompt: NotRequired[str]
204
+ """The prompt text corresponding to the token IDs, if available."""
205
+
206
207
def token_inputs(
208
prompt_token_ids: list[int],
209
+ *,
210
+ prompt: str | None = None,
211
cache_salt: str | None = None,
212
) -> TokenInputs:
213
"""Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional
214
values."""
215
inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
216
217
+ if prompt is not None:
218
+ inputs["prompt"] = prompt
219
if cache_salt is not None:
220
inputs["cache_salt"] = cache_salt
221
@@ -224,15 +231,22 @@ class EmbedsInputs(_InputOptions):
231
prompt_embeds: torch.Tensor
232
"""The embeddings of the prompt."""
233
234
+ prompt: NotRequired[str]
235
+ """The prompt text corresponding to the token IDs, if available."""
236
+
237
238
def embeds_inputs(
239
prompt_embeds: torch.Tensor,
240
+ *,
241
+ prompt: str | None = None,
242
cache_salt: str | None = None,
243
) -> EmbedsInputs:
244
"""Construct [`EmbedsInputs`][vllm.inputs.data.EmbedsInputs] from optional
245
values."""
246
inputs = EmbedsInputs(type="embeds", prompt_embeds=prompt_embeds)
247
248
+ if prompt is not None:
249
+ inputs["prompt"] = prompt
250
if cache_salt is not None:
251
inputs["cache_salt"] = cache_salt
252
@@ -278,10 +292,12 @@ class EncoderDecoderInputs(TypedDict):
292
for encoder-decoder models.
293
"""
294
281
- encoder: EncoderInputs
295
+ type: Literal["enc_dec"]
296
+
297
+ encoder_prompt: EncoderInputs
298
"""The inputs for the encoder portion."""
299
284
- decoder: DecoderInputs
300
+ decoder_prompt: DecoderInputs
301
"""The inputs for the decoder portion."""
302
303
@@ -296,3 +312,94 @@ which can be passed to
312
313
SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs
314
"""The inputs for a single encoder/decoder prompt."""
315
+
316
+
317
+def _validate_enc_inputs(inputs: SingletonInputs) -> EncoderInputs:
318
+ if inputs["type"] == "embeds":
319
+ raise ValueError(
320
+ "Embedding inputs are not supported for encoder-decoder models"
321
+ )
322
+
323
+ if inputs["type"] == "multimodal" and "encoder_prompt_token_ids" not in inputs:
324
+ raise RuntimeError(
325
+ "You should register an encoder-decoder multi-modal processor "
326
+ "for encoder-decoder models."
327
+ )
328
+
329
+ return inputs # type: ignore[return-value]
330
+
331
+
332
+def _validate_dec_inputs(inputs: SingletonInputs) -> DecoderInputs:
333
+ if inputs["type"] == "embeds":
334
+ raise ValueError(
335
+ "Embedding inputs are not supported for encoder-decoder models"
336
+ )
337
+
338
+ return inputs
339
+
340
+
341
+def _prepare_decoder_input_ids_for_generation(
342
+ decoder_input_ids: list[int],
343
+ decoder_start_token_id: int,
344
+) -> list[int]:
345
+ """
346
+ Prepare `decoder_input_ids` for generation with encoder-decoder models,
347
+ according to `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
348
+
349
+ Source:
350
+ https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/generation/utils.py
351
+ """
352
+ if len(decoder_input_ids) == 0 or decoder_input_ids[0] != decoder_start_token_id:
353
+ decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
354
+
355
+ return decoder_input_ids
356
+
357
+
358
+def build_enc_dec_inputs(
359
+ encoder_inputs: SingletonInputs,
360
+ decoder_inputs: SingletonInputs | None,
361
+ decoder_start_token_id: int,
362
+) -> EncoderDecoderInputs:
363
+ enc_inputs = _validate_enc_inputs(encoder_inputs)
364
+
365
+ if decoder_inputs is None:
366
+ dec_inputs: DecoderInputs = enc_inputs
367
+ else:
368
+ dec_inputs = _validate_dec_inputs(decoder_inputs)
369
+
370
+ enc_inputs_new: EncoderInputs
371
+ dec_inputs_new: DecoderInputs
372
+
373
+ if enc_inputs["type"] == "multimodal":
374
+ from vllm.multimodal.inputs import mm_inputs
375
+
376
+ enc_inputs_new = token_inputs(
377
+ enc_inputs["encoder_prompt_token_ids"],
378
+ prompt=enc_inputs.get("encoder_prompt"),
379
+ )
380
+ dec_inputs_new = mm_inputs(
381
+ prompt_token_ids=dec_inputs["prompt_token_ids"],
382
+ prompt=dec_inputs.get("prompt"),
383
+ mm_kwargs=enc_inputs["mm_kwargs"],
384
+ mm_hashes=enc_inputs["mm_hashes"],
385
+ mm_placeholders=enc_inputs["mm_placeholders"],
386
+ )
387
+ elif enc_inputs["type"] == "token":
388
+ enc_inputs_new = token_inputs(prompt_token_ids=[])
389
+ dec_inputs_new = dec_inputs
390
+ else:
391
+ assert_never(enc_inputs)
392
+
393
+ dec_inputs_new["prompt_token_ids"] = _prepare_decoder_input_ids_for_generation(
394
+ dec_inputs_new["prompt_token_ids"],
395
+ decoder_start_token_id,
396
+ )
397
+
398
+ if cache_salt := enc_inputs.get("cache_salt"):
399
+ dec_inputs_new["cache_salt"] = cache_salt
400
+
401
+ return EncoderDecoderInputs(
402
+ type="enc_dec",
403
+ encoder_prompt=enc_inputs_new,
404
+ decoder_prompt=dec_inputs_new,
405
+ )
vllm/inputs/parse.py
+2
-6
@@ -7,11 +7,7 @@ from .data import ProcessorInputs, SingletonInputs
7
def split_enc_dec_inputs(
8
inputs: ProcessorInputs,
9
) -> tuple[SingletonInputs | None, SingletonInputs]:
10
- if "encoder" in inputs and "decoder" in inputs:
11
- # NOTE: This passes pyright but not mypy
12
- return (
13
- inputs["encoder"], # type: ignore[typeddict-item]
14
- inputs["decoder"], # type: ignore[typeddict-item]
15
- )
10
+ if inputs["type"] == "enc_dec":
11
+ return inputs["encoder_prompt"], inputs["decoder_prompt"]
12
13
return None, inputs
vllm/inputs/preprocess.py
+3
-109
@@ -7,6 +7,7 @@ from typing import Any, overload
7
from typing_extensions import assert_never
8
9
from vllm.config import VllmConfig
10
+from vllm.inputs.data import build_enc_dec_inputs
11
from vllm.logger import init_logger
12
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
13
from vllm.multimodal.inputs import (
@@ -67,54 +68,6 @@ class InputPreprocessor:
68
def get_tokenizer(self) -> TokenizerLike:
69
return self.renderer.get_tokenizer()
70
70
- def get_decoder_start_token_id(self) -> int:
71
- """
72
- Obtain the decoder start token id employed by an encoder/decoder
73
- model. Raises an error if it is not available.
74
- """
75
- dec_start_token_id = getattr(
76
- self.model_config.hf_config, "decoder_start_token_id", None
77
- )
78
-
79
- if dec_start_token_id is None:
80
- logger.warning_once(
81
- "Falling back on <BOS> for decoder start token id "
82
- "because decoder start token id is not available."
83
- )
84
- dec_start_token_id = self.renderer.get_bos_token_id()
85
-
86
- if dec_start_token_id is None:
87
- raise RuntimeError("Cannot find decoder start token id or <BOS>")
88
-
89
- return dec_start_token_id
90
-
91
- def _prepare_decoder_input_ids(self, decoder_input_ids: list[int]) -> list[int]:
92
- """
93
- Prepares `decoder_input_ids` for generation with encoder-decoder models.
94
-
95
- Based on:
96
- https://github.com/huggingface/transformers/blob/4037a2b5b1278736e566aec12e169100275545ea/src/transformers/generation/utils.py
97
- specifically,
98
- `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
99
-
100
- Arguments:
101
-
102
- * decoder_input_ids: input token ids to preprocess
103
-
104
- Returns:
105
-
106
- * Processed token list
107
- """
108
- decoder_start_token_id = self.get_decoder_start_token_id()
109
-
110
- if (
111
- len(decoder_input_ids) == 0
112
- or decoder_input_ids[0] != decoder_start_token_id
113
- ):
114
- decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
115
-
116
- return decoder_input_ids
117
-
71
def _tokenize_prompt(
72
self,
73
prompt: str,
@@ -332,66 +285,6 @@ class InputPreprocessor:
285
286
assert_never(prompt) # type: ignore[arg-type]
287
335
- def _validate_enc_inputs(self, inputs: SingletonInputs) -> EncoderInputs:
336
- if inputs["type"] == "embeds":
337
- raise ValueError(
338
- "Embedding inputs are not supported for encoder-decoder models"
339
- )
340
-
341
- if inputs["type"] == "multimodal" and "encoder_prompt_token_ids" not in inputs:
342
- raise RuntimeError(
343
- "You should register an encoder-decoder "
344
- "multi-modal processor for encoder-decoder models."
345
- )
346
-
347
- return inputs # type: ignore[return-value]
348
-
349
- def _validate_dec_inputs(self, inputs: SingletonInputs) -> DecoderInputs:
350
- if inputs["type"] == "embeds":
351
- raise ValueError(
352
- "Embedding inputs are not supported for encoder-decoder models"
353
- )
354
-
355
- return inputs
356
-
357
- def _build_enc_dec_inputs(
358
- self,
359
- encoder_inputs: SingletonInputs,
360
- decoder_inputs: SingletonInputs | None = None,
361
- ) -> EncoderDecoderInputs:
362
- enc_inputs = self._validate_enc_inputs(encoder_inputs)
363
-
364
- if decoder_inputs is None:
365
- dec_inputs: DecoderInputs = enc_inputs # type: ignore[assignment]
366
- else:
367
- dec_inputs = self._validate_dec_inputs(decoder_inputs)
368
-
369
- enc_inputs_new: EncoderInputs
370
- dec_inputs_new: DecoderInputs
371
-
372
- if enc_inputs["type"] == "multimodal":
373
- enc_inputs_new = token_inputs(enc_inputs["encoder_prompt_token_ids"])
374
- dec_inputs_new = MultiModalInputs(
375
- type="multimodal",
376
- prompt_token_ids=dec_inputs["prompt_token_ids"],
377
- mm_kwargs=enc_inputs["mm_kwargs"],
378
- mm_hashes=enc_inputs["mm_hashes"],
379
- mm_placeholders=enc_inputs["mm_placeholders"],
380
- )
381
- elif enc_inputs["type"] == "token":
382
- enc_inputs_new = token_inputs(prompt_token_ids=[])
383
- dec_inputs_new = dec_inputs
384
- else:
385
- assert_never(enc_inputs)
386
-
387
- dec_inputs_new["prompt_token_ids"] = self._prepare_decoder_input_ids(
388
- dec_inputs_new["prompt_token_ids"]
389
- )
390
- if cache_salt := enc_inputs.get("cache_salt"):
391
- dec_inputs_new["cache_salt"] = cache_salt
392
-
393
- return EncoderDecoderInputs(encoder=enc_inputs_new, decoder=dec_inputs_new)
394
-
288
def _process_encoder_decoder_prompt(
289
self,
290
prompt: EncoderDecoderDictPrompt,
@@ -417,7 +310,7 @@ class InputPreprocessor:
310
encoder_prompt = prompt["encoder_prompt"]
311
decoder_prompt = prompt["decoder_prompt"]
312
420
- return self._build_enc_dec_inputs(
313
+ return build_enc_dec_inputs(
314
encoder_inputs=self._prompt_to_llm_inputs(
315
encoder_prompt,
316
tokenization_kwargs=tokenization_kwargs,
@@ -431,6 +324,7 @@ class InputPreprocessor:
324
tokenization_kwargs=tokenization_kwargs,
325
)
326
),
327
+ decoder_start_token_id=self.renderer.get_dec_start_token_id(),
328
)
329
330
def _process_decoder_only_prompt(
vllm/model_executor/models/llava.py
+2
-2
@@ -31,6 +31,7 @@ from vllm.multimodal.inputs import (
31
MultiModalInputs,
32
MultiModalKwargsItems,
33
MultiModalUUIDDict,
34
+ mm_inputs,
35
)
36
from vllm.multimodal.parse import (
37
ImageEmbeddingItems,
@@ -837,8 +838,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
838
for modality, placeholders in mm_placeholders.items()
839
}
840
840
- return MultiModalInputs(
841
- type="multimodal",
841
+ return mm_inputs(
842
prompt_token_ids=prompt_ids,
843
mm_kwargs=mm_kwargs,
844
mm_hashes=mm_hashes,
vllm/model_executor/models/terratorch.py
+2
-2
@@ -48,6 +48,7 @@ from vllm.multimodal.inputs import (
48
MultiModalKwargsItems,
49
MultiModalUUIDDict,
50
PlaceholderRange,
51
+ mm_inputs,
52
)
53
from vllm.multimodal.parse import (
54
DictEmbeddingItems,
@@ -222,8 +223,7 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
223
),
224
)
225
225
- return MultiModalInputs(
226
- type="multimodal",
226
+ return mm_inputs(
227
prompt_token_ids=[1],
228
mm_kwargs=mm_kwargs,
229
mm_hashes=mm_hashes,
vllm/model_executor/models/transformers/multimodal.py
+2
-2
@@ -33,6 +33,7 @@ from vllm.multimodal.inputs import (
33
MultiModalInputs,
34
MultiModalUUIDDict,
35
PlaceholderRange,
36
+ mm_inputs,
37
)
38
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
39
from vllm.multimodal.processing import (
@@ -260,8 +261,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
261
mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
262
)
263
263
- return MultiModalInputs(
264
- type="multimodal",
264
+ return mm_inputs(
265
prompt_token_ids=prompt_ids,
266
mm_kwargs=mm_kwargs,
267
mm_hashes=mm_hashes,
vllm/multimodal/inputs.py
+57
-1
@@ -20,7 +20,7 @@ from typing import (
20
21
import numpy as np
22
from PIL.Image import Image
23
-from typing_extensions import TypeVar
23
+from typing_extensions import NotRequired, TypeVar
24
25
from vllm.utils.collection_utils import is_list_of
26
from vllm.utils.import_utils import LazyLoader
@@ -1075,6 +1075,9 @@ class MultiModalInputs(_InputOptions):
1075
prompt_token_ids: list[int]
1076
"""The processed token IDs which includes placeholder tokens."""
1077
1078
+ prompt: NotRequired[str]
1079
+ """The prompt text corresponding to the token IDs, if available."""
1080
+
1081
mm_kwargs: MultiModalKwargsOptionalItems
1082
"""Keyword arguments to be directly passed to the model after batching."""
1083
@@ -1088,6 +1091,31 @@ class MultiModalInputs(_InputOptions):
1091
"""
1092
1093
1094
+def mm_inputs(
1095
+ prompt_token_ids: list[int],
1096
+ mm_kwargs: MultiModalKwargsOptionalItems,
1097
+ mm_hashes: MultiModalHashes,
1098
+ mm_placeholders: MultiModalPlaceholderDict,
1099
+ *,
1100
+ prompt: str | None = None,
1101
+ cache_salt: str | None = None,
1102
+) -> MultiModalInputs:
1103
+ inputs = MultiModalInputs(
1104
+ type="multimodal",
1105
+ prompt_token_ids=prompt_token_ids,
1106
+ mm_kwargs=mm_kwargs,
1107
+ mm_hashes=mm_hashes,
1108
+ mm_placeholders=mm_placeholders,
1109
+ )
1110
+
1111
+ if prompt is not None:
1112
+ inputs["prompt"] = prompt
1113
+ if cache_salt is not None:
1114
+ inputs["cache_salt"] = cache_salt
1115
+
1116
+ return inputs
1117
+
1118
+
1119
class MultiModalEncDecInputs(MultiModalInputs):
1120
"""
1121
Represents the outputs of
@@ -1101,3 +1129,31 @@ class MultiModalEncDecInputs(MultiModalInputs):
1129
1130
encoder_prompt_token_ids: list[int]
1131
"""The processed token IDs of the encoder prompt."""
1132
+
1133
+ encoder_prompt: NotRequired[str]
1134
+ """The prompt text corresponding to the encoder token IDs, if available."""
1135
+
1136
+
1137
+def mm_enc_dec_inputs(
1138
+ encoder_inputs: MultiModalInputs,
1139
+ decoder_prompt_token_ids: list[int],
1140
+ *,
1141
+ decoder_prompt: str | None = None,
1142
+) -> MultiModalEncDecInputs:
1143
+ inputs = MultiModalEncDecInputs(
1144
+ type="multimodal",
1145
+ prompt_token_ids=decoder_prompt_token_ids,
1146
+ encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1147
+ mm_kwargs=encoder_inputs["mm_kwargs"],
1148
+ mm_hashes=encoder_inputs["mm_hashes"],
1149
+ mm_placeholders=encoder_inputs["mm_placeholders"],
1150
+ )
1151
+
1152
+ if decoder_prompt is not None:
1153
+ inputs["prompt"] = decoder_prompt
1154
+ if "prompt" in encoder_inputs:
1155
+ inputs["encoder_prompt"] = encoder_inputs["prompt"]
1156
+ if "cache_salt" in encoder_inputs:
1157
+ inputs["cache_salt"] = encoder_inputs["cache_salt"]
1158
+
1159
+ return inputs
vllm/multimodal/media/base.py
+2
-2
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from abc import ABC, abstractmethod
5
-from dataclasses import dataclass
5
+from dataclasses import dataclass, field
6
from pathlib import Path
7
from typing import Any, Generic, TypeVar
8
@@ -26,7 +26,7 @@ class MediaWithBytes(Generic[_T]):
26
"""
27
28
media: _T
29
- original_bytes: bytes
29
+ original_bytes: bytes = field(repr=False)
30
31
def __array__(self, *args, **kwargs) -> np.ndarray:
32
"""Allow np.array(obj) to return np.array(obj.media)."""
vllm/multimodal/processing/processor.py
+6
-7
@@ -34,6 +34,8 @@ from ..inputs import (
34
MultiModalKwargsOptionalItems,
35
MultiModalUUIDDict,
36
PlaceholderRange,
37
+ mm_enc_dec_inputs,
38
+ mm_inputs,
39
)
40
from ..parse import (
41
DictEmbeddingItems,
@@ -1803,8 +1805,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1805
for modality, placeholders in mm_placeholders.items()
1806
}
1807
1806
- return MultiModalInputs(
1807
- type="multimodal",
1808
+ return mm_inputs(
1809
prompt_token_ids=prompt_ids,
1810
mm_kwargs=mm_info.kwargs,
1811
mm_hashes=mm_info.hashes,
@@ -1848,12 +1849,10 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1849
else:
1850
decoder_prompt_ids = decoder_prompt_raw
1851
1851
- mm_inputs = MultiModalEncDecInputs(
1852
- encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1853
- **encoder_inputs,
1852
+ return mm_enc_dec_inputs(
1853
+ encoder_inputs,
1854
+ decoder_prompt_ids,
1855
)
1855
- mm_inputs["prompt_token_ids"] = decoder_prompt_ids
1856
- return mm_inputs
1856
1857
def apply(
1858
self,
vllm/renderers/base.py
+21
@@ -153,6 +153,27 @@ class BaseRenderer(ABC, Generic[_T]):
153
154
return self.tokenizer.eos_token_id
155
156
+ def get_dec_start_token_id(self) -> int:
157
+ """
158
+ Obtain the decoder start token id employed by an encoder/decoder model,
159
+ raising an error if it is not available.
160
+ """
161
+ dec_start_token_id = getattr(
162
+ self.model_config.hf_config, "decoder_start_token_id", None
163
+ )
164
+
165
+ if dec_start_token_id is None:
166
+ logger.warning_once(
167
+ "Falling back on <BOS> for decoder start token id "
168
+ "because decoder start token id is not available."
169
+ )
170
+ dec_start_token_id = self.get_bos_token_id()
171
+
172
+ if dec_start_token_id is None:
173
+ raise RuntimeError("Cannot find decoder start token id or <BOS>")
174
+
175
+ return dec_start_token_id
176
+
177
@cached_property
178
def default_cmpl_tok_params(self) -> TokenizeParams:
179
mm_processor = self.mm_processor