[Bugfix][Frontend] Bound the prompt after multimodal expansion (#57076)
Signed-off-by: Dilber P Shakir <dilbersha@gmail.com>
Dilber committed
Sep 20, 2026 at 14:22 UTC
27b7757f6364be5b0b714793ccdc3e3931443bca
3 files changed
+285
-35
tests/renderers/test_truncate_after_mm_expansion.py
new
+166
@@ -0,0 +1,166 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Truncation must bound the prompt *after* multimodal placeholders expand.
4
+
5
+``TokenizeParams.apply_post_tokenization`` truncates the unexpanded prompt, so
6
+one placeholder token can expand into many and push the result back over
7
+``truncate_prompt_tokens`` (vllm-project/vllm#57038).
8
+"""
9
+
10
+import numpy as np
11
+import pytest
12
+import torch
13
+
14
+from vllm.config import ModelConfig, VllmConfig
15
+from vllm.exceptions import VLLMValidationError
16
+from vllm.multimodal.inputs import PlaceholderRange
17
+from vllm.renderers import renderer_from_config
18
+from vllm.renderers.base import BaseRenderer
19
+from vllm.renderers.hf import _ensure_prompt_embeds_placeholder_token
20
+from vllm.renderers.params import TokenizeParams
21
+
22
+
23
+def _renderer():
24
+ """A BaseRenderer stub: only the tokenizer attribute is needed here."""
25
+
26
+ class _StubRenderer(BaseRenderer):
27
+ def __init__(self):
28
+ self.tokenizer = None
29
+
30
+ def render_messages(self, *args, **kwargs):
31
+ raise NotImplementedError
32
+
33
+ return _StubRenderer()
34
+
35
+
36
+def _mm_input(num_tokens: int, offset: int, length: int, **extra):
37
+ return {
38
+ "type": "multimodal",
39
+ "prompt_token_ids": list(range(num_tokens)),
40
+ "mm_kwargs": {},
41
+ "mm_hashes": {},
42
+ "mm_placeholders": {"audio": [PlaceholderRange(offset=offset, length=length)]},
43
+ **extra,
44
+ }
45
+
46
+
47
+def _params(limit: int | None, side: str = "right"):
48
+ return TokenizeParams(
49
+ max_total_tokens=4096,
50
+ truncate_prompt_tokens=limit,
51
+ truncation_side=side,
52
+ )
53
+
54
+
55
+def test_expanded_prompt_is_bounded():
56
+ """The reported case: audio fits, trailing text is cut to the limit."""
57
+ engine_input = _mm_input(num_tokens=147, offset=1, length=52)
58
+
59
+ _renderer()._truncate_expanded_prompt(engine_input, _params(96))
60
+
61
+ assert engine_input["prompt_token_ids"] == list(range(96))
62
+ assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
63
+ offset=1, length=52
64
+ )
65
+
66
+
67
+def test_prompt_within_limit_is_untouched():
68
+ engine_input = _mm_input(num_tokens=64, offset=1, length=52)
69
+
70
+ _renderer()._truncate_expanded_prompt(engine_input, _params(96))
71
+
72
+ assert engine_input["prompt_token_ids"] == list(range(64))
73
+
74
+
75
+def test_split_multimodal_span_is_rejected():
76
+ """Cutting mid-span would desync features from token ids, so reject."""
77
+ engine_input = _mm_input(num_tokens=147, offset=80, length=50)
78
+
79
+ with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
80
+ _renderer()._truncate_expanded_prompt(engine_input, _params(96))
81
+
82
+
83
+def test_dropped_multimodal_span_is_rejected():
84
+ """A span entirely past the cut would be discarded silently."""
85
+ engine_input = _mm_input(num_tokens=147, offset=100, length=20)
86
+
87
+ with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
88
+ _renderer()._truncate_expanded_prompt(engine_input, _params(96))
89
+
90
+
91
+def test_left_truncation_shifts_placeholder_offsets():
92
+ engine_input = _mm_input(num_tokens=147, offset=100, length=20)
93
+
94
+ _renderer()._truncate_expanded_prompt(engine_input, _params(96, side="left"))
95
+
96
+ assert engine_input["prompt_token_ids"] == list(range(51, 147))
97
+ # The span started at 100 and 51 leading tokens were dropped.
98
+ assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
99
+ offset=49, length=20
100
+ )
101
+
102
+
103
+def test_text_only_input_is_untouched():
104
+ """Only multimodal prompts grow after tokenization, so nothing else runs."""
105
+ engine_input = {
106
+ "type": "token",
107
+ "prompt_token_ids": list(range(147)),
108
+ "prompt_token_offsets": [(i, i + 1) for i in range(147)],
109
+ }
110
+
111
+ _renderer()._truncate_expanded_prompt(engine_input, _params(96))
112
+
113
+ assert engine_input["prompt_token_ids"] == list(range(147))
114
+ assert len(engine_input["prompt_token_offsets"]) == 147
115
+
116
+
117
+def test_no_truncation_requested_is_noop():
118
+ engine_input = _mm_input(num_tokens=147, offset=1, length=52)
119
+
120
+ _renderer()._truncate_expanded_prompt(engine_input, _params(None))
121
+
122
+ assert engine_input["prompt_token_ids"] == list(range(147))
123
+
124
+
125
+def test_mixed_prompt_embeds_truncation_covers_embeds_spans():
126
+ """Bound in _process_singleton, where prompt_embeds spans are already attached."""
127
+ model_config = ModelConfig(
128
+ "Qwen/Qwen3-ASR-0.6B-hf",
129
+ revision="7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c",
130
+ dtype="float32",
131
+ max_model_len=4096,
132
+ mm_processor_cache_gb=0,
133
+ enable_prompt_embeds=True,
134
+ )
135
+ renderer = renderer_from_config(VllmConfig(model_config=model_config))
136
+ tok = renderer.tokenizer
137
+ pe_id = _ensure_prompt_embeds_placeholder_token(tok)
138
+ hidden = model_config.get_hidden_size()
139
+ audio = np.zeros(4 * 16000, dtype=np.float32)
140
+
141
+ pe_len = 20
142
+ tensors = [torch.zeros(pe_len, hidden, dtype=torch.float32)]
143
+
144
+ def _build_prompt():
145
+ text = "<|audio_start|><|audio_pad|><|audio_end|>"
146
+ ids = tok.encode(text) + [pe_id] + tok.encode(" tail" * 40)
147
+ return {
148
+ "prompt_token_ids": ids,
149
+ "multi_modal_data": {"audio": (audio, 16000)},
150
+ "_prompt_embeds": (tensors, pe_id),
151
+ }
152
+
153
+ # A limit past every span (audio ends at 52, prompt_embeds at 74):
154
+ # truncation should succeed and bound the prompt.
155
+ out = renderer._process_singleton(
156
+ _build_prompt(),
157
+ tok_params=_params(79),
158
+ )
159
+ assert len(out["prompt_token_ids"]) == 79
160
+
161
+ # A limit inside the prompt_embeds span (54..74): must be rejected.
162
+ with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
163
+ renderer._process_singleton(
164
+ _build_prompt(),
165
+ tok_params=_params(64),
166
+ )
vllm/renderers/base.py
+94
-8
@@ -7,11 +7,13 @@ from abc import ABC, abstractmethod
7
from collections.abc import Mapping, Sequence
8
from concurrent.futures import Future, ThreadPoolExecutor
9
from contextlib import ExitStack
10
+from dataclasses import replace
11
from functools import cached_property
12
from typing import TYPE_CHECKING, Any, Generic, overload
13
14
from typing_extensions import TypeVar
15
16
+from vllm.exceptions import VLLMValidationError
17
from vllm.inputs import (
18
EmbedsInput,
19
EmbedsPrompt,
@@ -962,27 +964,103 @@ class BaseRenderer(ABC, Generic[_T]):
964
965
return engine_input
966
967
+ def _truncate_expanded_prompt(
968
+ self,
969
+ engine_input: TokensInput | MultiModalInput,
970
+ params: TokenizeParams | None,
971
+ ) -> None:
972
+ """Re-apply `truncate_prompt_tokens` once placeholders are expanded.
973
+
974
+ `TokenizeParams.apply_post_tokenization` truncates the prompt before
975
+ multimodal expansion, so a single placeholder token can expand into
976
+ many and push the result back over the requested bound.
977
+
978
+ Multimodal spans are never cut: a request whose spans would not all
979
+ survive intact is rejected, matching the post-expansion rejection in
980
+ `vllm.v1.engine.input_processor._validate_model_input`.
981
+
982
+ Only multimodal inputs can grow after `apply_post_tokenization`, so
983
+ everything else returns immediately.
984
+ """
985
+ if engine_input["type"] != "multimodal":
986
+ return
987
+
988
+ if params is None or params.truncate_prompt_tokens is None:
989
+ return
990
+
991
+ prompt_token_ids = engine_input.get("prompt_token_ids")
992
+ if prompt_token_ids is None:
993
+ return
994
+
995
+ num_tokens = len(prompt_token_ids)
996
+ truncation = params._truncation_slice(self.tokenizer, num_tokens)
997
+ if truncation is None:
998
+ return
999
+
1000
+ start, stop, _ = truncation.indices(num_tokens)
1001
+
1002
+ mm_placeholders = engine_input["mm_placeholders"]
1003
+ for modality, positions in mm_placeholders.items():
1004
+ for position in positions:
1005
+ end = position.offset + position.length
1006
+ if position.offset >= start and end <= stop:
1007
+ continue
1008
+ raise VLLMValidationError(
1009
+ f"truncate_prompt_tokens={params.truncate_prompt_tokens} "
1010
+ f"would split or drop a(n) {modality} item, which spans "
1011
+ f"tokens {position.offset} to {end} of the "
1012
+ f"{num_tokens}-token prompt. Multimodal placeholders "
1013
+ f"expand after truncation is applied, so the limit must "
1014
+ f"leave every multimodal item intact. Please raise "
1015
+ f"truncate_prompt_tokens or shorten the multimodal input.",
1016
+ parameter="truncate_prompt_tokens",
1017
+ value=params.truncate_prompt_tokens,
1018
+ )
1019
+
1020
+ if start:
1021
+ engine_input["mm_placeholders"] = {
1022
+ modality: [
1023
+ replace(position, offset=position.offset - start)
1024
+ for position in positions
1025
+ ]
1026
+ for modality, positions in mm_placeholders.items()
1027
+ }
1028
+
1029
+ engine_input["prompt_token_ids"] = prompt_token_ids[truncation]
1030
+
1031
def _process_singleton(
1032
self,
1033
prompt: SingletonTokPrompt,
1034
*,
1035
skip_mm_cache: bool = False,
1036
+ tok_params: TokenizeParams | None = None,
1037
) -> SingletonInput:
1038
if "prompt_embeds" in prompt:
1039
return self._process_embeds(prompt) # type: ignore[arg-type]
1040
974
- return self._process_tokens(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
1041
+ engine_input = self._process_tokens( # type: ignore[arg-type]
1042
+ prompt, # type: ignore[arg-type]
1043
+ skip_mm_cache=skip_mm_cache,
1044
+ )
1045
+ self._truncate_expanded_prompt(engine_input, tok_params)
1046
+ return engine_input
1047
1048
async def _process_singleton_async(
1049
self,
1050
prompt: SingletonTokPrompt,
1051
*,
1052
skip_mm_cache: bool = False,
1053
+ tok_params: TokenizeParams | None = None,
1054
) -> SingletonInput:
1055
if "prompt_embeds" in prompt:
1056
return self._process_embeds(prompt) # type: ignore[arg-type]
1057
985
- return await self._process_tokens_async(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
1058
+ engine_input = await self._process_tokens_async( # type: ignore[arg-type]
1059
+ prompt, # type: ignore[arg-type]
1060
+ skip_mm_cache=skip_mm_cache,
1061
+ )
1062
+ self._truncate_expanded_prompt(engine_input, tok_params)
1063
+ return engine_input
1064
1065
def _get_skip_decoder_start_token(self) -> bool:
1066
"""Whether the multimodal processor supplies a complete decoder prefix."""
@@ -1048,12 +1126,15 @@ class BaseRenderer(ABC, Generic[_T]):
1126
arrival_time: float,
1127
*,
1128
skip_mm_cache: bool = False,
1129
+ tok_params: TokenizeParams | None = None,
1130
) -> EngineInput:
1131
engine_input: EngineInput
1132
if "encoder_prompt" in prompt:
1133
engine_input = self._process_enc_dec(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
1134
else:
1056
- engine_input = self._process_singleton(prompt, skip_mm_cache=skip_mm_cache)
1135
+ engine_input = self._process_singleton(
1136
+ prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params
1137
+ )
1138
1139
engine_input["arrival_time"] = arrival_time
1140
@@ -1065,6 +1146,7 @@ class BaseRenderer(ABC, Generic[_T]):
1146
arrival_time: float,
1147
*,
1148
skip_mm_cache: bool = False,
1149
+ tok_params: TokenizeParams | None = None,
1150
) -> EngineInput:
1151
engine_input: EngineInput
1152
if "encoder_prompt" in prompt:
@@ -1074,7 +1156,7 @@ class BaseRenderer(ABC, Generic[_T]):
1156
)
1157
else:
1158
engine_input = await self._process_singleton_async(
1077
- prompt, skip_mm_cache=skip_mm_cache
1159
+ prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params
1160
)
1161
1162
engine_input["arrival_time"] = arrival_time
@@ -1101,7 +1183,9 @@ class BaseRenderer(ABC, Generic[_T]):
1183
self._apply_prompt_extras(tok_prompts, prompt_extras)
1184
1185
return [
1104
- self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)
1186
+ self.process_for_engine(
1187
+ prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
1188
+ )
1189
for prompt in tok_prompts
1190
]
1191
@@ -1126,7 +1210,7 @@ class BaseRenderer(ABC, Generic[_T]):
1210
return await asyncio.gather(
1211
*(
1212
self.process_for_engine_async(
1129
- p, arrival_time, skip_mm_cache=skip_mm_cache
1213
+ p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
1214
)
1215
for p in tok_prompts
1216
)
@@ -1164,7 +1248,9 @@ class BaseRenderer(ABC, Generic[_T]):
1248
self._apply_prompt_extras(tok_prompts, prompt_extras)
1249
1250
eng_prompts = [
1167
- self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)
1251
+ self.process_for_engine(
1252
+ prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
1253
+ )
1254
for prompt in tok_prompts
1255
]
1256
@@ -1204,7 +1290,7 @@ class BaseRenderer(ABC, Generic[_T]):
1290
eng_prompts = await asyncio.gather(
1291
*(
1292
self.process_for_engine_async(
1207
- p, arrival_time, skip_mm_cache=skip_mm_cache
1293
+ p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
1294
)
1295
for p in tok_prompts
1296
)
vllm/renderers/hf.py
+25
-27
@@ -1194,22 +1194,20 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
1194
coordinate space, no offset shifting needed afterwards.
1195
"""
1196
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
1197
- if prompt_embeds_info is not None:
1198
- tensors, placeholder_token_id = prompt_embeds_info
1199
- mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
1200
- cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
1201
- list(prompt["prompt_token_ids"]), mm_updates
1202
- )
1197
+ if prompt_embeds_info is None:
1198
+ return super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
1199
1200
+ tensors, placeholder_token_id = prompt_embeds_info
1201
+ mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
1202
+ cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
1203
+ list(prompt["prompt_token_ids"]), mm_updates
1204
+ )
1205
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
1205
-
1206
- if prompt_embeds_info is not None:
1207
- tensors, _ = prompt_embeds_info
1208
- self._apply_prompt_embeds_to_engine_input(
1209
- cast(MultiModalInput, engine_input),
1210
- tensors,
1211
- mm_updates,
1212
- )
1206
+ self._apply_prompt_embeds_to_engine_input(
1207
+ cast(MultiModalInput, engine_input),
1208
+ tensors,
1209
+ mm_updates,
1210
+ )
1211
1212
return engine_input
1213
@@ -1222,24 +1220,24 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
1220
) -> TokensInput | MultiModalInput:
1221
"""Async equivalent of `_process_tokens`."""
1222
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
1225
- if prompt_embeds_info is not None:
1226
- tensors, placeholder_token_id = prompt_embeds_info
1227
- mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
1228
- cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
1229
- list(prompt["prompt_token_ids"]), mm_updates
1223
+ if prompt_embeds_info is None:
1224
+ return await super()._process_tokens_async(
1225
+ prompt, skip_mm_cache=skip_mm_cache
1226
)
1227
1228
+ tensors, placeholder_token_id = prompt_embeds_info
1229
+ mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
1230
+ cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
1231
+ list(prompt["prompt_token_ids"]), mm_updates
1232
+ )
1233
engine_input = await super()._process_tokens_async(
1234
prompt, skip_mm_cache=skip_mm_cache
1235
)
1235
-
1236
- if prompt_embeds_info is not None:
1237
- tensors, _ = prompt_embeds_info
1238
- self._apply_prompt_embeds_to_engine_input(
1239
- cast(MultiModalInput, engine_input),
1240
- tensors,
1241
- mm_updates,
1242
- )
1236
+ self._apply_prompt_embeds_to_engine_input(
1237
+ cast(MultiModalInput, engine_input),
1238
+ tensors,
1239
+ mm_updates,
1240
+ )
1241
1242
return engine_input
1243