A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Renderer] Clean up renderer code (#26216)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 5, 2025 at 01:05 UTC
119f00630b8b6aea29b5c3141728f96963521df1
5 files changed
+94
-134
tests/entrypoints/openai/test_token_in_token_out.py
+1
-1
@@ -54,7 +54,7 @@ async def test_token_in_token_out_and_logprobs(server):
54
prompt=token_ids,
55
max_tokens=20,
56
temperature=0,
57
- echo=True,
57
+ echo=False,
58
extra_body={
59
"return_token_ids": True,
60
},
tests/test_inputs.py
+13
-13
@@ -4,7 +4,7 @@
4
import pytest
5
6
from vllm.inputs import zip_enc_dec_prompts
7
-from vllm.inputs.parse import parse_and_batch_prompt
7
+from vllm.inputs.parse import parse_raw_prompts
8
9
pytestmark = pytest.mark.cpu_test
10
@@ -31,30 +31,30 @@ INPUTS_SLICES = [
31
]
32
33
34
-def test_parse_single_batch_empty():
34
+def test_parse_raw_single_batch_empty():
35
with pytest.raises(ValueError, match="at least one prompt"):
36
- parse_and_batch_prompt([])
36
+ parse_raw_prompts([])
37
38
with pytest.raises(ValueError, match="at least one prompt"):
39
- parse_and_batch_prompt([[]])
39
+ parse_raw_prompts([[]])
40
41
42
@pytest.mark.parametrize('string_input', STRING_INPUTS)
43
-def test_parse_single_batch_string_consistent(string_input: str):
44
- assert parse_and_batch_prompt(string_input) \
45
- == parse_and_batch_prompt([string_input])
43
+def test_parse_raw_single_batch_string_consistent(string_input: str):
44
+ assert parse_raw_prompts(string_input) \
45
+ == parse_raw_prompts([string_input])
46
47
48
@pytest.mark.parametrize('token_input', TOKEN_INPUTS)
49
-def test_parse_single_batch_token_consistent(token_input: list[int]):
50
- assert parse_and_batch_prompt(token_input) \
51
- == parse_and_batch_prompt([token_input])
49
+def test_parse_raw_single_batch_token_consistent(token_input: list[int]):
50
+ assert parse_raw_prompts(token_input) \
51
+ == parse_raw_prompts([token_input])
52
53
54
@pytest.mark.parametrize('inputs_slice', INPUTS_SLICES)
55
-def test_parse_single_batch_string_slice(inputs_slice: slice):
56
- assert parse_and_batch_prompt(STRING_INPUTS)[inputs_slice] \
57
- == parse_and_batch_prompt(STRING_INPUTS[inputs_slice])
55
+def test_parse_raw_single_batch_string_slice(inputs_slice: slice):
56
+ assert parse_raw_prompts(STRING_INPUTS)[inputs_slice] \
57
+ == parse_raw_prompts(STRING_INPUTS[inputs_slice])
58
59
60
# yapf: disable
vllm/entrypoints/openai/serving_completion.py
+1
-2
@@ -691,6 +691,5 @@ class OpenAIServingCompletion(OpenAIServing):
691
truncate_prompt_tokens=request.truncate_prompt_tokens,
692
add_special_tokens=request.add_special_tokens,
693
cache_salt=request.cache_salt,
694
- needs_detokenization=bool(request.echo
695
- and not request.return_token_ids),
694
+ needs_detokenization=bool(request.echo),
695
)
vllm/entrypoints/renderer.py
+72
-64
@@ -13,8 +13,9 @@ from pydantic import Field
13
14
from vllm.config import ModelConfig
15
from vllm.inputs.data import EmbedsPrompt as EngineEmbedsPrompt
16
+from vllm.inputs.data import TextPrompt as EngineTextPrompt
17
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
17
-from vllm.inputs.parse import parse_and_batch_prompt
18
+from vllm.inputs.parse import get_prompt_components, parse_raw_prompts
19
from vllm.transformers_utils.tokenizer import AnyTokenizer
20
from vllm.utils import AsyncMicrobatchTokenizer
21
@@ -41,6 +42,27 @@ class RenderConfig:
42
needs_detokenization: Optional[bool] = False
43
"""If True, detokenize IDs back to text for inclusion in outputs."""
44
45
+ def verify_truncate_prompt_tokens(
46
+ self, model_config: ModelConfig) -> Optional[int]:
47
+ """Validate and normalize `truncate_prompt_tokens` parameter."""
48
+ truncate_prompt_tokens = self.truncate_prompt_tokens
49
+ if truncate_prompt_tokens is None:
50
+ return None
51
+
52
+ if truncate_prompt_tokens == 0:
53
+ return 0
54
+
55
+ if truncate_prompt_tokens < 0:
56
+ truncate_prompt_tokens = model_config.max_model_len
57
+
58
+ max_length = self.max_length
59
+ if max_length is not None and truncate_prompt_tokens > max_length: # type: ignore[operator]
60
+ raise ValueError(
61
+ f"{truncate_prompt_tokens=} cannot be greater than "
62
+ f"{max_length=}. Please select a smaller truncation size.")
63
+
64
+ return truncate_prompt_tokens
65
+
66
67
class BaseRenderer(ABC):
68
"""
@@ -74,7 +96,7 @@ class BaseRenderer(ABC):
96
self,
97
*,
98
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
77
- config: "RenderConfig",
99
+ config: RenderConfig,
100
) -> list[EngineTokensPrompt]:
101
"""
102
Convert text or token inputs into engine-ready TokensPrompt objects.
@@ -107,7 +129,7 @@ class BaseRenderer(ABC):
129
prompt_or_prompts: Optional[Union[str, list[str], list[int],
130
list[list[int]]]] = None,
131
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
110
- config: "RenderConfig",
132
+ config: RenderConfig,
133
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
134
"""
135
Convert text/token and/or base64-encoded embeddings inputs into
@@ -189,47 +211,25 @@ class CompletionRenderer(BaseRenderer):
211
self,
212
*,
213
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
192
- config: "RenderConfig",
214
+ config: RenderConfig,
215
) -> list[EngineTokensPrompt]:
216
"""Implementation of prompt rendering for completion-style requests.
217
218
Uses async tokenizer pooling for improved performance. See base class
219
for detailed parameter documentation.
220
"""
199
- truncate_prompt_tokens = self._validate_and_normalize_truncate_tokens(
200
- config.truncate_prompt_tokens, config.max_length)
221
+ truncate_prompt_tokens = config.verify_truncate_prompt_tokens(
222
+ self.model_config)
223
if truncate_prompt_tokens == 0:
224
return []
225
204
- # Parse and batch the input prompts
205
- batch_inputs = parse_and_batch_prompt(prompt_or_prompts)
206
-
207
- tasks = []
208
- for prompt_input in batch_inputs:
209
- if prompt_input["is_tokens"] is True:
210
- # Token input
211
- # Note: detokenization is needed when echo is enabled,
212
- # where the input token IDs are decoded back to text.
213
- task = self._maybe_detokenize(prompt_input["content"],
214
- config.max_length,
215
- truncate_prompt_tokens,
216
- config.cache_salt,
217
- config.needs_detokenization)
218
- else:
219
- # Text input
220
- task = self._tokenize(prompt_input["content"],
221
- config.max_length,
222
- truncate_prompt_tokens,
223
- config.add_special_tokens,
224
- config.cache_salt)
225
- tasks.append(task)
226
-
227
- # Wait for all text tokenization to finish
228
- if tasks:
229
- tokenized_text_prompts = await asyncio.gather(*tasks)
230
- return tokenized_text_prompts
231
-
232
- return []
226
+ tasks = (self._create_prompt(
227
+ prompt_input,
228
+ config=config,
229
+ truncate_prompt_tokens=truncate_prompt_tokens,
230
+ ) for prompt_input in parse_raw_prompts(prompt_or_prompts))
231
+
232
+ return await asyncio.gather(*tasks)
233
234
async def render_prompt_and_embeds(
235
self,
@@ -237,14 +237,14 @@ class CompletionRenderer(BaseRenderer):
237
prompt_or_prompts: Optional[Union[str, list[str], list[int],
238
list[list[int]]]] = None,
239
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
240
- config: "RenderConfig",
240
+ config: RenderConfig,
241
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
242
"""
243
Render text/token prompts and/or precomputed embedding prompts. At
244
least one of `prompt_or_prompts` or `prompt_embeds` must be provided.
245
"""
246
- truncate_prompt_tokens = self._validate_and_normalize_truncate_tokens(
247
- config.truncate_prompt_tokens, config.max_length)
246
+ truncate_prompt_tokens = config.verify_truncate_prompt_tokens(
247
+ self.model_config)
248
if truncate_prompt_tokens == 0:
249
return []
250
@@ -265,29 +265,6 @@ class CompletionRenderer(BaseRenderer):
265
266
return rendered
267
268
- def _validate_and_normalize_truncate_tokens(
269
- self,
270
- truncate_prompt_tokens: Optional[int],
271
- max_length: Optional[int],
272
- ) -> Optional[int]:
273
- """Validate and normalize truncate_prompt_tokens parameter."""
274
- if truncate_prompt_tokens is None:
275
- return None
276
-
277
- if truncate_prompt_tokens == 0:
278
- return 0
279
-
280
- if truncate_prompt_tokens < 0:
281
- truncate_prompt_tokens = self.model_config.max_model_len
282
-
283
- if max_length is not None and truncate_prompt_tokens > max_length: # type: ignore[operator]
284
- raise ValueError(
285
- f"truncate_prompt_tokens ({truncate_prompt_tokens}) "
286
- f"cannot be greater than max_length ({max_length}). "
287
- f"Please select a smaller truncation size.")
288
-
289
- return truncate_prompt_tokens
290
-
268
def _maybe_apply_truncation(
269
self, token_ids: list[int],
270
truncate_prompt_tokens: Optional[int]) -> list[int]:
@@ -299,7 +276,38 @@ class CompletionRenderer(BaseRenderer):
276
277
return token_ids[-truncate_prompt_tokens:]
278
302
- async def _tokenize(
279
+ async def _create_prompt(
280
+ self,
281
+ prompt_input: Union[EngineTextPrompt, EngineTokensPrompt],
282
+ config: RenderConfig,
283
+ truncate_prompt_tokens: Optional[int],
284
+ ) -> EngineTokensPrompt:
285
+ prompt, prompt_token_ids, _ = get_prompt_components(prompt_input)
286
+
287
+ if prompt_token_ids is not None:
288
+ # NOTE: detokenization is needed when echo is enabled,
289
+ # where the input token IDs are decoded back to text.
290
+ return await self._create_prompt_from_token_ids(
291
+ prompt_token_ids,
292
+ config.max_length,
293
+ truncate_prompt_tokens,
294
+ config.cache_salt,
295
+ config.needs_detokenization,
296
+ )
297
+
298
+ if prompt is not None:
299
+ return await self._create_prompt_from_text(
300
+ prompt,
301
+ config.max_length,
302
+ truncate_prompt_tokens,
303
+ config.add_special_tokens,
304
+ config.cache_salt,
305
+ )
306
+
307
+ # TODO: Also handle embeds prompt using this method
308
+ raise NotImplementedError
309
+
310
+ async def _create_prompt_from_text(
311
self,
312
text: str,
313
max_length: Optional[int],
@@ -330,7 +338,7 @@ class CompletionRenderer(BaseRenderer):
338
return self._create_tokens_prompt(encoded.input_ids, max_length,
339
cache_salt, text)
340
333
- async def _maybe_detokenize(
341
+ async def _create_prompt_from_token_ids(
342
self,
343
token_ids: list[int],
344
max_length: Optional[int],
@@ -343,7 +351,7 @@ class CompletionRenderer(BaseRenderer):
351
truncate_prompt_tokens)
352
353
prompt = None
346
- if needs_detokenization is True:
354
+ if needs_detokenization:
355
async_tokenizer = self._get_async_tokenizer()
356
prompt = await async_tokenizer.decode(token_ids)
357
vllm/inputs/parse.py
+7
-54
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Sequence
4
from typing import (TYPE_CHECKING, Literal, NamedTuple, Optional, TypedDict,
5
- Union, cast, overload)
5
+ Union, cast)
6
7
from typing_extensions import TypeIs
8
@@ -16,34 +16,12 @@ if TYPE_CHECKING:
16
import torch
17
18
19
-class ParsedText(TypedDict):
20
- content: str
21
- is_tokens: Literal[False]
22
-
23
-
24
-class ParsedTokens(TypedDict):
25
- content: list[int]
26
- is_tokens: Literal[True]
27
-
28
-
29
-@overload
30
-def parse_and_batch_prompt(
31
- prompt: Union[str, list[str]], ) -> Sequence[ParsedText]:
32
- ...
33
-
34
-
35
-@overload
36
-def parse_and_batch_prompt(
37
- prompt: Union[list[int], list[list[int]]], ) -> Sequence[ParsedTokens]:
38
- ...
39
-
40
-
41
-def parse_and_batch_prompt(
19
+def parse_raw_prompts(
20
prompt: Union[str, list[str], list[int], list[list[int]]],
43
-) -> Union[Sequence[ParsedText], Sequence[ParsedTokens]]:
21
+) -> Union[Sequence[TextPrompt], Sequence[TokensPrompt]]:
22
if isinstance(prompt, str):
23
# case 1: a string
46
- return [ParsedText(content=prompt, is_tokens=False)]
24
+ return [TextPrompt(prompt=prompt)]
25
26
if isinstance(prompt, list):
27
if len(prompt) == 0:
@@ -52,13 +30,11 @@ def parse_and_batch_prompt(
30
if is_list_of(prompt, str):
31
# case 2: array of strings
32
prompt = cast(list[str], prompt)
55
- return [
56
- ParsedText(content=elem, is_tokens=False) for elem in prompt
57
- ]
33
+ return [TextPrompt(prompt=elem) for elem in prompt]
34
if is_list_of(prompt, int):
35
# case 3: array of tokens
36
prompt = cast(list[int], prompt)
61
- return [ParsedTokens(content=prompt, is_tokens=True)]
37
+ return [TokensPrompt(prompt_token_ids=prompt)]
38
if is_list_of(prompt, list):
39
prompt = cast(list[list[int]], prompt)
40
if len(prompt[0]) == 0:
@@ -66,10 +42,7 @@ def parse_and_batch_prompt(
42
43
if is_list_of(prompt[0], int):
44
# case 4: array of token arrays
69
- return [
70
- ParsedTokens(content=elem, is_tokens=True)
71
- for elem in prompt
72
- ]
45
+ return [TokensPrompt(prompt_token_ids=elem) for elem in prompt]
46
47
raise TypeError("prompt must be a string, array of strings, "
48
"array of tokens, or array of token arrays")
@@ -99,26 +72,6 @@ ParsedSingletonPrompt = Union[ParsedStrPrompt, ParsedTextPrompt,
72
ParsedTokensPrompt, ParsedEmbedsPrompt]
73
74
102
-@overload
103
-def parse_singleton_prompt(prompt: str) -> ParsedStrPrompt:
104
- ...
105
-
106
-
107
-@overload
108
-def parse_singleton_prompt(prompt: TextPrompt) -> ParsedTextPrompt:
109
- ...
110
-
111
-
112
-@overload
113
-def parse_singleton_prompt(prompt: TokensPrompt) -> ParsedTokensPrompt:
114
- ...
115
-
116
-
117
-@overload
118
-def parse_singleton_prompt(prompt: EmbedsPrompt) -> ParsedEmbedsPrompt:
119
- ...
120
-
121
-
75
def parse_singleton_prompt(prompt: SingletonPrompt) -> ParsedSingletonPrompt:
76
if isinstance(prompt, str):
77
return ParsedStrPrompt(type="str", content=prompt)