A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Consolidate rendering parameters into RenderConfig dataclass (#24543)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Sep 10, 2025 at 01:44 UTC
77f62613f9eb0963dffca1f58d22f718505e80c7
8 files changed
+167
-108
tests/entrypoints/test_renderer.py
+45
-25
@@ -10,7 +10,7 @@ import pybase64
10
import pytest
11
import torch
12
13
-from vllm.entrypoints.renderer import CompletionRenderer
13
+from vllm.entrypoints.renderer import CompletionRenderer, RenderConfig
14
from vllm.inputs.data import is_embeds_prompt
15
16
@@ -56,8 +56,8 @@ class TestRenderPrompt:
56
@pytest.mark.asyncio
57
async def test_token_input(self, renderer):
58
tokens = [101, 7592, 2088]
59
- results = await renderer.render_prompt(prompt_or_prompts=tokens,
60
- max_length=100)
59
+ results = await renderer.render_prompt(
60
+ prompt_or_prompts=tokens, config=RenderConfig(max_length=100))
61
62
assert len(results) == 1
63
assert results[0]["prompt_token_ids"] == tokens
@@ -65,8 +65,8 @@ class TestRenderPrompt:
65
@pytest.mark.asyncio
66
async def test_token_list_input(self, renderer):
67
token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]
68
- results = await renderer.render_prompt(prompt_or_prompts=token_lists,
69
- max_length=100)
68
+ results = await renderer.render_prompt(
69
+ prompt_or_prompts=token_lists, config=RenderConfig(max_length=100))
70
71
assert len(results) == 3
72
assert results[0]["prompt_token_ids"] == [101, 7592, 2088]
@@ -80,8 +80,9 @@ class TestRenderPrompt:
80
renderer.async_tokenizer_pool[
81
renderer.tokenizer] = mock_async_tokenizer
82
83
- results = await renderer.render_prompt(prompt_or_prompts="Hello world",
84
- max_length=100)
83
+ results = await renderer.render_prompt(
84
+ prompt_or_prompts="Hello world",
85
+ config=RenderConfig(max_length=100))
86
87
assert len(results) == 1
88
assert results[0]["prompt_token_ids"] == [101, 7592, 2088]
@@ -96,7 +97,8 @@ class TestRenderPrompt:
97
98
text_list_input = ["Hello world", "How are you?", "Good morning"]
99
results = await renderer.render_prompt(
99
- prompt_or_prompts=text_list_input, max_length=100)
100
+ prompt_or_prompts=text_list_input,
101
+ config=RenderConfig(max_length=100))
102
103
assert len(results) == 3
104
for result in results:
@@ -110,8 +112,9 @@ class TestRenderPrompt:
112
renderer.async_tokenizer_pool[
113
renderer.tokenizer] = mock_async_tokenizer
114
113
- results = await renderer.render_prompt(prompt_or_prompts="Hello world",
114
- max_length=100)
115
+ results = await renderer.render_prompt(
116
+ prompt_or_prompts="Hello world",
117
+ config=RenderConfig(max_length=100))
118
119
assert len(results) == 1
120
call_args = mock_async_tokenizer.call_args
@@ -126,8 +129,9 @@ class TestRenderPrompt:
129
renderer.tokenizer] = mock_async_tokenizer
130
131
results = await renderer.render_prompt(prompt_or_prompts="Hello world",
129
- max_length=100,
130
- truncate_prompt_tokens=50)
132
+ config=RenderConfig(
133
+ max_length=100,
134
+ truncate_prompt_tokens=50))
135
136
assert len(results) == 1
137
call_args = mock_async_tokenizer.call_args
@@ -143,8 +147,9 @@ class TestRenderPrompt:
147
renderer.tokenizer] = mock_async_tokenizer
148
149
results = await renderer.render_prompt(prompt_or_prompts="Hello world",
146
- max_length=200,
147
- truncate_prompt_tokens=-1)
150
+ config=RenderConfig(
151
+ max_length=200,
152
+ truncate_prompt_tokens=-1))
153
154
assert len(results) == 1
155
call_args = mock_async_tokenizer.call_args
@@ -157,8 +162,9 @@ class TestRenderPrompt:
162
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108,
163
109] # 10 tokens
164
results = await renderer.render_prompt(prompt_or_prompts=long_tokens,
160
- max_length=100,
161
- truncate_prompt_tokens=5)
165
+ config=RenderConfig(
166
+ max_length=100,
167
+ truncate_prompt_tokens=5))
168
169
assert len(results) == 1
170
# Should keep the last 5 tokens: [105, 106, 107, 108, 109]
@@ -170,7 +176,7 @@ class TestRenderPrompt:
176
177
with pytest.raises(ValueError, match="maximum context length"):
178
await renderer.render_prompt(prompt_or_prompts=long_tokens,
173
- max_length=100)
179
+ config=RenderConfig(max_length=100))
180
181
@pytest.mark.asyncio
182
async def test_no_tokenizer_for_text(self, mock_model_config):
@@ -181,7 +187,8 @@ class TestRenderPrompt:
187
188
with pytest.raises(ValueError, match="No tokenizer available"):
189
await renderer_no_tokenizer.render_prompt(
184
- prompt_or_prompts="Hello world", max_length=100)
190
+ prompt_or_prompts="Hello world",
191
+ config=RenderConfig(max_length=100))
192
193
@pytest.mark.asyncio
194
async def test_token_input_with_needs_detokenization(
@@ -196,7 +203,7 @@ class TestRenderPrompt:
203
tokens = [1, 2, 3, 4]
204
results = await renderer.render_prompt(
205
prompt_or_prompts=tokens,
199
- needs_detokenization=True,
206
+ config=RenderConfig(needs_detokenization=True),
207
)
208
209
assert len(results) == 1
@@ -221,7 +228,9 @@ class TestRenderEmbedPrompt:
228
embed_bytes = self._create_test_embed_bytes(test_tensor)
229
230
results = await renderer.render_prompt_and_embeds(
224
- prompt_embeds=embed_bytes, cache_salt="test_salt")
231
+ prompt_embeds=embed_bytes,
232
+ config=RenderConfig(cache_salt="test_salt"),
233
+ )
234
235
assert len(results) == 1
236
assert is_embeds_prompt(results[0])
@@ -240,7 +249,9 @@ class TestRenderEmbedPrompt:
249
]
250
251
results = await renderer.render_prompt_and_embeds(
243
- prompt_embeds=embed_bytes_list)
252
+ prompt_embeds=embed_bytes_list,
253
+ config=RenderConfig(),
254
+ )
255
256
assert len(results) == 2
257
for i, result in enumerate(results):
@@ -254,7 +265,9 @@ class TestRenderEmbedPrompt:
265
embed_bytes = self._create_test_embed_bytes(test_tensor)
266
267
results = await renderer.render_prompt_and_embeds(
257
- prompt_embeds=embed_bytes, truncate_prompt_tokens=10)
268
+ prompt_embeds=embed_bytes,
269
+ config=RenderConfig(truncate_prompt_tokens=10),
270
+ )
271
272
assert len(results) == 1
273
# Should keep last 10 tokens
@@ -271,7 +284,9 @@ class TestRenderEmbedPrompt:
284
embed_bytes = self._create_test_embed_bytes(test_tensor)
285
286
results = await renderer.render_prompt_and_embeds(
274
- prompt_embeds=embed_bytes)
287
+ prompt_embeds=embed_bytes,
288
+ config=RenderConfig(),
289
+ )
290
291
assert len(results) == 1
292
assert results[0]["prompt_embeds"].dtype == dtype
@@ -283,7 +298,9 @@ class TestRenderEmbedPrompt:
298
embed_bytes = self._create_test_embed_bytes(test_tensor)
299
300
results = await renderer.render_prompt_and_embeds(
286
- prompt_embeds=embed_bytes)
301
+ prompt_embeds=embed_bytes,
302
+ config=RenderConfig(),
303
+ )
304
305
assert len(results) == 1
306
# Should be squeezed to 2D
@@ -303,7 +320,10 @@ class TestRenderEmbedPrompt:
320
embed_bytes = self._create_test_embed_bytes(test_tensor)
321
322
results = await renderer.render_prompt_and_embeds(
306
- prompt_or_prompts="Hello world", prompt_embeds=embed_bytes)
323
+ prompt_or_prompts="Hello world",
324
+ prompt_embeds=embed_bytes,
325
+ config=RenderConfig(),
326
+ )
327
328
assert len(results) == 2
329
# First should be embed prompt
vllm/entrypoints/openai/serving_classification.py
+8
-2
@@ -20,6 +20,7 @@ from vllm.entrypoints.openai.serving_engine import (ClassificationServeContext,
20
OpenAIServing,
21
ServeContext)
22
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
23
+from vllm.entrypoints.renderer import RenderConfig
24
from vllm.logger import init_logger
25
from vllm.outputs import ClassificationOutput, PoolingRequestOutput
26
from vllm.pooling_params import PoolingParams
@@ -57,8 +58,7 @@ class ClassificationMixin(OpenAIServing):
58
renderer = self._get_renderer(ctx.tokenizer)
59
ctx.engine_prompts = await renderer.render_prompt(
60
prompt_or_prompts=ctx.request.input,
60
- max_length=self.max_model_len,
61
- truncate_prompt_tokens=ctx.request.truncate_prompt_tokens)
61
+ config=self._build_render_config(ctx.request))
62
63
return None
64
@@ -114,6 +114,12 @@ class ClassificationMixin(OpenAIServing):
114
usage=usage,
115
)
116
117
+ def _build_render_config(self,
118
+ request: ClassificationRequest) -> RenderConfig:
119
+ return RenderConfig(
120
+ max_length=self.max_model_len,
121
+ truncate_prompt_tokens=request.truncate_prompt_tokens)
122
+
123
124
class ServingClassification(ClassificationMixin):
125
request_id_prefix = "classify"
vllm/entrypoints/openai/serving_completion.py
+17
-8
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
30
clamp_prompt_logprobs)
31
# yapf: enable
32
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
33
+from vllm.entrypoints.renderer import RenderConfig
34
from vllm.entrypoints.utils import get_max_tokens
35
from vllm.inputs.data import (EmbedsPrompt, TokensPrompt, is_embeds_prompt,
36
is_tokens_prompt)
@@ -129,18 +130,11 @@ class OpenAIServingCompletion(OpenAIServing):
130
tokenizer = await self.engine_client.get_tokenizer(lora_request
131
)
132
renderer = self._get_renderer(tokenizer)
132
- max_input_tokens_len = self.max_model_len - (request.max_tokens
133
- or 0)
133
134
engine_prompts = await renderer.render_prompt_and_embeds(
135
prompt_or_prompts=request.prompt,
136
prompt_embeds=request.prompt_embeds,
138
- max_length=max_input_tokens_len,
139
- truncate_prompt_tokens=request.truncate_prompt_tokens,
140
- add_special_tokens=request.add_special_tokens,
141
- cache_salt=request.cache_salt,
142
- needs_detokenization=bool(request.echo
143
- and not request.return_token_ids),
137
+ config=self._build_render_config(request),
138
)
139
except ValueError as e:
140
logger.exception("Error in preprocessing prompt inputs")
@@ -677,3 +671,18 @@ class OpenAIServingCompletion(OpenAIServing):
671
tokens=out_tokens,
672
top_logprobs=out_top_logprobs,
673
)
674
+
675
+ def _build_render_config(
676
+ self,
677
+ request: CompletionRequest,
678
+ max_input_length: Optional[int] = None,
679
+ ) -> RenderConfig:
680
+ max_input_tokens_len = self.max_model_len - (request.max_tokens or 0)
681
+ return RenderConfig(
682
+ max_length=max_input_tokens_len,
683
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
684
+ add_special_tokens=request.add_special_tokens,
685
+ cache_salt=request.cache_salt,
686
+ needs_detokenization=bool(request.echo
687
+ and not request.return_token_ids),
688
+ )
vllm/entrypoints/openai/serving_embedding.py
+15
-9
@@ -28,6 +28,7 @@ from vllm.entrypoints.openai.serving_engine import (EmbeddingServeContext,
28
TextTokensPrompt)
29
# yapf: enable
30
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
31
+from vllm.entrypoints.renderer import RenderConfig
32
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
33
from vllm.logger import init_logger
34
from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
@@ -97,23 +98,28 @@ class EmbeddingMixin(OpenAIServing):
98
add_special_tokens=ctx.request.add_special_tokens,
99
)
100
else:
100
- # Set max_length based on chunked processing capability
101
- if self._should_use_chunked_processing(ctx.request):
102
- max_length = None
103
- else:
104
- max_length = self.max_embed_len or self.max_model_len
105
-
101
ctx.engine_prompts = await renderer.render_prompt(
102
prompt_or_prompts=ctx.request.input,
108
- max_length=max_length,
109
- truncate_prompt_tokens=ctx.request.truncate_prompt_tokens,
110
- add_special_tokens=ctx.request.add_special_tokens,
103
+ config=self._build_render_config(ctx.request),
104
)
105
return None
106
except (ValueError, TypeError) as e:
107
logger.exception("Error in preprocessing prompt inputs")
108
return self.create_error_response(str(e))
109
110
+ def _build_render_config(
111
+ self, request: EmbeddingCompletionRequest) -> RenderConfig:
112
+ # Set max_length based on chunked processing capability
113
+ if self._should_use_chunked_processing(request):
114
+ max_length = None
115
+ else:
116
+ max_length = self.max_embed_len or self.max_model_len
117
+
118
+ return RenderConfig(
119
+ max_length=max_length,
120
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
121
+ add_special_tokens=request.add_special_tokens)
122
+
123
@override
124
def _build_response(
125
self,
vllm/entrypoints/openai/serving_engine.py
+15
-1
@@ -58,7 +58,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
58
TranslationRequest)
59
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
60
from vllm.entrypoints.openai.tool_parsers import ToolParser
61
-from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer
61
+from vllm.entrypoints.renderer import (BaseRenderer, CompletionRenderer,
62
+ RenderConfig)
63
# yapf: enable
64
from vllm.inputs.data import PromptType
65
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
@@ -248,6 +249,19 @@ class OpenAIServing:
249
tokenizer=tokenizer,
250
async_tokenizer_pool=self._async_tokenizer_pool)
251
252
+ def _build_render_config(
253
+ self,
254
+ request: Any,
255
+ ) -> RenderConfig:
256
+ """
257
+ Build and return a `RenderConfig` for an endpoint.
258
+
259
+ Used by the renderer to control how prompts are prepared
260
+ (e.g., tokenization and length handling). Endpoints should
261
+ implement this with logic appropriate to their request type.
262
+ """
263
+ raise NotImplementedError
264
+
265
def _get_async_tokenizer(self, tokenizer) -> AsyncMicrobatchTokenizer:
266
"""
267
Return (and cache) an `AsyncMicrobatchTokenizer` bound to the
vllm/entrypoints/openai/serving_pooling.py
+9
-4
@@ -28,6 +28,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
28
# yapf: enable
29
from vllm.entrypoints.openai.serving_engine import OpenAIServing
30
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
31
+from vllm.entrypoints.renderer import RenderConfig
32
from vllm.entrypoints.utils import _validate_truncation_size
33
from vllm.logger import init_logger
34
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -149,10 +150,7 @@ class OpenAIServingPooling(OpenAIServing):
150
elif isinstance(request, PoolingCompletionRequest):
151
engine_prompts = await renderer.render_prompt(
152
prompt_or_prompts=request.input,
152
- max_length=self.max_model_len,
153
- truncate_prompt_tokens=truncate_prompt_tokens,
154
- add_special_tokens=request.add_special_tokens,
155
- cache_salt=getattr(request, 'cache_salt', None),
153
+ config=self._build_render_config(request),
154
)
155
else:
156
raise ValueError(
@@ -270,3 +268,10 @@ class OpenAIServingPooling(OpenAIServing):
268
data=items,
269
usage=usage,
270
)
271
+
272
+ def _build_render_config(
273
+ self, request: PoolingCompletionRequest) -> RenderConfig:
274
+ return RenderConfig(
275
+ max_length=self.max_model_len,
276
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
277
+ add_special_tokens=request.add_special_tokens)
vllm/entrypoints/openai/serving_tokenization.py
+7
-4
@@ -22,6 +22,7 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
22
# yapf: enable
23
from vllm.entrypoints.openai.serving_engine import OpenAIServing
24
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
25
+from vllm.entrypoints.renderer import RenderConfig
26
from vllm.logger import init_logger
27
from vllm.transformers_utils.tokenizer import AnyTokenizer
28
@@ -72,7 +73,7 @@ class OpenAIServingTokenization(OpenAIServing):
73
[tool.model_dump() for tool in request.tools])
74
(
75
_,
75
- request_prompts,
76
+ _,
77
engine_prompts,
78
) = await self._preprocess_chat(
79
request,
@@ -90,15 +91,14 @@ class OpenAIServingTokenization(OpenAIServing):
91
else:
92
engine_prompts = await renderer.render_prompt(
93
prompt_or_prompts=request.prompt,
93
- add_special_tokens=request.add_special_tokens,
94
- cache_salt=getattr(request, 'cache_salt', None),
94
+ config=self._build_render_config(request),
95
)
96
except (ValueError, TypeError, jinja2.TemplateError) as e:
97
logger.exception("Error in preprocessing prompt inputs")
98
return self.create_error_response(f"{e} {e.__cause__}")
99
100
input_ids: list[int] = []
101
- for i, engine_prompt in enumerate(engine_prompts):
101
+ for engine_prompt in engine_prompts:
102
self._log_inputs(request_id,
103
engine_prompt,
104
params=None,
@@ -157,6 +157,9 @@ class OpenAIServingTokenization(OpenAIServing):
157
return self.create_error_response(
158
f"Failed to get tokenizer info: {str(e)}")
159
160
+ def _build_render_config(self, request: TokenizeRequest) -> RenderConfig:
161
+ return RenderConfig(add_special_tokens=request.add_special_tokens)
162
+
163
164
@dataclass
165
class TokenizerInfo:
vllm/entrypoints/renderer.py
+51
-55
@@ -4,6 +4,7 @@
4
import asyncio
5
import io
6
from abc import ABC, abstractmethod
7
+from dataclasses import dataclass
8
from typing import Annotated, Optional, Union
9
10
import pybase64
@@ -18,6 +19,29 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
19
from vllm.utils import AsyncMicrobatchTokenizer
20
21
22
+@dataclass(frozen=True)
23
+class RenderConfig:
24
+ """Configuration to control how prompts are prepared."""
25
+
26
+ max_length: Optional[int] = None
27
+ """Maximum allowable total input token length. If provided,
28
+ token inputs longer than this raise ``ValueError``."""
29
+
30
+ truncate_prompt_tokens: Optional[int] = None
31
+ """Number of tokens to keep. ``None`` means no truncation.
32
+ ``0`` yields an empty list (and skips embeds).
33
+ ``-1`` maps to ``model_config.max_model_len``."""
34
+
35
+ add_special_tokens: Optional[bool] = True
36
+ """Whether to add model-specific special tokens during tokenization."""
37
+
38
+ cache_salt: Optional[str] = None
39
+ """String to disambiguate prefix cache entries."""
40
+
41
+ needs_detokenization: Optional[bool] = False
42
+ """If True, detokenize IDs back to text for inclusion in outputs."""
43
+
44
+
45
class BaseRenderer(ABC):
46
"""
47
Base class for unified input processing and rendering.
@@ -48,12 +72,9 @@ class BaseRenderer(ABC):
72
@abstractmethod
73
async def render_prompt(
74
self,
75
+ *,
76
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
52
- max_length: Optional[int] = None,
53
- truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,
54
- add_special_tokens: Optional[bool] = True,
55
- cache_salt: Optional[str] = None,
56
- needs_detokenization: Optional[bool] = False,
77
+ config: "RenderConfig",
78
) -> list[EngineTokensPrompt]:
79
"""
80
Convert text or token inputs into engine-ready TokensPrompt objects.
@@ -68,16 +89,8 @@ class BaseRenderer(ABC):
89
- ``list[str]``: Batch of text prompts.
90
- ``list[int]``: Single pre-tokenized sequence.
91
- ``list[list[int]]``: Batch of pre-tokenized sequences.
71
- max_length: Maximum allowable total input token length. If provided,
72
- token inputs longer than this raise ``ValueError``.
73
- truncate_prompt_tokens: Number of tokens to keep. ``None`` means no
74
- truncation. ``0`` yields an empty list (and skips embeds).
75
- ``-1`` maps to ``model_config.max_model_len``.
76
- add_special_tokens: Whether to add model-specific special tokens
77
- during text tokenization.
78
- cache_salt: Optional string to disambiguate prefix cache entries.
79
- needs_detokenization: If True and ``prompt_or_prompts`` is token
80
- input, detokenize IDs back to text for inclusion in outputs.
92
+ config: Render configuration controlling how prompts are prepared
93
+ (e.g., tokenization and length handling).
94
95
Returns:
96
list[EngineTokensPrompt]: Engine-ready token prompts.
@@ -90,18 +103,15 @@ class BaseRenderer(ABC):
103
@abstractmethod
104
async def render_prompt_and_embeds(
105
self,
106
+ *,
107
prompt_or_prompts: Optional[Union[str, list[str], list[int],
108
list[list[int]]]] = None,
109
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
96
- max_length: Optional[int] = None,
97
- truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,
98
- add_special_tokens: Optional[bool] = True,
99
- cache_salt: Optional[str] = None,
100
- needs_detokenization: Optional[bool] = False,
110
+ config: "RenderConfig",
111
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
112
"""
113
Convert text/token and/or base64-encoded embeddings inputs into
104
- engine-ready prompt objects.
114
+ engine-ready prompt objects using a unified RenderConfig.
115
116
At least one of ``prompt_or_prompts`` or ``prompt_embeds`` must be
117
provided and non-empty. If both are omitted or empty (e.g., empty
@@ -111,15 +121,8 @@ class BaseRenderer(ABC):
121
prompt_or_prompts: Text or token inputs to include.
122
prompt_embeds: Base64-encoded bytes (or list thereof) containing a
123
torch-saved tensor to be used as prompt embeddings.
114
- max_length: Maximum allowable total input token length. If provided,
115
- inputs longer than this raise ``ValueError``.
116
- truncate_prompt_tokens: Number of tokens/rows to keep from the end
117
- of the sequence. ``-1`` maps to ``model_config.max_model_len``.
118
- add_special_tokens: Whether to add model-specific special tokens
119
- during text tokenization.
120
- cache_salt: Optional string to disambiguate prefix cache entries.
121
- needs_detokenization: If True and ``prompt_or_prompts`` is token
122
- input, detokenize IDs back to text for inclusion in outputs.
124
+ config: Render configuration controlling how prompts are prepared
125
+ (e.g., tokenization and length handling).
126
127
Returns:
128
list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
@@ -184,12 +187,9 @@ class CompletionRenderer(BaseRenderer):
187
188
async def render_prompt(
189
self,
190
+ *,
191
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
188
- max_length: Optional[int] = None,
189
- truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,
190
- add_special_tokens: Optional[bool] = True,
191
- cache_salt: Optional[str] = None,
192
- needs_detokenization: Optional[bool] = False,
192
+ config: "RenderConfig",
193
) -> list[EngineTokensPrompt]:
194
"""Implementation of prompt rendering for completion-style requests.
195
@@ -197,7 +197,7 @@ class CompletionRenderer(BaseRenderer):
197
for detailed parameter documentation.
198
"""
199
truncate_prompt_tokens = self._validate_and_normalize_truncate_tokens(
200
- truncate_prompt_tokens, max_length)
200
+ config.truncate_prompt_tokens, config.max_length)
201
if truncate_prompt_tokens == 0:
202
return []
203
@@ -211,16 +211,19 @@ class CompletionRenderer(BaseRenderer):
211
detokenize_task = asyncio.create_task(
212
# Note: detokenization is needed when echo is enabled,
213
# where the input token IDs are decoded back to text.
214
- self._maybe_detokenize(prompt_input["content"], max_length,
215
- truncate_prompt_tokens, cache_salt,
216
- needs_detokenization))
214
+ self._maybe_detokenize(prompt_input["content"],
215
+ config.max_length,
216
+ truncate_prompt_tokens,
217
+ config.cache_salt,
218
+ config.needs_detokenization))
219
tasks.append(detokenize_task)
220
else:
221
# Text input
222
tokenize_task = asyncio.create_task(
221
- self._tokenize(prompt_input["content"], max_length,
222
- truncate_prompt_tokens, add_special_tokens,
223
- cache_salt))
223
+ self._tokenize(prompt_input["content"], config.max_length,
224
+ truncate_prompt_tokens,
225
+ config.add_special_tokens,
226
+ config.cache_salt))
227
tasks.append(tokenize_task)
228
229
# Wait for all text tokenization to finish
@@ -232,21 +235,18 @@ class CompletionRenderer(BaseRenderer):
235
236
async def render_prompt_and_embeds(
237
self,
238
+ *,
239
prompt_or_prompts: Optional[Union[str, list[str], list[int],
240
list[list[int]]]] = None,
241
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
238
- max_length: Optional[int] = None,
239
- truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,
240
- add_special_tokens: Optional[bool] = True,
241
- cache_salt: Optional[str] = None,
242
- needs_detokenization: Optional[bool] = False,
242
+ config: "RenderConfig",
243
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
244
"""
245
Render text/token prompts and/or precomputed embedding prompts. At
246
least one of `prompt_or_prompts` or `prompt_embeds` must be provided.
247
"""
248
truncate_prompt_tokens = self._validate_and_normalize_truncate_tokens(
249
- truncate_prompt_tokens, max_length)
249
+ config.truncate_prompt_tokens, config.max_length)
250
if truncate_prompt_tokens == 0:
251
return []
252
@@ -255,17 +255,13 @@ class CompletionRenderer(BaseRenderer):
255
if prompt_embeds is not None:
256
rendered.extend(
257
self.load_prompt_embeds(prompt_embeds, truncate_prompt_tokens,
258
- cache_salt))
258
+ config.cache_salt))
259
if prompt_or_prompts is None or prompt_or_prompts == "":
260
return rendered
261
262
token_prompts = await self.render_prompt(
263
prompt_or_prompts=prompt_or_prompts,
264
- max_length=max_length,
265
- truncate_prompt_tokens=truncate_prompt_tokens,
266
- add_special_tokens=add_special_tokens,
267
- cache_salt=cache_salt,
268
- needs_detokenization=needs_detokenization,
264
+ config=config,
265
)
266
rendered.extend(token_prompts)
267
@@ -394,4 +390,4 @@ class CompletionRenderer(BaseRenderer):
390
tokens_prompt["cache_salt"] = cache_salt
391
if prompt is not None:
392
tokens_prompt["prompt"] = prompt
397
- return tokens_prompt
\ No newline at end of file
393
+ return tokens_prompt