A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 00:15 UTC
[Frontend] Add render endpoints for prompt preprocessing (#32473)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com> Signed-off-by: Hyunkyun Moon <mhg5303@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Hyunkyun Moon committed
Jan 19, 2026 at 13:21 UTC
3c8740aacb2b288390c986c169993ed2d97363f8
5 files changed
+373
-42
tests/entrypoints/openai/test_render.py
new
+226
@@ -0,0 +1,226 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+"""Tests for the /render endpoints that expose prompt preprocessing."""
5
+
6
+import httpx
7
+import pytest
8
+import pytest_asyncio
9
+
10
+from ...utils import RemoteOpenAIServer
11
+
12
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
13
+
14
+
15
+@pytest.fixture(scope="module")
16
+def server():
17
+ args: list[str] = []
18
+
19
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
20
+ yield remote_server
21
+
22
+
23
+@pytest_asyncio.fixture
24
+async def client(server):
25
+ async with httpx.AsyncClient(
26
+ base_url=server.url_for(""), timeout=30.0
27
+ ) as http_client:
28
+ yield http_client
29
+
30
+
31
+@pytest.mark.asyncio
32
+async def test_completion_render_basic(client):
33
+ """Test basic completion render endpoint."""
34
+ # Make request to render endpoint
35
+ response = await client.post(
36
+ "/v1/completions/render",
37
+ json={
38
+ "model": MODEL_NAME,
39
+ "prompt": "When should a chat-completions handler return an empty string?",
40
+ },
41
+ )
42
+
43
+ assert response.status_code == 200
44
+ data = response.json()
45
+
46
+ # Verify response structure
47
+ assert isinstance(data, list)
48
+ assert len(data) > 0
49
+
50
+ # Verify first prompt
51
+ first_prompt = data[0]
52
+ assert "prompt_token_ids" in first_prompt
53
+ assert "prompt" in first_prompt
54
+ assert isinstance(first_prompt["prompt_token_ids"], list)
55
+ assert len(first_prompt["prompt_token_ids"]) > 0
56
+ assert isinstance(first_prompt["prompt"], str)
57
+
58
+ # Verify prompt text is preserved
59
+ assert (
60
+ "When should a chat-completions handler return an empty string?"
61
+ in first_prompt["prompt"]
62
+ )
63
+
64
+
65
+@pytest.mark.asyncio
66
+async def test_chat_completion_render_basic(client):
67
+ """Test basic chat completion render endpoint."""
68
+ # Make request to render endpoint
69
+ response = await client.post(
70
+ "/v1/chat/completions/render",
71
+ json={
72
+ "model": MODEL_NAME,
73
+ "messages": [
74
+ {
75
+ "role": "user",
76
+ "content": (
77
+ "Returning an empty string for the prompt may be confusing."
78
+ ),
79
+ }
80
+ ],
81
+ },
82
+ )
83
+
84
+ assert response.status_code == 200
85
+ data = response.json()
86
+
87
+ # Verify response structure - should be [conversation, engine_prompts]
88
+ assert isinstance(data, list)
89
+ assert len(data) == 2
90
+
91
+ conversation, engine_prompts = data
92
+
93
+ # Verify conversation
94
+ assert isinstance(conversation, list)
95
+ assert len(conversation) > 0
96
+ assert conversation[0]["role"] == "user"
97
+ assert "empty string" in conversation[0]["content"]
98
+
99
+ # Verify engine_prompts
100
+ assert isinstance(engine_prompts, list)
101
+ assert len(engine_prompts) > 0
102
+
103
+ first_prompt = engine_prompts[0]
104
+ assert "prompt_token_ids" in first_prompt
105
+ assert "prompt" in first_prompt
106
+ assert isinstance(first_prompt["prompt_token_ids"], list)
107
+ assert len(first_prompt["prompt_token_ids"]) > 0
108
+
109
+ # Verify chat template was applied (should have instruction markers)
110
+ assert "[INST]" in first_prompt["prompt"]
111
+ assert "[/INST]" in first_prompt["prompt"]
112
+
113
+ # Verify token IDs are correctly preserved as integers
114
+ token_ids = first_prompt["prompt_token_ids"]
115
+ assert all(isinstance(tid, int) for tid in token_ids)
116
+ # Verify BOS token (usually 1 for LLaMA models)
117
+ assert token_ids[0] == 1
118
+
119
+
120
+@pytest.mark.asyncio
121
+async def test_completion_render_multiple_prompts(client):
122
+ """Test completion render with multiple prompts."""
123
+ response = await client.post(
124
+ "/v1/completions/render",
125
+ json={
126
+ "model": MODEL_NAME,
127
+ "prompt": ["Hello world", "Goodbye world"],
128
+ },
129
+ )
130
+
131
+ assert response.status_code == 200
132
+ data = response.json()
133
+
134
+ # Should return two prompts
135
+ assert isinstance(data, list)
136
+ assert len(data) == 2
137
+
138
+ # Verify both prompts have required fields
139
+ for prompt in data:
140
+ assert "prompt_token_ids" in prompt
141
+ assert "prompt" in prompt
142
+ assert len(prompt["prompt_token_ids"]) > 0
143
+
144
+
145
+@pytest.mark.asyncio
146
+async def test_chat_completion_render_multi_turn(client):
147
+ """Test chat completion render with multi-turn conversation."""
148
+ response = await client.post(
149
+ "/v1/chat/completions/render",
150
+ json={
151
+ "model": MODEL_NAME,
152
+ "messages": [
153
+ {"role": "user", "content": "Hello"},
154
+ {"role": "assistant", "content": "Hi there!"},
155
+ {"role": "user", "content": "How are you?"},
156
+ ],
157
+ },
158
+ )
159
+
160
+ assert response.status_code == 200
161
+ data = response.json()
162
+
163
+ conversation, engine_prompts = data
164
+
165
+ # Verify all messages preserved
166
+ assert len(conversation) == 3
167
+ assert conversation[0]["role"] == "user"
168
+ assert conversation[1]["role"] == "assistant"
169
+ assert conversation[2]["role"] == "user"
170
+
171
+ # Verify tokenization occurred
172
+ assert len(engine_prompts) > 0
173
+ assert len(engine_prompts[0]["prompt_token_ids"]) > 0
174
+
175
+
176
+@pytest.mark.asyncio
177
+async def test_completion_render_error_invalid_model(client):
178
+ """Test completion render with invalid model returns error."""
179
+ response = await client.post(
180
+ "/v1/completions/render",
181
+ json={
182
+ "model": "invalid-model-name",
183
+ "prompt": "Hello",
184
+ },
185
+ )
186
+
187
+ assert response.status_code == 404
188
+ data = response.json()
189
+ assert "error" in data
190
+
191
+
192
+@pytest.mark.asyncio
193
+async def test_chat_completion_render_error_invalid_model(client):
194
+ """Test chat completion render with invalid model returns error."""
195
+ response = await client.post(
196
+ "/v1/chat/completions/render",
197
+ json={
198
+ "model": "invalid-model-name",
199
+ "messages": [{"role": "user", "content": "Hello"}],
200
+ },
201
+ )
202
+
203
+ assert response.status_code == 404
204
+ data = response.json()
205
+ assert "error" in data
206
+
207
+
208
+@pytest.mark.asyncio
209
+async def test_completion_render_no_generation(client):
210
+ """Verify render endpoint does not generate text."""
211
+ # This test verifies that calling render is fast (no generation)
212
+ import time
213
+
214
+ start = time.perf_counter()
215
+ response = await client.post(
216
+ "/v1/completions/render",
217
+ json={
218
+ "model": MODEL_NAME,
219
+ "prompt": "Tell me a very long story about " * 10,
220
+ },
221
+ )
222
+ elapsed = time.perf_counter() - start
223
+
224
+ assert response.status_code == 200
225
+ # Render should be fast (< 1 second) since no generation
226
+ assert elapsed < 1.0
vllm/entrypoints/openai/chat_completion/api_router.py
+31
@@ -73,5 +73,36 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
73
return StreamingResponse(content=generator, media_type="text/event-stream")
74
75
76
+@router.post(
77
+ "/v1/chat/completions/render",
78
+ dependencies=[Depends(validate_json_request)],
79
+ response_model=list,
80
+ responses={
81
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
82
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
83
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
84
+ },
85
+)
86
+async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
87
+ """Render chat completion request and return conversation and engine
88
+ prompts without generating."""
89
+ handler = chat(raw_request)
90
+ if handler is None:
91
+ base_server = raw_request.app.state.openai_serving_tokenization
92
+ return base_server.create_error_response(
93
+ message="The model does not support Chat Completions API"
94
+ )
95
+
96
+ try:
97
+ result = await handler.render_chat_request(request)
98
+ except Exception as e:
99
+ return handler.create_error_response(e)
100
+
101
+ if isinstance(result, ErrorResponse):
102
+ return JSONResponse(content=result.model_dump(), status_code=result.error.code)
103
+
104
+ return JSONResponse(content=result)
105
+
106
+
107
def attach_router(app: FastAPI):
108
app.include_router(router)
vllm/entrypoints/openai/chat_completion/serving.py
+39
-14
@@ -224,17 +224,16 @@ class OpenAIServingChat(OpenAIServing):
224
# Log but don't fail server startup if warmup fails
225
logger.exception("Chat template warmup failed")
226
227
- async def create_chat_completion(
227
+ async def render_chat_request(
228
self,
229
request: ChatCompletionRequest,
230
- raw_request: Request | None = None,
231
- ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
230
+ ) -> tuple[list[ConversationMessage], list[Any]] | ErrorResponse:
231
"""
233
- Chat Completion API similar to OpenAI's API.
232
+ render chat request by validating and preprocessing inputs.
233
235
- See https://platform.openai.com/docs/api-reference/chat/create
236
- for the API specification. This API mimics the OpenAI
237
- Chat Completion API.
234
+ Returns:
235
+ A tuple of (conversation, engine_prompts) on success,
236
+ or an ErrorResponse on failure.
237
"""
238
error_check_ret = await self._check_model(request)
239
if error_check_ret is not None:
@@ -248,12 +247,6 @@ class OpenAIServingChat(OpenAIServing):
247
raise self.engine_client.dead_error
248
249
try:
251
- lora_request = self._maybe_get_adapters(
252
- request, supports_default_mm_loras=True
253
- )
254
-
255
- model_name = self.models.model_name(lora_request)
256
-
250
tokenizer = await self.engine_client.get_tokenizer()
251
252
tool_parser = self.tool_parser
@@ -336,7 +329,27 @@ class OpenAIServingChat(OpenAIServing):
329
)
330
except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
331
logger.exception("Error in preprocessing prompt inputs")
339
- return self.create_error_response(f"{e} {e.__cause__}")
332
+ return self.create_error_response(e)
333
+
334
+ return conversation, engine_prompts
335
+
336
+ async def create_chat_completion(
337
+ self,
338
+ request: ChatCompletionRequest,
339
+ raw_request: Request | None = None,
340
+ ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
341
+ """
342
+ Chat Completion API similar to OpenAI's API.
343
+
344
+ See https://platform.openai.com/docs/api-reference/chat/create
345
+ for the API specification. This API mimics the OpenAI
346
+ Chat Completion API.
347
+ """
348
+ result = await self.render_chat_request(request)
349
+ if isinstance(result, ErrorResponse):
350
+ return result
351
+
352
+ conversation, engine_prompts = result
353
354
request_id = (
355
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
@@ -346,6 +359,18 @@ class OpenAIServingChat(OpenAIServing):
359
if raw_request:
360
raw_request.state.request_metadata = request_metadata
361
362
+ try:
363
+ lora_request = self._maybe_get_adapters(
364
+ request, supports_default_mm_loras=True
365
+ )
366
+
367
+ model_name = self.models.model_name(lora_request)
368
+
369
+ tokenizer = await self.engine_client.get_tokenizer()
370
+ except (ValueError, TypeError, RuntimeError) as e:
371
+ logger.exception("Error preparing request components")
372
+ return self.create_error_response(e)
373
+
374
# Extract data_parallel_rank from header (router can inject it)
375
data_parallel_rank = self._get_data_parallel_rank(raw_request)
376
vllm/entrypoints/openai/completion/api_router.py
+30
@@ -72,5 +72,35 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
72
return StreamingResponse(content=generator, media_type="text/event-stream")
73
74
75
+@router.post(
76
+ "/v1/completions/render",
77
+ dependencies=[Depends(validate_json_request)],
78
+ response_model=list,
79
+ responses={
80
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
81
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
82
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
83
+ },
84
+)
85
+async def render_completion(request: CompletionRequest, raw_request: Request):
86
+ """render completion request and return engine prompts without generating."""
87
+ handler = completion(raw_request)
88
+ if handler is None:
89
+ base_server = raw_request.app.state.openai_serving_tokenization
90
+ return base_server.create_error_response(
91
+ message="The model does not support Completions API"
92
+ )
93
+
94
+ try:
95
+ result = await handler.render_completion_request(request)
96
+ except Exception as e:
97
+ return handler.create_error_response(e)
98
+
99
+ if isinstance(result, ErrorResponse):
100
+ return JSONResponse(content=result.model_dump(), status_code=result.error.code)
101
+
102
+ return JSONResponse(content=result)
103
+
104
+
105
def attach_router(app: FastAPI):
106
app.include_router(router)
vllm/entrypoints/openai/completion/serving.py
+47
-28
@@ -83,19 +83,16 @@ class OpenAIServingCompletion(OpenAIServing):
83
self.default_sampling_params,
84
)
85
86
- async def create_completion(
86
+ async def render_completion_request(
87
self,
88
request: CompletionRequest,
89
- raw_request: Request | None = None,
90
- ) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
91
- """Completion API similar to OpenAI's API.
92
-
93
- See https://platform.openai.com/docs/api-reference/completions/create
94
- for the API specification. This API mimics the OpenAI Completion API.
89
+ ) -> list[TokensPrompt | EmbedsPrompt] | ErrorResponse:
90
+ """
91
+ render completion request by validating and preprocessing inputs.
92
96
- NOTE: Currently we do not support the following feature:
97
- - suffix (the language models we currently support do not support
98
- suffix)
93
+ Returns:
94
+ A list of engine_prompts on success,
95
+ or an ErrorResponse on failure.
96
"""
97
error_check_ret = await self._check_model(request)
98
if error_check_ret is not None:
@@ -119,16 +116,7 @@ class OpenAIServingCompletion(OpenAIServing):
116
"prompt_logprobs is not compatible with prompt embeds."
117
)
118
122
- request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
123
- created_time = int(time.time())
124
-
125
- request_metadata = RequestResponseMetadata(request_id=request_id)
126
- if raw_request:
127
- raw_request.state.request_metadata = request_metadata
128
-
119
try:
130
- lora_request = self._maybe_get_adapters(request)
131
-
120
if self.model_config.skip_tokenizer_init:
121
tokenizer = None
122
else:
@@ -140,17 +128,48 @@ class OpenAIServingCompletion(OpenAIServing):
128
prompt_embeds=request.prompt_embeds,
129
config=self._build_render_config(request),
130
)
143
- except ValueError as e:
144
- logger.exception("Error in preprocessing prompt inputs")
145
- return self.create_error_response(e)
146
- except TypeError as e:
147
- logger.exception("Error in preprocessing prompt inputs")
148
- return self.create_error_response(e)
149
- except RuntimeError as e:
131
+ except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
132
logger.exception("Error in preprocessing prompt inputs")
133
return self.create_error_response(e)
152
- except jinja2.TemplateError as e:
153
- logger.exception("Error in preprocessing prompt inputs")
134
+
135
+ return engine_prompts
136
+
137
+ async def create_completion(
138
+ self,
139
+ request: CompletionRequest,
140
+ raw_request: Request | None = None,
141
+ ) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
142
+ """Completion API similar to OpenAI's API.
143
+
144
+ See https://platform.openai.com/docs/api-reference/completions/create
145
+ for the API specification. This API mimics the OpenAI Completion API.
146
+
147
+ NOTE: Currently we do not support the following feature:
148
+ - suffix (the language models we currently support do not support
149
+ suffix)
150
+ """
151
+ result = await self.render_completion_request(request)
152
+ if isinstance(result, ErrorResponse):
153
+ return result
154
+
155
+ engine_prompts = result
156
+
157
+ request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
158
+ created_time = int(time.time())
159
+
160
+ request_metadata = RequestResponseMetadata(request_id=request_id)
161
+ if raw_request:
162
+ raw_request.state.request_metadata = request_metadata
163
+
164
+ try:
165
+ lora_request = self._maybe_get_adapters(request)
166
+
167
+ if self.model_config.skip_tokenizer_init:
168
+ tokenizer = None
169
+ else:
170
+ tokenizer = await self.engine_client.get_tokenizer()
171
+ except (ValueError, TypeError, RuntimeError) as e:
172
+ logger.exception("Error preparing request components")
173
return self.create_error_response(e)
174
175
# Extract data_parallel_rank from header (router can inject it)