A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 19:15 UTC
[Frontend] Delegate preprocessing to `OpenAIServingRender` (#36483)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
Mar 13, 2026 at 09:39 UTC
a2268617cfe91c4eebed1944327d8869ad628b8b
10 files changed
+203
-196
tests/entrypoints/openai/test_chat_error.py
+14
-1
@@ -13,6 +13,7 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
13
from vllm.entrypoints.openai.engine.protocol import GenerationError
14
from vllm.entrypoints.openai.models.protocol import BaseModelPath
15
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
17
from vllm.outputs import CompletionOutput, RequestOutput
18
from vllm.renderers.hf import HfRenderer
19
from vllm.tokenizers.registry import tokenizer_args_from_config
@@ -84,10 +85,20 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
85
engine_client=engine,
86
base_model_paths=BASE_MODEL_PATHS,
87
)
88
+ serving_render = OpenAIServingRender(
89
+ model_config=engine.model_config,
90
+ renderer=engine.renderer,
91
+ io_processor=engine.io_processor,
92
+ model_registry=models.registry,
93
+ request_logger=None,
94
+ chat_template=None,
95
+ chat_template_content_format="auto",
96
+ )
97
serving_chat = OpenAIServingChat(
98
engine,
99
models,
100
response_role="assistant",
101
+ openai_serving_render=serving_render,
102
request_logger=None,
103
chat_template=None,
104
chat_template_content_format="auto",
@@ -100,7 +111,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
111
[{"prompt_token_ids": [1, 2, 3]}],
112
)
113
103
- serving_chat._preprocess_chat = AsyncMock(side_effect=_fake_preprocess_chat)
114
+ serving_chat.openai_serving_render._preprocess_chat = AsyncMock(
115
+ side_effect=_fake_preprocess_chat
116
+ )
117
return serving_chat
118
119
tests/entrypoints/openai/test_completion_error.py
+11
@@ -13,6 +13,7 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
13
from vllm.entrypoints.openai.engine.protocol import GenerationError
14
from vllm.entrypoints.openai.models.protocol import BaseModelPath
15
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
17
from vllm.outputs import CompletionOutput, RequestOutput
18
from vllm.renderers.hf import HfRenderer
19
from vllm.tokenizers.registry import tokenizer_args_from_config
@@ -74,9 +75,19 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
75
engine_client=engine,
76
base_model_paths=BASE_MODEL_PATHS,
77
)
78
+ serving_render = OpenAIServingRender(
79
+ model_config=engine.model_config,
80
+ renderer=engine.renderer,
81
+ io_processor=engine.io_processor,
82
+ model_registry=models.registry,
83
+ request_logger=None,
84
+ chat_template=None,
85
+ chat_template_content_format="auto",
86
+ )
87
return OpenAIServingCompletion(
88
engine,
89
models,
90
+ openai_serving_render=serving_render,
91
request_logger=None,
92
)
93
tests/entrypoints/openai/test_lora_resolvers.py
+11
-1
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
18
from vllm.lora.request import LoRARequest
19
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
20
from vllm.renderers.hf import HfRenderer
@@ -145,8 +146,17 @@ def mock_serving_setup():
146
base_model_paths=BASE_MODEL_PATHS,
147
)
148
149
+ serving_render = OpenAIServingRender(
150
+ model_config=mock_engine.model_config,
151
+ renderer=mock_engine.renderer,
152
+ io_processor=mock_engine.io_processor,
153
+ model_registry=models.registry,
154
+ request_logger=None,
155
+ chat_template=None,
156
+ chat_template_content_format="auto",
157
+ )
158
serving_completion = OpenAIServingCompletion(
149
- mock_engine, models, request_logger=None
159
+ mock_engine, models, openai_serving_render=serving_render, request_logger=None
160
)
161
162
return mock_engine, serving_completion
tests/entrypoints/openai/test_serving_chat.py
+68
-14
@@ -21,8 +21,13 @@ from vllm.entrypoints.openai.engine.protocol import (
21
ErrorResponse,
22
RequestResponseMetadata,
23
)
24
-from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels
24
+from vllm.entrypoints.openai.models.serving import (
25
+ BaseModelPath,
26
+ OpenAIModelRegistry,
27
+ OpenAIServingModels,
28
+)
29
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
30
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
31
from vllm.exceptions import VLLMValidationError
32
from vllm.inputs import TokensPrompt
33
from vllm.outputs import CompletionOutput, RequestOutput
@@ -557,15 +562,32 @@ def _build_renderer(model_config: MockModelConfig):
562
)
563
564
565
+def _build_serving_render(
566
+ engine, model_registry: OpenAIModelRegistry
567
+) -> OpenAIServingRender:
568
+ return OpenAIServingRender(
569
+ model_config=engine.model_config,
570
+ renderer=engine.renderer,
571
+ io_processor=engine.io_processor,
572
+ model_registry=model_registry,
573
+ request_logger=None,
574
+ chat_template=CHAT_TEMPLATE,
575
+ chat_template_content_format="auto",
576
+ )
577
+
578
+
579
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
580
models = OpenAIServingModels(
581
engine_client=engine,
582
base_model_paths=BASE_MODEL_PATHS,
583
)
584
+ openai_serving_render = _build_serving_render(engine, models.registry)
585
+
586
serving_chat = OpenAIServingChat(
587
engine,
588
models,
589
response_role="assistant",
590
+ openai_serving_render=openai_serving_render,
591
chat_template=CHAT_TEMPLATE,
592
chat_template_content_format="auto",
593
request_logger=None,
@@ -586,10 +608,13 @@ async def _async_serving_chat_init():
608
engine = MockEngine()
609
610
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
611
+ openai_serving_render = _build_serving_render(engine, models.registry)
612
+
613
serving_completion = OpenAIServingChat(
614
engine,
615
models,
616
response_role="assistant",
617
+ openai_serving_render=openai_serving_render,
618
chat_template=CHAT_TEMPLATE,
619
chat_template_content_format="auto",
620
request_logger=None,
@@ -1182,7 +1207,9 @@ class TestServingChatWithHarmony:
1207
1208
# Test the Harmony messages for the first turn's input
1209
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1185
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1210
+ input_messages, _ = (
1211
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1212
+ )
1213
verify_harmony_messages(
1214
input_messages,
1215
[
@@ -1209,7 +1236,9 @@ class TestServingChatWithHarmony:
1236
1237
# Test the Harmony messages for the second turn's input
1238
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1212
- input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1239
+ input_messages_2, _ = (
1240
+ serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1241
+ )
1242
verify_harmony_messages(
1243
input_messages_2,
1244
[
@@ -1230,7 +1259,9 @@ class TestServingChatWithHarmony:
1259
1260
# Test the Harmony messages for the first turn's input
1261
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1233
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1262
+ input_messages, _ = (
1263
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1264
+ )
1265
verify_harmony_messages(
1266
input_messages,
1267
[
@@ -1274,7 +1305,9 @@ class TestServingChatWithHarmony:
1305
1306
# Test the Harmony messages for the second turn's input
1307
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1277
- input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1308
+ input_messages_2, _ = (
1309
+ serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1310
+ )
1311
verify_harmony_messages(
1312
input_messages_2,
1313
[
@@ -1311,7 +1344,9 @@ class TestServingChatWithHarmony:
1344
1345
# Test the Harmony messages for the first turn's input
1346
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1314
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1347
+ input_messages, _ = (
1348
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1349
+ )
1350
verify_harmony_messages(
1351
input_messages,
1352
[
@@ -1355,7 +1390,9 @@ class TestServingChatWithHarmony:
1390
1391
# Test the Harmony messages for the second turn's input
1392
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1358
- input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1393
+ input_messages_2, _ = (
1394
+ serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1395
+ )
1396
verify_harmony_messages(
1397
input_messages_2,
1398
[
@@ -1392,7 +1429,9 @@ class TestServingChatWithHarmony:
1429
1430
# Test the Harmony messages for the first turn's input
1431
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1395
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1432
+ input_messages, _ = (
1433
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1434
+ )
1435
verify_harmony_messages(
1436
input_messages,
1437
[
@@ -1436,7 +1475,9 @@ class TestServingChatWithHarmony:
1475
1476
# Test the Harmony messages for the second turn's input
1477
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1439
- input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1478
+ input_messages_2, _ = (
1479
+ serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1480
+ )
1481
verify_harmony_messages(
1482
input_messages_2,
1483
[
@@ -1486,7 +1527,9 @@ class TestServingChatWithHarmony:
1527
1528
# Test the Harmony messages for the third turn's input
1529
req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1489
- input_messages_3, _ = serving_chat._make_request_with_harmony(req_3)
1530
+ input_messages_3, _ = (
1531
+ serving_chat.openai_serving_render._make_request_with_harmony(req_3)
1532
+ )
1533
verify_harmony_messages(
1534
input_messages_3,
1535
[
@@ -1549,7 +1592,9 @@ class TestServingChatWithHarmony:
1592
1593
# Test the Harmony messages for the fourth turn's input
1594
req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1552
- input_messages_4, _ = serving_chat._make_request_with_harmony(req_4)
1595
+ input_messages_4, _ = (
1596
+ serving_chat.openai_serving_render._make_request_with_harmony(req_4)
1597
+ )
1598
verify_harmony_messages(
1599
input_messages_4,
1600
[
@@ -1598,7 +1643,9 @@ class TestServingChatWithHarmony:
1643
},
1644
]
1645
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1601
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1646
+ input_messages, _ = (
1647
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1648
+ )
1649
1650
verify_harmony_messages(
1651
input_messages,
@@ -1629,7 +1676,9 @@ class TestServingChatWithHarmony:
1676
},
1677
]
1678
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1632
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1679
+ input_messages, _ = (
1680
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1681
+ )
1682
1683
verify_harmony_messages(
1684
input_messages,
@@ -1658,7 +1707,9 @@ class TestServingChatWithHarmony:
1707
},
1708
]
1709
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1661
- input_messages, _ = serving_chat._make_request_with_harmony(req)
1710
+ input_messages, _ = (
1711
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1712
+ )
1713
1714
verify_harmony_messages(
1715
input_messages,
@@ -1689,11 +1740,14 @@ async def test_tool_choice_validation_without_parser():
1740
engine_client=mock_engine,
1741
base_model_paths=BASE_MODEL_PATHS,
1742
)
1743
+ openai_serving_render = _build_serving_render(mock_engine, models.registry)
1744
+
1745
# Create serving_chat without tool_parser (enable_auto_tools=False)
1746
serving_chat = OpenAIServingChat(
1747
mock_engine,
1748
models,
1749
response_role="assistant",
1750
+ openai_serving_render=openai_serving_render,
1751
chat_template=CHAT_TEMPLATE,
1752
chat_template_content_format="auto",
1753
request_logger=None,
tests/v1/engine/test_async_llm.py
+14
@@ -508,11 +508,25 @@ async def test_header_dp_rank_argument():
508
base_model_paths=BASE_MODEL_PATHS,
509
)
510
511
+ # Create render serving instance (required by OpenAIServingChat)
512
+ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
513
+
514
+ serving_render = OpenAIServingRender(
515
+ model_config=engine.model_config,
516
+ renderer=engine.renderer,
517
+ io_processor=engine.io_processor,
518
+ model_registry=models.registry,
519
+ request_logger=None,
520
+ chat_template=None,
521
+ chat_template_content_format="auto",
522
+ )
523
+
524
# Create serving chat instance
525
serving_chat = OpenAIServingChat(
526
engine_client=engine,
527
models=models,
528
response_role="assistant",
529
+ openai_serving_render=serving_render,
530
chat_template=None,
531
chat_template_content_format="auto",
532
request_logger=None,
vllm/entrypoints/anthropic/serving.py
+6
-1
@@ -10,7 +10,7 @@ import logging
10
import time
11
import uuid
12
from collections.abc import AsyncGenerator
13
-from typing import Any
13
+from typing import TYPE_CHECKING, Any
14
15
from fastapi import Request
16
@@ -43,6 +43,9 @@ from vllm.entrypoints.openai.engine.protocol import (
43
)
44
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
45
46
+if TYPE_CHECKING:
47
+ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
48
+
49
logger = logging.getLogger(__name__)
50
51
@@ -59,6 +62,7 @@ class AnthropicServingMessages(OpenAIServingChat):
62
models: OpenAIServingModels,
63
response_role: str,
64
*,
65
+ openai_serving_render: "OpenAIServingRender",
66
request_logger: RequestLogger | None,
67
chat_template: str | None,
68
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -73,6 +77,7 @@ class AnthropicServingMessages(OpenAIServingChat):
77
engine_client=engine_client,
78
models=models,
79
response_role=response_role,
80
+ openai_serving_render=openai_serving_render,
81
request_logger=request_logger,
82
chat_template=chat_template,
83
chat_template_content_format=chat_template_content_format,
vllm/entrypoints/openai/chat_completion/serving.py
+12
-128
@@ -6,12 +6,11 @@ import json
6
import time
7
from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
-from typing import Any, Final
9
+from typing import TYPE_CHECKING, Any, Final
10
11
import partial_json_parser
12
import regex as re
13
from fastapi import Request
14
-from openai_harmony import Message as OpenAIMessage
14
from partial_json_parser.core.options import Allow
15
16
from vllm.engine.protocol import EngineClient
@@ -56,17 +55,13 @@ from vllm.entrypoints.openai.engine.serving import (
55
)
56
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
57
from vllm.entrypoints.openai.parser.harmony_utils import (
59
- get_developer_message,
58
get_stop_tokens_for_assistant_actions,
59
get_streamable_parser_for_assistant,
62
- get_system_message,
63
- parse_chat_inputs_to_harmony_messages,
60
parse_chat_output,
65
- render_for_completion,
61
)
62
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
63
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
69
-from vllm.inputs.data import ProcessorInputs, TokensPrompt
64
+from vllm.inputs.data import ProcessorInputs
65
from vllm.logger import init_logger
66
from vllm.logprobs import Logprob
67
from vllm.outputs import CompletionOutput, RequestOutput
@@ -80,7 +75,9 @@ from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
75
from vllm.tool_parsers.utils import partial_json_loads
76
from vllm.utils.collection_utils import as_list
77
from vllm.utils.mistral import is_mistral_tokenizer
83
-from vllm.utils.mistral import mt as _mt
78
+
79
+if TYPE_CHECKING:
80
+ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
81
82
logger = init_logger(__name__)
83
@@ -92,6 +89,7 @@ class OpenAIServingChat(OpenAIServing):
89
models: OpenAIServingModels,
90
response_role: str,
91
*,
92
+ openai_serving_render: "OpenAIServingRender",
93
request_logger: RequestLogger | None,
94
chat_template: str | None,
95
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -114,6 +112,7 @@ class OpenAIServingChat(OpenAIServing):
112
return_tokens_as_token_ids=return_tokens_as_token_ids,
113
)
114
115
+ self.openai_serving_render = openai_serving_render
116
self.response_role = response_role
117
self.chat_template = chat_template
118
self.chat_template_content_format: Final = chat_template_content_format
@@ -186,7 +185,10 @@ class OpenAIServingChat(OpenAIServing):
185
request: ChatCompletionRequest,
186
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
187
"""
189
- render chat request by validating and preprocessing inputs.
188
+ Validate the model and preprocess a chat completion request.
189
+
190
+ Delegates preprocessing logic to OpenAIServingRender, adding the
191
+ engine-aware checks (LoRA model validation, engine health).
192
193
Returns:
194
A tuple of (conversation, engine_prompts) on success,
@@ -203,78 +205,7 @@ class OpenAIServingChat(OpenAIServing):
205
if self.engine_client.errored:
206
raise self.engine_client.dead_error
207
206
- tokenizer = self.renderer.tokenizer
207
-
208
- tool_parser = self.tool_parser
209
-
210
- if is_mistral_tokenizer(tokenizer):
211
- # because of issues with pydantic we need to potentially
212
- # re-serialize the tool_calls field of the request
213
- # for more info: see comment in `maybe_serialize_tool_calls`
214
- _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
215
- _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
216
- _mt.validate_request_params(request)
217
-
218
- # Check if tool parsing is unavailable (common condition)
219
- tool_parsing_unavailable = (
220
- tool_parser is None
221
- and not is_mistral_tokenizer(tokenizer)
222
- and not self.use_harmony
223
- )
224
-
225
- # Validate tool_choice when tool parsing is required but unavailable
226
- if tool_parsing_unavailable and request.tool_choice not in (
227
- None,
228
- "none",
229
- ):
230
- if request.tool_choice == "auto" and not self.enable_auto_tools:
231
- # for hf tokenizers, "auto" tools requires
232
- # --enable-auto-tool-choice and --tool-call-parser
233
- return self.create_error_response(
234
- '"auto" tool choice requires '
235
- "--enable-auto-tool-choice and --tool-call-parser to be set"
236
- )
237
- elif request.tool_choice != "auto":
238
- # "required" or named tool requires tool parser
239
- return self.create_error_response(
240
- f'tool_choice="{request.tool_choice}" requires '
241
- "--tool-call-parser to be set"
242
- )
243
-
244
- if request.tools is None or (
245
- request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
246
- ):
247
- tool_dicts = None
248
- else:
249
- tool_dicts = [tool.model_dump() for tool in request.tools]
250
-
251
- if not self.use_harmony:
252
- # Common case.
253
- error_check_ret = self._validate_chat_template(
254
- request_chat_template=request.chat_template,
255
- chat_template_kwargs=request.chat_template_kwargs,
256
- trust_request_chat_template=self.trust_request_chat_template,
257
- )
258
- if error_check_ret is not None:
259
- return error_check_ret
260
-
261
- conversation, engine_prompts = await self._preprocess_chat(
262
- request,
263
- request.messages,
264
- default_template=self.chat_template,
265
- default_template_content_format=self.chat_template_content_format,
266
- default_template_kwargs=self.default_chat_template_kwargs,
267
- tool_dicts=tool_dicts,
268
- tool_parser=tool_parser,
269
- )
270
- else:
271
- # For GPT-OSS.
272
- should_include_tools = tool_dicts is not None
273
- conversation, engine_prompts = self._make_request_with_harmony(
274
- request, should_include_tools
275
- )
276
-
277
- return conversation, engine_prompts
208
+ return await self.openai_serving_render.render_chat(request)
209
210
async def create_chat_completion(
211
self,
@@ -1875,50 +1806,3 @@ class OpenAIServingChat(OpenAIServing):
1806
)
1807
]
1808
)
1878
-
1879
- def _make_request_with_harmony(
1880
- self,
1881
- request: ChatCompletionRequest,
1882
- should_include_tools: bool = True,
1883
- ):
1884
- messages: list[OpenAIMessage] = []
1885
-
1886
- # because of issues with pydantic we need to potentially
1887
- # re-serialize the tool_calls field of the request
1888
- # for more info: see comment in `maybe_serialize_tool_calls`
1889
- _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
1890
-
1891
- # Add system message.
1892
- # NOTE: In Chat Completion API, browsing is enabled by default
1893
- # if the model supports it. TODO: Support browsing.
1894
- assert not self.supports_browsing
1895
- assert not self.supports_code_interpreter
1896
- if (reasoning_effort := request.reasoning_effort) == "none":
1897
- raise ValueError(f"Harmony does not support {reasoning_effort=}")
1898
- sys_msg = get_system_message(
1899
- reasoning_effort=reasoning_effort,
1900
- browser_description=None,
1901
- python_description=None,
1902
- with_custom_tools=should_include_tools,
1903
- )
1904
- messages.append(sys_msg)
1905
-
1906
- # Add developer message.
1907
- if request.tools:
1908
- dev_msg = get_developer_message(
1909
- tools=request.tools if should_include_tools else None # type: ignore[arg-type]
1910
- )
1911
- messages.append(dev_msg)
1912
-
1913
- # Add user message.
1914
- messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1915
-
1916
- # Render prompt token ids.
1917
- prompt_token_ids = render_for_completion(messages)
1918
- engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1919
-
1920
- # Add cache_salt if provided in the request
1921
- if request.cache_salt is not None:
1922
- engine_prompt["cache_salt"] = request.cache_salt
1923
-
1924
- return messages, [engine_prompt]
vllm/entrypoints/openai/completion/serving.py
+11
-21
@@ -5,7 +5,7 @@ import asyncio
5
import time
6
from collections.abc import AsyncGenerator, AsyncIterator
7
from collections.abc import Sequence as GenericSequence
8
-from typing import cast
8
+from typing import TYPE_CHECKING, cast
9
10
from fastapi import Request
11
@@ -42,6 +42,9 @@ from vllm.tokenizers import TokenizerLike
42
from vllm.utils.async_utils import merge_async_iterators
43
from vllm.utils.collection_utils import as_list
44
45
+if TYPE_CHECKING:
46
+ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
47
+
48
logger = init_logger(__name__)
49
50
@@ -51,6 +54,7 @@ class OpenAIServingCompletion(OpenAIServing):
54
engine_client: EngineClient,
55
models: OpenAIServingModels,
56
*,
57
+ openai_serving_render: "OpenAIServingRender",
58
request_logger: RequestLogger | None,
59
return_tokens_as_token_ids: bool = False,
60
enable_prompt_tokens_details: bool = False,
@@ -63,6 +67,7 @@ class OpenAIServingCompletion(OpenAIServing):
67
return_tokens_as_token_ids=return_tokens_as_token_ids,
68
)
69
70
+ self.openai_serving_render = openai_serving_render
71
self.enable_prompt_tokens_details = enable_prompt_tokens_details
72
self.enable_force_include_usage = enable_force_include_usage
73
@@ -79,7 +84,10 @@ class OpenAIServingCompletion(OpenAIServing):
84
request: CompletionRequest,
85
) -> list[ProcessorInputs] | ErrorResponse:
86
"""
82
- render completion request by validating and preprocessing inputs.
87
+ Validate the model and preprocess a completion request.
88
+
89
+ Delegates preprocessing logic to OpenAIServingRender, adding the
90
+ engine-aware checks (LoRA model validation, engine health).
91
92
Returns:
93
A list of engine_prompts on success,
@@ -95,25 +103,7 @@ class OpenAIServingCompletion(OpenAIServing):
103
if self.engine_client.errored:
104
raise self.engine_client.dead_error
105
98
- # Return error for unsupported features.
99
- if request.suffix is not None:
100
- return self.create_error_response("suffix is not currently supported")
101
-
102
- if request.echo and request.prompt_embeds is not None:
103
- return self.create_error_response("Echo is unsupported with prompt embeds.")
104
-
105
- if request.prompt_logprobs is not None and request.prompt_embeds is not None:
106
- return self.create_error_response(
107
- "prompt_logprobs is not compatible with prompt embeds."
108
- )
109
-
110
- engine_prompts = await self._preprocess_completion(
111
- request,
112
- prompt_input=request.prompt,
113
- prompt_embeds=request.prompt_embeds,
114
- )
115
-
116
- return engine_prompts
106
+ return await self.openai_serving_render.render_completion(request)
107
108
async def create_completion(
109
self,
vllm/entrypoints/openai/generate/api_router.py
+26
-21
@@ -72,6 +72,29 @@ async def init_generate_state(
72
tool_server = None
73
resolved_chat_template = load_chat_template(args.chat_template)
74
75
+ # Render endpoints are always backed by OpenAIServingRender so that
76
+ # /v1/chat/completions/render and /v1/completions/render work on both
77
+ # generate-mode and render-only servers.
78
+ # It is created first so that OpenAIServingChat and OpenAIServingCompletion
79
+ # can delegate their preprocessing logic to it.
80
+ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
81
+
82
+ state.openai_serving_render = OpenAIServingRender(
83
+ model_config=engine_client.model_config,
84
+ renderer=engine_client.renderer,
85
+ io_processor=engine_client.io_processor,
86
+ model_registry=state.openai_serving_models.registry,
87
+ request_logger=request_logger,
88
+ chat_template=resolved_chat_template,
89
+ chat_template_content_format=args.chat_template_content_format,
90
+ trust_request_chat_template=args.trust_request_chat_template,
91
+ enable_auto_tools=args.enable_auto_tool_choice,
92
+ exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
93
+ tool_parser=args.tool_call_parser,
94
+ default_chat_template_kwargs=args.default_chat_template_kwargs,
95
+ log_error_stack=args.log_error_stack,
96
+ )
97
+
98
state.openai_serving_responses = (
99
OpenAIServingResponses(
100
engine_client,
@@ -96,6 +119,7 @@ async def init_generate_state(
119
engine_client,
120
state.openai_serving_models,
121
args.response_role,
122
+ openai_serving_render=state.openai_serving_render,
123
request_logger=request_logger,
124
chat_template=resolved_chat_template,
125
chat_template_content_format=args.chat_template_content_format,
@@ -120,6 +144,7 @@ async def init_generate_state(
144
OpenAIServingCompletion(
145
engine_client,
146
state.openai_serving_models,
147
+ openai_serving_render=state.openai_serving_render,
148
request_logger=request_logger,
149
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
150
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
@@ -133,6 +158,7 @@ async def init_generate_state(
158
engine_client,
159
state.openai_serving_models,
160
args.response_role,
161
+ openai_serving_render=state.openai_serving_render,
162
request_logger=request_logger,
163
chat_template=resolved_chat_template,
164
chat_template_content_format=args.chat_template_content_format,
@@ -159,24 +185,3 @@ async def init_generate_state(
185
if "generate" in supported_tasks
186
else None
187
)
162
-
163
- # Render endpoints are always backed by OpenAIServingRender so that
164
- # /v1/chat/completions/render and /v1/completions/render work on both
165
- # generate-mode and render-only servers.
166
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
167
-
168
- state.openai_serving_render = OpenAIServingRender(
169
- model_config=engine_client.model_config,
170
- renderer=engine_client.renderer,
171
- io_processor=engine_client.io_processor,
172
- model_registry=state.openai_serving_models.registry,
173
- request_logger=request_logger,
174
- chat_template=resolved_chat_template,
175
- chat_template_content_format=args.chat_template_content_format,
176
- trust_request_chat_template=args.trust_request_chat_template,
177
- enable_auto_tools=args.enable_auto_tool_choice,
178
- exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
179
- tool_parser=args.tool_call_parser,
180
- default_chat_template_kwargs=args.default_chat_template_kwargs,
181
- log_error_stack=args.log_error_stack,
182
- )
vllm/entrypoints/serve/render/serving.py
+30
-9
@@ -87,15 +87,26 @@ class OpenAIServingRender:
87
self,
88
request: ChatCompletionRequest,
89
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
90
- """Copied from OpenAIServingChat.render_chat_request.
90
+ """Validate the model and preprocess a chat completion request.
91
92
- Differences: engine_client.errored check removed (no engine client).
92
+ This is the authoritative implementation used directly by the
93
+ GPU-less render server and delegated to by OpenAIServingChat.
94
"""
95
error_check_ret = await self._check_model(request)
96
if error_check_ret is not None:
97
logger.error("Error with model %s", error_check_ret)
98
return error_check_ret
99
+ return await self.render_chat(request)
100
101
+ async def render_chat(
102
+ self,
103
+ request: ChatCompletionRequest,
104
+ ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
105
+ """Core preprocessing logic for chat requests (no model/engine check).
106
+
107
+ Called directly by render_chat_request and delegated to by
108
+ OpenAIServingChat.render_chat_request after its engine-aware checks.
109
+ """
110
tokenizer = self.renderer.tokenizer
111
112
tool_parser = self.tool_parser
@@ -173,14 +184,25 @@ class OpenAIServingRender:
184
self,
185
request: CompletionRequest,
186
) -> list[ProcessorInputs] | ErrorResponse:
176
- """Copied from OpenAIServingCompletion.render_completion_request.
187
+ """Validate the model and preprocess a completion request.
188
178
- Differences: engine_client.errored check removed (no engine client).
189
+ This is the authoritative implementation used directly by the
190
+ GPU-less render server and delegated to by OpenAIServingCompletion.
191
"""
192
error_check_ret = await self._check_model(request)
193
if error_check_ret is not None:
194
return error_check_ret
195
+ return await self.render_completion(request)
196
197
+ async def render_completion(
198
+ self,
199
+ request: CompletionRequest,
200
+ ) -> list[ProcessorInputs] | ErrorResponse:
201
+ """Core preprocessing logic for completion requests (no model/engine check).
202
+
203
+ Called directly by render_completion_request and delegated to by
204
+ OpenAIServingCompletion.render_completion_request after its engine-aware checks.
205
+ """
206
# Return error for unsupported features.
207
if request.suffix is not None:
208
return self.create_error_response("suffix is not currently supported")
@@ -206,7 +228,7 @@ class OpenAIServingRender:
228
request: ChatCompletionRequest,
229
should_include_tools: bool = True,
230
):
209
- """Copied from OpenAIServingChat._make_request_with_harmony."""
231
+ """Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
232
messages: list[OpenAIMessage] = []
233
234
# because of issues with pydantic we need to potentially
@@ -219,11 +241,10 @@ class OpenAIServingRender:
241
# if the model supports it. TODO: Support browsing.
242
assert not self.supports_browsing
243
assert not self.supports_code_interpreter
222
- assert request.reasoning_effort != "none", (
223
- "Harmony does not support reasoning_effort='none'"
224
- )
244
+ if (reasoning_effort := request.reasoning_effort) == "none":
245
+ raise ValueError(f"Harmony does not support {reasoning_effort=}")
246
sys_msg = get_system_message(
226
- reasoning_effort=request.reasoning_effort,
247
+ reasoning_effort=reasoning_effort,
248
browser_description=None,
249
python_description=None,
250
with_custom_tools=should_include_tools,