A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 14:30 UTC
[Frontend] Delegate tokenization serving preprocessing to OpenAIServingRender (#37266)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
Mar 17, 2026 at 13:22 UTC
00f8e0d2113098b5fd37c8c24ba594fa4268ccc3
5 files changed
+33
-30
tests/entrypoints/openai/chat_completion/test_chat_error.py
+1
-1
@@ -111,7 +111,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
111
[{"prompt_token_ids": [1, 2, 3]}],
112
)
113
114
- serving_chat.openai_serving_render._preprocess_chat = AsyncMock(
114
+ serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
115
side_effect=_fake_preprocess_chat
116
)
117
return serving_chat
vllm/entrypoints/openai/api_server.py
+19
@@ -46,6 +46,7 @@ from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
46
from vllm.entrypoints.serve.elastic_ep.middleware import (
47
ScalingMiddleware,
48
)
49
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
50
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
51
from vllm.entrypoints.utils import (
52
cli_env_setup,
@@ -365,9 +366,27 @@ async def init_app_state(
366
lora_modules=lora_modules,
367
)
368
await state.openai_serving_models.init_static_loras()
369
+
370
+ state.openai_serving_render = OpenAIServingRender(
371
+ model_config=engine_client.model_config,
372
+ renderer=engine_client.renderer,
373
+ io_processor=engine_client.io_processor,
374
+ model_registry=state.openai_serving_models.registry,
375
+ request_logger=request_logger,
376
+ chat_template=resolved_chat_template,
377
+ chat_template_content_format=args.chat_template_content_format,
378
+ trust_request_chat_template=args.trust_request_chat_template,
379
+ enable_auto_tools=args.enable_auto_tool_choice,
380
+ exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
381
+ tool_parser=args.tool_call_parser,
382
+ default_chat_template_kwargs=args.default_chat_template_kwargs,
383
+ log_error_stack=args.log_error_stack,
384
+ )
385
+
386
state.openai_serving_tokenization = OpenAIServingTokenization(
387
engine_client,
388
state.openai_serving_models,
389
+ state.openai_serving_render,
390
request_logger=request_logger,
391
chat_template=resolved_chat_template,
392
chat_template_content_format=args.chat_template_content_format,
vllm/entrypoints/openai/generate/api_router.py
+1
-20
@@ -74,26 +74,7 @@ async def init_generate_state(
74
75
# Render endpoints are always backed by OpenAIServingRender so that
76
# /v1/chat/completions/render and /v1/completions/render work on both
77
- # generate-mode and render-only servers.
78
- # It is created first so that OpenAIServingChat and OpenAIServingCompletion
79
- # can delegate their preprocessing logic to it.
80
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
81
-
82
- state.openai_serving_render = OpenAIServingRender(
83
- model_config=engine_client.model_config,
84
- renderer=engine_client.renderer,
85
- io_processor=engine_client.io_processor,
86
- model_registry=state.openai_serving_models.registry,
87
- request_logger=request_logger,
88
- chat_template=resolved_chat_template,
89
- chat_template_content_format=args.chat_template_content_format,
90
- trust_request_chat_template=args.trust_request_chat_template,
91
- enable_auto_tools=args.enable_auto_tool_choice,
92
- exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
93
- tool_parser=args.tool_call_parser,
94
- default_chat_template_kwargs=args.default_chat_template_kwargs,
95
- log_error_stack=args.log_error_stack,
96
- )
77
+ # generate-mode and render-only servers. Created in init_app_state.
78
79
state.openai_serving_responses = (
80
OpenAIServingResponses(
vllm/entrypoints/serve/render/serving.py
+6
-6
@@ -226,7 +226,7 @@ class OpenAIServingRender:
226
227
if not self.use_harmony:
228
# Common case.
229
- error_check_ret = self._validate_chat_template(
229
+ error_check_ret = self.validate_chat_template(
230
request_chat_template=request.chat_template,
231
chat_template_kwargs=request.chat_template_kwargs,
232
trust_request_chat_template=self.trust_request_chat_template,
@@ -234,7 +234,7 @@ class OpenAIServingRender:
234
if error_check_ret is not None:
235
return error_check_ret
236
237
- conversation, engine_prompts = await self._preprocess_chat(
237
+ conversation, engine_prompts = await self.preprocess_chat(
238
request,
239
request.messages,
240
default_template=self.chat_template,
@@ -328,7 +328,7 @@ class OpenAIServingRender:
328
"prompt_logprobs is not compatible with prompt embeds."
329
)
330
331
- engine_prompts = await self._preprocess_completion(
331
+ engine_prompts = await self.preprocess_completion(
332
request,
333
prompt_input=request.prompt,
334
prompt_embeds=request.prompt_embeds,
@@ -426,7 +426,7 @@ class OpenAIServingRender:
426
) -> ErrorResponse | None:
427
return await self.model_registry.check_model(request.model)
428
429
- def _validate_chat_template(
429
+ def validate_chat_template(
430
self,
431
request_chat_template: str | None,
432
chat_template_kwargs: dict[str, Any] | None,
@@ -447,7 +447,7 @@ class OpenAIServingRender:
447
)
448
return None
449
450
- async def _preprocess_completion(
450
+ async def preprocess_completion(
451
self,
452
request: Any,
453
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
@@ -490,7 +490,7 @@ class OpenAIServingRender:
490
},
491
)
492
493
- async def _preprocess_chat(
493
+ async def preprocess_chat(
494
self,
495
request: Any,
496
messages: list[Any],
vllm/entrypoints/serve/tokenize/serving.py
+6
-3
@@ -11,6 +11,7 @@ from vllm.entrypoints.logger import RequestLogger
11
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12
from vllm.entrypoints.openai.engine.serving import OpenAIServing
13
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
14
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
15
from vllm.entrypoints.serve.tokenize.protocol import (
16
DetokenizeRequest,
17
DetokenizeResponse,
@@ -31,6 +32,7 @@ class OpenAIServingTokenization(OpenAIServing):
32
self,
33
engine_client: EngineClient,
34
models: OpenAIServingModels,
35
+ openai_serving_render: OpenAIServingRender,
36
*,
37
request_logger: RequestLogger | None,
38
chat_template: str | None,
@@ -44,6 +46,7 @@ class OpenAIServingTokenization(OpenAIServing):
46
request_logger=request_logger,
47
)
48
49
+ self.openai_serving_render = openai_serving_render
50
self.chat_template = chat_template
51
self.chat_template_content_format: Final = chat_template_content_format
52
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
@@ -68,7 +71,7 @@ class OpenAIServingTokenization(OpenAIServing):
71
if request.tools is None
72
else [tool.model_dump() for tool in request.tools]
73
)
71
- error_check_ret = self._validate_chat_template(
74
+ error_check_ret = self.openai_serving_render.validate_chat_template(
75
request_chat_template=request.chat_template,
76
chat_template_kwargs=request.chat_template_kwargs,
77
trust_request_chat_template=self.trust_request_chat_template,
@@ -76,7 +79,7 @@ class OpenAIServingTokenization(OpenAIServing):
79
if error_check_ret is not None:
80
return error_check_ret
81
79
- _, engine_prompts = await self._preprocess_chat(
82
+ _, engine_prompts = await self.openai_serving_render.preprocess_chat(
83
request,
84
request.messages,
85
default_template=self.chat_template,
@@ -85,7 +88,7 @@ class OpenAIServingTokenization(OpenAIServing):
88
tool_dicts=tool_dicts,
89
)
90
else:
88
- engine_prompts = await self._preprocess_completion(
91
+ engine_prompts = await self.openai_serving_render.preprocess_completion(
92
request,
93
prompt_input=request.prompt,
94
prompt_embeds=None,