A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[openapi] refactor render related openapi [3/N] (#36749)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Mar 11, 2026 at 18:14 UTC
40c0461f24b27df3c86918d30826d2a412c40e5f
1 file changed
+71
-131
vllm/entrypoints/serve/render/serving.py
+71
-131
@@ -1,12 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import sys
4
-import traceback
3
from collections.abc import Callable, Sequence
4
from http import HTTPStatus
5
from typing import Any
6
9
-import jinja2
7
from openai_harmony import Message as OpenAIMessage
8
9
from vllm.config import ModelConfig
@@ -18,7 +15,6 @@ from vllm.entrypoints.logger import RequestLogger
15
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
16
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
17
from vllm.entrypoints.openai.engine.protocol import (
21
- ErrorInfo,
18
ErrorResponse,
19
ModelCard,
20
ModelList,
@@ -30,7 +26,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
26
parse_chat_inputs_to_harmony_messages,
27
render_for_completion,
28
)
33
-from vllm.entrypoints.utils import sanitize_message
29
+from vllm.entrypoints.utils import create_error_response
30
from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
31
from vllm.logger import init_logger
32
from vllm.parser import ParserManager
@@ -102,81 +98,76 @@ class OpenAIServingRender:
98
logger.error("Error with model %s", error_check_ret)
99
return error_check_ret
100
105
- try:
106
- tokenizer = self.renderer.tokenizer
101
+ tokenizer = self.renderer.tokenizer
102
108
- tool_parser = self.tool_parser
103
+ tool_parser = self.tool_parser
104
110
- if is_mistral_tokenizer(tokenizer):
111
- # because of issues with pydantic we need to potentially
112
- # re-serialize the tool_calls field of the request
113
- # for more info: see comment in `maybe_serialize_tool_calls`
114
- _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
115
- _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
116
- _mt.validate_request_params(request)
105
+ if is_mistral_tokenizer(tokenizer):
106
+ # because of issues with pydantic we need to potentially
107
+ # re-serialize the tool_calls field of the request
108
+ # for more info: see comment in `maybe_serialize_tool_calls`
109
+ _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
110
+ _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
111
+ _mt.validate_request_params(request)
112
118
- # Check if tool parsing is unavailable (common condition)
119
- tool_parsing_unavailable = (
120
- tool_parser is None
121
- and not is_mistral_tokenizer(tokenizer)
122
- and not self.use_harmony
123
- )
124
-
125
- # Validate tool_choice when tool parsing is required but unavailable
126
- if tool_parsing_unavailable and request.tool_choice not in (
127
- None,
128
- "none",
129
- ):
130
- if request.tool_choice == "auto" and not self.enable_auto_tools:
131
- # for hf tokenizers, "auto" tools requires
132
- # --enable-auto-tool-choice and --tool-call-parser
133
- return self.create_error_response(
134
- '"auto" tool choice requires '
135
- "--enable-auto-tool-choice and --tool-call-parser to be set"
136
- )
137
- elif request.tool_choice != "auto":
138
- # "required" or named tool requires tool parser
139
- return self.create_error_response(
140
- f'tool_choice="{request.tool_choice}" requires '
141
- "--tool-call-parser to be set"
142
- )
113
+ # Check if tool parsing is unavailable (common condition)
114
+ tool_parsing_unavailable = (
115
+ tool_parser is None
116
+ and not is_mistral_tokenizer(tokenizer)
117
+ and not self.use_harmony
118
+ )
119
144
- if request.tools is None or (
145
- request.tool_choice == "none"
146
- and self.exclude_tools_when_tool_choice_none
147
- ):
148
- tool_dicts = None
149
- else:
150
- tool_dicts = [tool.model_dump() for tool in request.tools]
151
-
152
- if not self.use_harmony:
153
- # Common case.
154
- error_check_ret = self._validate_chat_template(
155
- request_chat_template=request.chat_template,
156
- chat_template_kwargs=request.chat_template_kwargs,
157
- trust_request_chat_template=self.trust_request_chat_template,
158
- )
159
- if error_check_ret is not None:
160
- return error_check_ret
161
-
162
- conversation, engine_prompts = await self._preprocess_chat(
163
- request,
164
- request.messages,
165
- default_template=self.chat_template,
166
- default_template_content_format=self.chat_template_content_format,
167
- default_template_kwargs=self.default_chat_template_kwargs,
168
- tool_dicts=tool_dicts,
169
- tool_parser=tool_parser,
120
+ # Validate tool_choice when tool parsing is required but unavailable
121
+ if tool_parsing_unavailable and request.tool_choice not in (
122
+ None,
123
+ "none",
124
+ ):
125
+ if request.tool_choice == "auto" and not self.enable_auto_tools:
126
+ # for hf tokenizers, "auto" tools requires
127
+ # --enable-auto-tool-choice and --tool-call-parser
128
+ return self.create_error_response(
129
+ '"auto" tool choice requires '
130
+ "--enable-auto-tool-choice and --tool-call-parser to be set"
131
)
171
- else:
172
- # For GPT-OSS.
173
- should_include_tools = tool_dicts is not None
174
- conversation, engine_prompts = self._make_request_with_harmony(
175
- request, should_include_tools
132
+ elif request.tool_choice != "auto":
133
+ # "required" or named tool requires tool parser
134
+ return self.create_error_response(
135
+ f'tool_choice="{request.tool_choice}" requires '
136
+ "--tool-call-parser to be set"
137
)
177
- except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
178
- logger.exception("Error in preprocessing prompt inputs")
179
- return self.create_error_response(e)
138
+
139
+ if request.tools is None or (
140
+ request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
141
+ ):
142
+ tool_dicts = None
143
+ else:
144
+ tool_dicts = [tool.model_dump() for tool in request.tools]
145
+
146
+ if not self.use_harmony:
147
+ # Common case.
148
+ error_check_ret = self._validate_chat_template(
149
+ request_chat_template=request.chat_template,
150
+ chat_template_kwargs=request.chat_template_kwargs,
151
+ trust_request_chat_template=self.trust_request_chat_template,
152
+ )
153
+ if error_check_ret is not None:
154
+ return error_check_ret
155
+
156
+ conversation, engine_prompts = await self._preprocess_chat(
157
+ request,
158
+ request.messages,
159
+ default_template=self.chat_template,
160
+ default_template_content_format=self.chat_template_content_format,
161
+ default_template_kwargs=self.default_chat_template_kwargs,
162
+ tool_dicts=tool_dicts,
163
+ tool_parser=tool_parser,
164
+ )
165
+ else:
166
+ # For GPT-OSS.
167
+ should_include_tools = tool_dicts is not None
168
+ conversation, engine_prompts = self._make_request_with_harmony(
169
+ request, should_include_tools
170
+ )
171
172
return conversation, engine_prompts
173
@@ -204,15 +195,11 @@ class OpenAIServingRender:
195
"prompt_logprobs is not compatible with prompt embeds."
196
)
197
207
- try:
208
- engine_prompts = await self._preprocess_completion(
209
- request,
210
- prompt_input=request.prompt,
211
- prompt_embeds=request.prompt_embeds,
212
- )
213
- except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
214
- logger.exception("Error in preprocessing prompt inputs")
215
- return self.create_error_response(e)
198
+ engine_prompts = await self._preprocess_completion(
199
+ request,
200
+ prompt_input=request.prompt,
201
+ prompt_embeds=request.prompt_embeds,
202
+ )
203
204
return engine_prompts
205
@@ -284,54 +271,7 @@ class OpenAIServingRender:
271
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
272
param: str | None = None,
273
) -> ErrorResponse:
287
- """Copied from OpenAIServing.create_error_response."""
288
- exc: Exception | None = None
289
-
290
- if isinstance(message, Exception):
291
- exc = message
292
-
293
- from vllm.exceptions import VLLMValidationError
294
-
295
- if isinstance(exc, VLLMValidationError):
296
- err_type = "BadRequestError"
297
- status_code = HTTPStatus.BAD_REQUEST
298
- param = exc.parameter
299
- elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
300
- # Common validation errors from user input
301
- err_type = "BadRequestError"
302
- status_code = HTTPStatus.BAD_REQUEST
303
- param = None
304
- elif isinstance(exc, NotImplementedError):
305
- err_type = "NotImplementedError"
306
- status_code = HTTPStatus.NOT_IMPLEMENTED
307
- param = None
308
- elif exc.__class__.__name__ == "TemplateError":
309
- # jinja2.TemplateError (avoid importing jinja2)
310
- err_type = "BadRequestError"
311
- status_code = HTTPStatus.BAD_REQUEST
312
- param = None
313
- else:
314
- err_type = "InternalServerError"
315
- status_code = HTTPStatus.INTERNAL_SERVER_ERROR
316
- param = None
317
-
318
- message = str(exc)
319
-
320
- if self.log_error_stack:
321
- exc_type, _, _ = sys.exc_info()
322
- if exc_type is not None:
323
- traceback.print_exc()
324
- else:
325
- traceback.print_stack()
326
-
327
- return ErrorResponse(
328
- error=ErrorInfo(
329
- message=sanitize_message(message),
330
- type=err_type,
331
- code=status_code.value,
332
- param=param,
333
- )
334
- )
274
+ return create_error_response(message, err_type, status_code, param)
275
276
def _is_model_supported(self, model_name: str) -> bool:
277
"""Simplified from OpenAIServing._is_model_supported (no LoRA support)."""