A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Frontend] Standardize use of `create_error_response` (#32319)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 14, 2026 at 19:22 UTC
3f28174c6a6d16dd42405016986a36f9d17e57c0
12 files changed
+62
-67
vllm/entrypoints/openai/api_server.py
+1
-7
@@ -368,14 +368,8 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
368
369
try:
370
generator = await handler.create_completion(request, raw_request)
371
- except OverflowError as e:
372
- raise HTTPException(
373
- status_code=HTTPStatus.BAD_REQUEST.value, detail=str(e)
374
- ) from e
371
except Exception as e:
376
- raise HTTPException(
377
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
378
- ) from e
372
+ return handler.create_error_response(e)
373
374
if isinstance(generator, ErrorResponse):
375
return JSONResponse(
vllm/entrypoints/openai/chat_completion/api_router.py
+4
-4
@@ -4,7 +4,7 @@
4
5
from http import HTTPStatus
6
7
-from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
7
+from fastapi import APIRouter, Depends, FastAPI, Request
8
from fastapi.responses import JSONResponse, StreamingResponse
9
10
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -53,12 +53,12 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
53
return base_server.create_error_response(
54
message="The model does not support Chat Completions API"
55
)
56
+
57
try:
58
generator = await handler.create_chat_completion(request, raw_request)
59
except Exception as e:
59
- raise HTTPException(
60
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
61
- ) from e
60
+ return handler.create_error_response(e)
61
+
62
if isinstance(generator, ErrorResponse):
63
return JSONResponse(
64
content=generator.model_dump(), status_code=generator.error.code
vllm/entrypoints/openai/engine/serving.py
+8
-3
@@ -94,7 +94,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
94
TokenizeCompletionRequest,
95
TokenizeResponse,
96
)
97
-from vllm.entrypoints.utils import _validate_truncation_size
97
+from vllm.entrypoints.utils import _validate_truncation_size, sanitize_message
98
from vllm.inputs.data import PromptType, TokensPrompt
99
from vllm.inputs.parse import (
100
PromptComponents,
@@ -768,11 +768,15 @@ class OpenAIServing:
768
err_type = "BadRequestError"
769
status_code = HTTPStatus.BAD_REQUEST
770
param = exc.parameter
771
- elif isinstance(exc, (ValueError, TypeError, RuntimeError)):
771
+ elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
772
# Common validation errors from user input
773
err_type = "BadRequestError"
774
status_code = HTTPStatus.BAD_REQUEST
775
param = None
776
+ elif isinstance(exc, NotImplementedError):
777
+ err_type = "NotImplementedError"
778
+ status_code = HTTPStatus.NOT_IMPLEMENTED
779
+ param = None
780
elif exc.__class__.__name__ == "TemplateError":
781
# jinja2.TemplateError (avoid importing jinja2)
782
err_type = "BadRequestError"
@@ -791,9 +795,10 @@ class OpenAIServing:
795
traceback.print_exc()
796
else:
797
traceback.print_stack()
798
+
799
return ErrorResponse(
800
error=ErrorInfo(
796
- message=message,
801
+ message=sanitize_message(message),
802
type=err_type,
803
code=status_code.value,
804
param=param,
vllm/entrypoints/openai/responses/api_router.py
+4
-10
@@ -5,7 +5,7 @@
5
from collections.abc import AsyncGenerator
6
from http import HTTPStatus
7
8
-from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
8
+from fastapi import APIRouter, Depends, FastAPI, Request
9
from fastapi.responses import JSONResponse, StreamingResponse
10
11
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
@@ -64,9 +64,7 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
64
try:
65
generator = await handler.create_responses(request, raw_request)
66
except Exception as e:
67
- raise HTTPException(
68
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
69
- ) from e
67
+ return handler.create_error_response(e)
68
69
if isinstance(generator, ErrorResponse):
70
return JSONResponse(
@@ -101,9 +99,7 @@ async def retrieve_responses(
99
stream=stream,
100
)
101
except Exception as e:
104
- raise HTTPException(
105
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
106
- ) from e
102
+ return handler.create_error_response(e)
103
104
if isinstance(response, ErrorResponse):
105
return JSONResponse(
@@ -128,9 +124,7 @@ async def cancel_responses(response_id: str, raw_request: Request):
124
try:
125
response = await handler.cancel_responses(response_id)
126
except Exception as e:
131
- raise HTTPException(
132
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
133
- ) from e
127
+ return handler.create_error_response(e)
128
129
if isinstance(response, ErrorResponse):
130
return JSONResponse(
vllm/entrypoints/openai/serving_models.py
+6
-1
@@ -18,6 +18,7 @@ from vllm.entrypoints.serve.lora.protocol import (
18
LoadLoRAAdapterRequest,
19
UnloadLoRAAdapterRequest,
20
)
21
+from vllm.entrypoints.utils import sanitize_message
22
from vllm.logger import init_logger
23
from vllm.lora.request import LoRARequest
24
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
@@ -302,5 +303,9 @@ def create_error_response(
303
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
304
) -> ErrorResponse:
305
return ErrorResponse(
305
- error=ErrorInfo(message=message, type=err_type, code=status_code.value)
306
+ error=ErrorInfo(
307
+ message=sanitize_message(message),
308
+ type=err_type,
309
+ code=status_code.value,
310
+ )
311
)
vllm/entrypoints/pooling/classify/api_router.py
+3
-5
@@ -1,8 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from http import HTTPStatus
3
5
-from fastapi import APIRouter, Depends, HTTPException, Request
4
+from fastapi import APIRouter, Depends, Request
5
from starlette.responses import JSONResponse
6
from typing_extensions import assert_never
7
@@ -36,9 +35,8 @@ async def create_classify(request: ClassificationRequest, raw_request: Request):
35
try:
36
generator = await handler.create_classify(request, raw_request)
37
except Exception as e:
39
- raise HTTPException(
40
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
41
- ) from e
38
+ return handler.create_error_response(e)
39
+
40
if isinstance(generator, ErrorResponse):
41
return JSONResponse(
42
content=generator.model_dump(), status_code=generator.error.code
vllm/entrypoints/pooling/embed/api_router.py
+2
-4
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from http import HTTPStatus
4
5
-from fastapi import APIRouter, Depends, HTTPException, Request
5
+from fastapi import APIRouter, Depends, Request
6
from fastapi.responses import JSONResponse, StreamingResponse
7
from typing_extensions import assert_never
8
@@ -47,9 +47,7 @@ async def create_embedding(
47
try:
48
generator = await handler.create_embedding(request, raw_request)
49
except Exception as e:
50
- raise HTTPException(
51
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
52
- ) from e
50
+ return handler.create_error_response(e)
51
52
if isinstance(generator, ErrorResponse):
53
return JSONResponse(
vllm/entrypoints/pooling/pooling/api_router.py
+3
-4
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from http import HTTPStatus
4
5
-from fastapi import APIRouter, Depends, HTTPException, Request
5
+from fastapi import APIRouter, Depends, Request
6
from fastapi.responses import JSONResponse, StreamingResponse
7
from typing_extensions import assert_never
8
@@ -44,9 +44,8 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
44
try:
45
generator = await handler.create_pooling(request, raw_request)
46
except Exception as e:
47
- raise HTTPException(
48
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
49
- ) from e
47
+ return handler.create_error_response(e)
48
+
49
if isinstance(generator, ErrorResponse):
50
return JSONResponse(
51
content=generator.model_dump(), status_code=generator.error.code
vllm/entrypoints/pooling/score/api_router.py
+5
-7
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from http import HTTPStatus
4
5
-from fastapi import APIRouter, Depends, HTTPException, Request
5
+from fastapi import APIRouter, Depends, Request
6
from fastapi.responses import JSONResponse
7
from typing_extensions import assert_never
8
@@ -52,9 +52,8 @@ async def create_score(request: ScoreRequest, raw_request: Request):
52
try:
53
generator = await handler.create_score(request, raw_request)
54
except Exception as e:
55
- raise HTTPException(
56
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
57
- ) from e
55
+ return handler.create_error_response(e)
56
+
57
if isinstance(generator, ErrorResponse):
58
return JSONResponse(
59
content=generator.model_dump(), status_code=generator.error.code
@@ -104,9 +103,8 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
103
try:
104
generator = await handler.do_rerank(request, raw_request)
105
except Exception as e:
107
- raise HTTPException(
108
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
109
- ) from e
106
+ return handler.create_error_response(e)
107
+
108
if isinstance(generator, ErrorResponse):
109
return JSONResponse(
110
content=generator.model_dump(), status_code=generator.error.code
vllm/entrypoints/serve/disagg/api_router.py
+2
-3
@@ -67,9 +67,8 @@ async def generate(request: GenerateRequest, raw_request: Request):
67
try:
68
generator = await handler.serve_tokens(request, raw_request)
69
except Exception as e:
70
- raise HTTPException(
71
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
72
- ) from e
70
+ return handler.create_error_response(e)
71
+
72
if isinstance(generator, ErrorResponse):
73
return JSONResponse(
74
content=generator.model_dump(), status_code=generator.error.code
vllm/entrypoints/serve/tokenize/api_router.py
+1
-7
@@ -51,14 +51,8 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
51
52
try:
53
generator = await handler.create_tokenize(request, raw_request)
54
- except NotImplementedError as e:
55
- raise HTTPException(
56
- status_code=HTTPStatus.NOT_IMPLEMENTED.value, detail=str(e)
57
- ) from e
54
except Exception as e:
59
- raise HTTPException(
60
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
61
- ) from e
55
+ return handler.create_error_response(e)
56
57
if isinstance(generator, ErrorResponse):
58
return JSONResponse(
vllm/entrypoints/utils.py
+23
-12
@@ -7,7 +7,7 @@ import functools
7
import os
8
from argparse import Namespace
9
from pathlib import Path
10
-from typing import Any
10
+from typing import TYPE_CHECKING, Any
11
12
import regex as re
13
from fastapi import Request
@@ -22,20 +22,27 @@ from vllm.entrypoints.chat_utils import (
22
resolve_hf_chat_template,
23
resolve_mistral_chat_template,
24
)
25
-from vllm.entrypoints.openai.chat_completion.protocol import (
26
- ChatCompletionRequest,
27
-)
28
-from vllm.entrypoints.openai.cli_args import make_arg_parser
29
-from vllm.entrypoints.openai.engine.protocol import (
30
- CompletionRequest,
31
- StreamOptions,
32
-)
33
-from vllm.entrypoints.openai.serving_models import LoRAModulePath
25
from vllm.logger import init_logger
26
from vllm.platforms import current_platform
27
from vllm.tokenizers.mistral import MistralTokenizer
28
from vllm.utils.argparse_utils import FlexibleArgumentParser
29
30
+if TYPE_CHECKING:
31
+ from vllm.entrypoints.openai.chat_completion.protocol import (
32
+ ChatCompletionRequest,
33
+ )
34
+ from vllm.entrypoints.openai.engine.protocol import (
35
+ CompletionRequest,
36
+ StreamOptions,
37
+ )
38
+ from vllm.entrypoints.openai.serving_models import LoRAModulePath
39
+else:
40
+ ChatCompletionRequest = object
41
+ CompletionRequest = object
42
+ StreamOptions = object
43
+ LoRAModulePath = object
44
+
45
+
46
logger = init_logger(__name__)
47
48
VLLM_SUBCMD_PARSER_EPILOG = (
@@ -208,7 +215,7 @@ def _validate_truncation_size(
215
216
def get_max_tokens(
217
max_model_len: int,
211
- request: ChatCompletionRequest | CompletionRequest,
218
+ request: "ChatCompletionRequest | CompletionRequest",
219
input_length: int,
220
default_sampling_params: dict,
221
) -> int:
@@ -229,6 +236,8 @@ def get_max_tokens(
236
237
238
def log_non_default_args(args: Namespace | EngineArgs):
239
+ from vllm.entrypoints.openai.cli_args import make_arg_parser
240
+
241
non_default_args = {}
242
243
# Handle Namespace
@@ -257,7 +266,7 @@ def log_non_default_args(args: Namespace | EngineArgs):
266
267
268
def should_include_usage(
260
- stream_options: StreamOptions | None, enable_force_include_usage: bool
269
+ stream_options: "StreamOptions | None", enable_force_include_usage: bool
270
) -> tuple[bool, bool]:
271
if stream_options:
272
include_usage = stream_options.include_usage or enable_force_include_usage
@@ -272,6 +281,8 @@ def should_include_usage(
281
def process_lora_modules(
282
args_lora_modules: list[LoRAModulePath], default_mm_loras: dict[str, str] | None
283
) -> list[LoRAModulePath]:
284
+ from vllm.entrypoints.openai.serving_models import LoRAModulePath
285
+
286
lora_modules = args_lora_modules
287
if default_mm_loras:
288
default_mm_lora_paths = [