[Frontend] Consolidate entrypoint exception handler (#52261)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Aug 14, 2026 at 17:27 UTC
b8165e5e58ee25a9f5183e692b6259d37c3873ad
31 files changed
+418
-343
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+3
-1
@@ -44,7 +44,9 @@ from vllm.entrypoints.openai.engine.protocol import (
44
PromptTokenUsageInfo,
45
UsageInfo,
46
)
47
-from vllm.entrypoints.serve.utils.server_utils import validation_exception_handler
47
+from vllm.entrypoints.serve.exception_handling.handlers.validation import (
48
+ validation_exception_handler,
49
+)
50
51
_convert = AnthropicServingMessages._convert_anthropic_to_openai_request
52
_img_url = AnthropicServingMessages._convert_image_source_to_url
tests/entrypoints/cohere/test_api_router.py
+3
-1
@@ -30,8 +30,10 @@ from vllm.entrypoints.cohere.protocol import (
30
CohereChatV2Response,
31
)
32
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
33
-from vllm.entrypoints.serve.utils.server_utils import (
33
+from vllm.entrypoints.serve.exception_handling.handlers.http import (
34
http_exception_handler,
35
+)
36
+from vllm.entrypoints.serve.exception_handling.handlers.validation import (
37
validation_exception_handler,
38
)
39
tests/entrypoints/serve/exception_handling/__init__.py
tests/entrypoints/serve/exception_handling/test_error_sanitization.py
renamed
+54
-1
@@ -10,7 +10,60 @@ speech-to-text WebSocket paths.
10
11
import pytest
12
13
-from vllm.entrypoints.serve.utils.api_utils import sanitize_message
13
+from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
14
+
15
+
16
+def test_sanitize_message():
17
+ assert (
18
+ sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
19
+ == "<_io.BytesIO object>"
20
+ )
21
+
22
+
23
+class TestSanitizeMessageFilePaths:
24
+ """sanitize_message should also strip file paths and traceback
25
+ frames, not just memory addresses - see #31683."""
26
+
27
+ def test_strips_traceback_style_frame(self):
28
+ msg = (
29
+ "1 validation error:\n"
30
+ " {'type': 'list_type', 'loc': ('body', 'messages')}\n"
31
+ '\n File "/usr/local/lib/python3.12/dist-packages/vllm/'
32
+ 'entrypoints/serve/utils/api_utils.py", line 40, '
33
+ "in create_chat_completion\n"
34
+ " POST /v1/chat/completions"
35
+ )
36
+ result = sanitize_message(msg)
37
+ assert "/usr/local/" not in result
38
+ assert "api_utils.py" not in result
39
+ assert "list_type" in result
40
+
41
+ def test_strips_arbitrary_absolute_path(self):
42
+ result = sanitize_message("Error in /home/user/project/vllm/server.py")
43
+ assert "/home/user" not in result
44
+
45
+ def test_strips_single_parent_container_path(self):
46
+ """Regression: /app/server.py and /workspace/server.py (common in
47
+ container deployments) were missed by the original {2,} quantifier."""
48
+ assert "/app/" not in sanitize_message("Error in /app/server.py")
49
+ assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
50
+
51
+ def test_preserves_api_endpoint_paths(self):
52
+ msg = "POST /v1/chat/completions failed"
53
+ assert "/v1/chat/completions" in sanitize_message(msg)
54
+
55
+ def test_preserves_short_field_references(self):
56
+ msg = "Invalid value for field 'body.messages'"
57
+ assert sanitize_message(msg) == msg
58
+
59
+ def test_strips_both_address_and_path(self):
60
+ msg = (
61
+ "<Request at 0x7f123> failed at "
62
+ "/usr/local/lib/python3.12/dist-packages/vllm/server.py"
63
+ )
64
+ result = sanitize_message(msg)
65
+ assert "0x" not in result
66
+ assert "/usr/local/" not in result
67
68
69
class TestSanitizeMessageCoversLeakPatterns:
tests/entrypoints/serve/exception_handling/test_http_status_metrics.py
renamed
tests/entrypoints/serve/exception_handling/test_validation_exception_handler.py
renamed
+1
-1
@@ -16,7 +16,7 @@ from types import SimpleNamespace
16
import pytest
17
from fastapi.exceptions import RequestValidationError
18
19
-from vllm.entrypoints.serve.utils.server_utils import (
19
+from vllm.entrypoints.serve.exception_handling.handlers.validation import (
20
clean_loc_for_param,
21
validation_exception_handler,
22
)
tests/entrypoints/serve/utils/test_api_utils.py
-54
@@ -6,18 +6,10 @@ import pytest
6
from vllm.entrypoints.openai.engine.protocol import StreamOptions
7
from vllm.entrypoints.serve.utils.api_utils import (
8
get_max_tokens,
9
- sanitize_message,
9
should_include_usage,
10
)
11
12
14
-def test_sanitize_message():
15
- assert (
16
- sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
17
- == "<_io.BytesIO object>"
18
- )
19
-
20
-
13
@pytest.mark.parametrize(
14
("stream_options", "expected"),
15
[
@@ -118,49 +110,3 @@ class TestGetMaxTokens:
110
input_length=150,
111
default_sampling_params={"max_tokens": 2048},
112
)
121
-
122
-
123
-class TestSanitizeMessageFilePaths:
124
- """sanitize_message should also strip file paths and traceback
125
- frames, not just memory addresses - see #31683."""
126
-
127
- def test_strips_traceback_style_frame(self):
128
- msg = (
129
- "1 validation error:\n"
130
- " {'type': 'list_type', 'loc': ('body', 'messages')}\n"
131
- '\n File "/usr/local/lib/python3.12/dist-packages/vllm/'
132
- 'entrypoints/serve/utils/api_utils.py", line 40, '
133
- "in create_chat_completion\n"
134
- " POST /v1/chat/completions"
135
- )
136
- result = sanitize_message(msg)
137
- assert "/usr/local/" not in result
138
- assert "api_utils.py" not in result
139
- assert "list_type" in result
140
-
141
- def test_strips_arbitrary_absolute_path(self):
142
- result = sanitize_message("Error in /home/user/project/vllm/server.py")
143
- assert "/home/user" not in result
144
-
145
- def test_strips_single_parent_container_path(self):
146
- """Regression: /app/server.py and /workspace/server.py (common in
147
- container deployments) were missed by the original {2,} quantifier."""
148
- assert "/app/" not in sanitize_message("Error in /app/server.py")
149
- assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
150
-
151
- def test_preserves_api_endpoint_paths(self):
152
- msg = "POST /v1/chat/completions failed"
153
- assert "/v1/chat/completions" in sanitize_message(msg)
154
-
155
- def test_preserves_short_field_references(self):
156
- msg = "Invalid value for field 'body.messages'"
157
- assert sanitize_message(msg) == msg
158
-
159
- def test_strips_both_address_and_path(self):
160
- msg = (
161
- "<Request at 0x7f123> failed at "
162
- "/usr/local/lib/python3.12/dist-packages/vllm/server.py"
163
- )
164
- result = sanitize_message(msg)
165
- assert "0x" not in result
166
- assert "/usr/local/" not in result
tests/multimodal/media/test_unprocessable_entity_error.py
+1
-1
@@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
13
import aiohttp
14
import pytest
15
16
-from vllm.entrypoints.serve.utils.error_response import create_error_response
16
+from vllm.entrypoints.serve import create_error_response
17
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
18
from vllm.multimodal.media import MediaConnector
19
tests/renderers/test_process_multi_modal_uuids.py
+1
-1
@@ -8,7 +8,7 @@ import pytest
8
from vllm.assets.image import ImageAsset
9
from vllm.assets.video import VideoAsset
10
from vllm.config import CacheConfig, ModelConfig, VllmConfig
11
-from vllm.entrypoints.serve.utils.error_response import create_error_response
11
+from vllm.entrypoints.serve import create_error_response
12
from vllm.multimodal.parse import parse_mm_uuids
13
from vllm.renderers.hf import HfRenderer
14
from vllm.tokenizers.registry import cached_tokenizer_from_config
vllm/entrypoints/anthropic/api_router.py
+1
-1
@@ -17,9 +17,9 @@ from vllm.entrypoints.anthropic.protocol import (
17
)
18
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
19
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
20
+from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
21
from vllm.entrypoints.serve.utils.api_utils import (
22
load_aware_call,
22
- sanitize_message,
23
validate_json_request,
24
with_cancellation,
25
)
vllm/entrypoints/anthropic/serving.py
+1
-1
@@ -46,7 +46,7 @@ from vllm.entrypoints.openai.engine.protocol import (
46
UsageInfo,
47
)
48
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
49
-from vllm.entrypoints.serve.utils.api_utils import sanitize_message
49
+from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
50
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
51
from vllm.renderers.online_renderer import OnlineRenderer
52
vllm/entrypoints/cohere/api_router.py
+1
-1
@@ -33,9 +33,9 @@ from starlette.middleware.base import BaseHTTPMiddleware
33
34
import vllm.envs as envs
35
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
36
+from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
37
from vllm.entrypoints.serve.utils.api_utils import (
38
load_aware_call,
38
- sanitize_message,
39
validate_json_request,
40
with_cancellation,
41
)
vllm/entrypoints/cohere/serving.py
+1
-1
@@ -86,7 +86,7 @@ from vllm.entrypoints.openai.engine.protocol import (
86
StreamOptions,
87
)
88
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
89
-from vllm.entrypoints.serve.utils.api_utils import sanitize_message
89
+from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
90
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
91
from vllm.parser.abstract_parser import Parser
92
from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY
vllm/entrypoints/openai/api_server.py
+4
-29
@@ -16,8 +16,7 @@ from contextlib import asynccontextmanager
16
from typing import Any, cast
17
18
import uvloop
19
-from fastapi import FastAPI, HTTPException
20
-from fastapi.exceptions import RequestValidationError
19
+from fastapi import FastAPI
20
from fastapi.middleware.cors import CORSMiddleware
21
from starlette.datastructures import State
22
@@ -31,6 +30,7 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se
30
from vllm.entrypoints.openai.models.protocol import BaseModelPath
31
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
32
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
33
+from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
34
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
35
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
36
from vllm.entrypoints.serve.utils.api_utils import (
@@ -41,15 +41,10 @@ from vllm.entrypoints.serve.utils.api_utils import (
41
)
42
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
43
from vllm.entrypoints.serve.utils.server_utils import (
44
- exception_handler,
44
get_uvicorn_log_config,
46
- http_exception_handler,
45
lifespan,
46
log_response,
49
- validation_exception_handler,
50
- vllm_error_handler,
47
)
52
-from vllm.exceptions import VLLMError
48
from vllm.logger import init_logger
49
from vllm.reasoning import ReasoningParserManager
50
from vllm.renderers.online_derenderer import OnlineDerenderer
@@ -275,6 +270,8 @@ def build_app(
270
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
271
_attach_endpoint_plugins(app, supported_tasks)
272
273
+ init_exception_handler(app)
274
+
275
app.root_path = args.root_path
276
app.add_middleware(
277
CORSMiddleware,
@@ -284,28 +281,6 @@ def build_app(
281
allow_headers=args.allowed_headers,
282
)
283
287
- # Exception handlers are registered in four layers:
288
- # 1. framework errors raised by FastAPI/Starlette
289
- # 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
290
- # 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
291
- # 4. the raw ``Exception`` handler as a safety net
292
- # Registering specific exception types (rather than only ``Exception``)
293
- # ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus
294
- # middleware) rather than ``ServerErrorMiddleware`` (outside it), so their
295
- # status codes are recorded correctly.
296
- app.exception_handler(HTTPException)(http_exception_handler)
297
- app.exception_handler(RequestValidationError)(validation_exception_handler)
298
-
299
- app.exception_handler(VLLMError)(vllm_error_handler)
300
-
301
- # TODO(zqzten): remove these fallback handlers after migration to VLLMError
302
- app.exception_handler(ValueError)(exception_handler)
303
- app.exception_handler(TypeError)(exception_handler)
304
- app.exception_handler(OverflowError)(exception_handler)
305
- app.exception_handler(NotImplementedError)(exception_handler)
306
-
307
- app.exception_handler(Exception)(exception_handler)
308
-
284
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
285
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
286
from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
vllm/entrypoints/openai/models/serving.py
+1
-1
@@ -14,11 +14,11 @@ from vllm.entrypoints.openai.engine.protocol import (
14
ModelPermission,
15
)
16
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
17
+from vllm.entrypoints.serve import create_error_response
18
from vllm.entrypoints.serve.lora.protocol import (
19
LoadLoRAAdapterRequest,
20
UnloadLoRAAdapterRequest,
21
)
21
-from vllm.entrypoints.serve.utils.error_response import create_error_response
22
from vllm.exceptions import LoRAAdapterNotFoundError
23
from vllm.logger import init_logger
24
from vllm.lora.request import LoRARequest
vllm/entrypoints/openai/run_batch.py
+1
-1
@@ -52,7 +52,7 @@ from vllm.entrypoints.pooling.scoring.protocol import (
52
ScoreRequest,
53
ScoreResponse,
54
)
55
-from vllm.entrypoints.serve.utils.error_response import create_error_response
55
+from vllm.entrypoints.serve import create_error_response
56
from vllm.entrypoints.speech_to_text.transcription.protocol import (
57
TranscriptionRequest,
58
TranscriptionResponse,
vllm/entrypoints/serve/__init__.py
+8
@@ -5,8 +5,16 @@ from fastapi import FastAPI
5
6
from vllm.logger import init_logger
7
8
+from .exception_handling.error_response import create_error_response
9
+
10
logger = init_logger(__name__)
11
12
+__all__ = [
13
+ "create_error_response",
14
+ "register_vllm_serve_api_routers",
15
+ "register_vllm_dev_api_routers",
16
+]
17
+
18
19
def register_vllm_serve_api_routers(app: FastAPI):
20
from .instrumentator import register_instrumentator_api_routers
vllm/entrypoints/serve/engine/serving.py
+1
-1
@@ -12,8 +12,8 @@ from vllm.entrypoints.openai.models.serving import (
12
OpenAIServingModels,
13
)
14
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
15
+from vllm.entrypoints.serve import create_error_response
16
from vllm.entrypoints.serve.engine.typing import AnyRequest
16
-from vllm.entrypoints.serve.utils.error_response import create_error_response
17
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
18
from vllm.exceptions import VLLMNotFoundError
19
from vllm.inputs import EngineInput
vllm/entrypoints/serve/exception_handling/__init__.py
vllm/entrypoints/serve/exception_handling/error_response.py
renamed
+2
-1
@@ -7,9 +7,10 @@ from vllm.entrypoints.openai.engine.protocol import (
7
ErrorResponse,
8
GenerationError,
9
)
10
-from vllm.entrypoints.serve.utils.api_utils import sanitize_message
10
from vllm.logger import init_logger
11
12
+from .utils import sanitize_message
13
+
14
logger = init_logger(__name__)
15
16
vllm/entrypoints/serve/exception_handling/handlers/__init__.py
vllm/entrypoints/serve/exception_handling/handlers/exception.py
new
+24
@@ -0,0 +1,24 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from fastapi import Request
5
+from starlette.responses import JSONResponse
6
+
7
+from vllm.logger import init_logger
8
+
9
+from ..error_response import create_error_response
10
+
11
+logger = init_logger(__name__)
12
+
13
+
14
+async def exception_handler(req: Request, exc: Exception):
15
+ if req.app.state.args.log_error_stack:
16
+ logger.error(
17
+ "Exception caught. Request id: %s",
18
+ req.state.request_metadata.request_id
19
+ if hasattr(req.state, "request_metadata")
20
+ else None,
21
+ )
22
+
23
+ err = create_error_response(exc)
24
+ return JSONResponse(err.model_dump(), status_code=err.error.code)
vllm/entrypoints/serve/exception_handling/handlers/http.py
new
+31
@@ -0,0 +1,31 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from http import HTTPStatus
4
+
5
+from fastapi import HTTPException, Request
6
+from starlette.responses import JSONResponse
7
+
8
+from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
9
+from vllm.logger import init_logger
10
+
11
+from ..utils import sanitize_message
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+async def http_exception_handler(req: Request, exc: HTTPException):
17
+ if req.app.state.args.log_error_stack:
18
+ logger.exception(
19
+ "HTTPException caught. Request id: %s",
20
+ req.state.request_metadata.request_id
21
+ if hasattr(req.state, "request_metadata")
22
+ else None,
23
+ )
24
+ err = ErrorResponse(
25
+ error=ErrorInfo(
26
+ message=sanitize_message(exc.detail),
27
+ type=HTTPStatus(exc.status_code).phrase,
28
+ code=exc.status_code,
29
+ )
30
+ )
31
+ return JSONResponse(err.model_dump(), status_code=exc.status_code)
vllm/entrypoints/serve/exception_handling/handlers/validation.py
new
+138
@@ -0,0 +1,138 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from http import HTTPStatus
5
+
6
+import regex as re
7
+from fastapi import Request
8
+from fastapi.exceptions import RequestValidationError
9
+from starlette.responses import JSONResponse
10
+
11
+from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
12
+from vllm.exceptions import VLLMValidationError
13
+from vllm.logger import init_logger
14
+
15
+from ..utils import sanitize_message
16
+
17
+logger = init_logger(__name__)
18
+
19
+
20
+_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
21
+
22
+# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
23
+# not a stable public API -- it can grow when pydantic-core adds new
24
+# wrapper/validator kinds. To refresh it after a pydantic upgrade:
25
+# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
26
+# /v1/completions, /v1/chat/completions) with deliberately
27
+# malformed values for union-typed and wrapped fields (e.g. `stop`,
28
+# `prompt`), and inspect the raw `loc` tuples in the response.
29
+# 2. Any *unbracketed* segment that isn't a real field name or list
30
+# index is a new internal marker -- add it here. Bracketed/
31
+# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
32
+# are already caught structurally by _BRACKETED_INTERNAL_RE and
33
+# don't need a list entry.
34
+# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
35
+# in the pydantic-core Rust crate) is the canonical reference if
36
+# you want to check before it shows up in a live fuzz run.
37
+_INTERNAL_LOC_MARKERS = frozenset(
38
+ {
39
+ "function-wrap",
40
+ "function-after",
41
+ "function-before",
42
+ "function-plain",
43
+ "json-or-python",
44
+ "lax-or-strict",
45
+ "chain",
46
+ "default",
47
+ "nullable",
48
+ "tagged-union",
49
+ "union",
50
+ "call",
51
+ "arguments",
52
+ "is-instance",
53
+ "is-subclass",
54
+ "callable",
55
+ "str",
56
+ "int",
57
+ "float",
58
+ "bool",
59
+ "bytes",
60
+ "bytearray",
61
+ "list",
62
+ "tuple",
63
+ "dict",
64
+ "set",
65
+ "frozenset",
66
+ "complex",
67
+ "none",
68
+ "nonetype",
69
+ }
70
+)
71
+
72
+
73
+def _is_internal_loc_segment(segment: str) -> bool:
74
+ """True if `segment` is a Pydantic-internal wrapper/union-branch
75
+ marker rather than a user-meaningful field name or list index."""
76
+ if _BRACKETED_INTERNAL_RE.search(segment):
77
+ return True
78
+ return segment.lower() in _INTERNAL_LOC_MARKERS
79
+
80
+
81
+def clean_loc_for_param(loc: tuple) -> str:
82
+ """Join a Pydantic error `loc` tuple into a clean dotted `param`
83
+ path, dropping internal wrapper/union-branch markers that don't
84
+ correspond to a real field name an API consumer would recognize.
85
+
86
+ E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
87
+ -> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
88
+ """
89
+ parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
90
+ if not parts:
91
+ return ".".join(str(p) for p in loc)
92
+ return ".".join(parts)
93
+
94
+
95
+async def validation_exception_handler(req: Request, exc: RequestValidationError):
96
+ if req.app.state.args.log_error_stack:
97
+ logger.exception(
98
+ "RequestValidationError caught. Request id: %s",
99
+ req.state.request_metadata.request_id
100
+ if hasattr(req.state, "request_metadata")
101
+ else None,
102
+ )
103
+
104
+ param = None
105
+ errors = exc.errors()
106
+ for error in errors:
107
+ if "ctx" in error and "error" in error["ctx"]:
108
+ ctx_error = error["ctx"]["error"]
109
+ if isinstance(ctx_error, VLLMValidationError):
110
+ param = ctx_error.parameter
111
+ break
112
+
113
+ if param is None and errors:
114
+ first_error = errors[0]
115
+ loc = first_error.get("loc") if isinstance(first_error, dict) else None
116
+ if loc:
117
+ param = clean_loc_for_param(loc)
118
+
119
+ # Build the message from exc.errors() instead of str(exc) - str(exc)
120
+ # leaks the server's file path via FastAPI's endpoint context.
121
+ if errors:
122
+ count = len(errors)
123
+ label = "error" if count == 1 else "errors"
124
+ message = f"{count} validation {label}:\n"
125
+ message += "".join(f" {err}\n" for err in errors)
126
+ message = message.rstrip()
127
+ else:
128
+ message = "Validation error"
129
+
130
+ err = ErrorResponse(
131
+ error=ErrorInfo(
132
+ message=sanitize_message(message),
133
+ type=HTTPStatus.BAD_REQUEST.phrase,
134
+ code=HTTPStatus.BAD_REQUEST,
135
+ param=param,
136
+ )
137
+ )
138
+ return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py
new
+80
@@ -0,0 +1,80 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from fastapi import Request
5
+from starlette.responses import JSONResponse
6
+
7
+from vllm.entrypoints.launcher import terminate_if_errored
8
+from vllm.entrypoints.openai.engine.protocol import GenerationError
9
+from vllm.exceptions import VLLMError
10
+from vllm.logger import init_logger
11
+from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
12
+
13
+from ..error_response import create_error_response
14
+from .exception import exception_handler
15
+
16
+logger = init_logger(__name__)
17
+
18
+
19
+async def vllm_error_handler(req: Request, exc: VLLMError):
20
+ """Dispatch a vLLM-specific error to the appropriate handler."""
21
+ if isinstance(exc, (EngineGenerateError, EngineDeadError)):
22
+ return await engine_error_handler(req, exc)
23
+ elif isinstance(exc, GenerationError):
24
+ return await generation_error_handler(req, exc)
25
+ else:
26
+ return await exception_handler(req, exc)
27
+
28
+
29
+async def engine_error_handler(
30
+ req: Request, exc: EngineDeadError | EngineGenerateError
31
+):
32
+ """
33
+ VLLM V1 AsyncLLM catches exceptions and returns
34
+ only two types: EngineGenerateError and EngineDeadError.
35
+
36
+ EngineGenerateError is raised by the per request generate()
37
+ method. This error could be request specific (and therefore
38
+ recoverable - e.g. if there is an error in input processing).
39
+
40
+ EngineDeadError is raised by the background output_handler
41
+ method. This error is global and therefore not recoverable.
42
+
43
+ We register these @app.exception_handlers to return nice
44
+ responses to the end user if they occur and shut down if needed.
45
+ See https://fastapi.tiangolo.com/tutorial/handling-errors/
46
+ for more details on how exception handlers work.
47
+
48
+ If an exception is encountered in a StreamingResponse
49
+ generator, the exception is not raised, since we already sent
50
+ a 200 status. Rather, we send an error message as the next chunk.
51
+ Since the exception is not raised, this means that the server
52
+ will not automatically shut down. Instead, we use the watchdog
53
+ background task for check for errored state.
54
+ """
55
+
56
+ if req.app.state.args.log_error_stack:
57
+ logger.exception(
58
+ "Engine Exception caught. Request id: %s",
59
+ req.state.request_metadata.request_id
60
+ if hasattr(req.state, "request_metadata")
61
+ else None,
62
+ )
63
+
64
+ terminate_if_errored(
65
+ server=req.app.state.server,
66
+ engine=req.app.state.engine_client,
67
+ )
68
+ err = create_error_response(exc)
69
+ return JSONResponse(err.model_dump(), status_code=err.error.code)
70
+
71
+
72
+async def generation_error_handler(req: Request, exc: GenerationError):
73
+ """Handle GenerationError without logging stack traces.
74
+
75
+ GenerationError is a known, expected error (e.g. KV cache load failure)
76
+ that should be returned to the client as a 500 response without polluting
77
+ server logs with stack traces.
78
+ """
79
+ err = create_error_response(exc)
80
+ return JSONResponse(err.model_dump(), status_code=err.error.code)
vllm/entrypoints/serve/exception_handling/register.py
new
+43
@@ -0,0 +1,43 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+"""
5
+vLLM Exception handlers are registered in four layers:
6
+1. framework errors raised by FastAPI/Starlette
7
+2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
8
+3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
9
+4. the raw ``Exception`` handler as a safety net
10
+Registering specific exception types (rather than only ``Exception``)
11
+ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus
12
+middleware) rather than ``ServerErrorMiddleware`` (outside it), so their
13
+status codes are recorded correctly.
14
+"""
15
+
16
+from fastapi import FastAPI, HTTPException
17
+from fastapi.exceptions import RequestValidationError
18
+
19
+from vllm.exceptions import VLLMError
20
+
21
+from .handlers.exception import exception_handler
22
+from .handlers.http import http_exception_handler
23
+from .handlers.validation import validation_exception_handler
24
+from .handlers.vllm_error import vllm_error_handler
25
+
26
+
27
+def init_exception_handler(app: FastAPI):
28
+ # 1. framework errors raised by FastAPI/Starlette
29
+ app.exception_handler(HTTPException)(http_exception_handler)
30
+ app.exception_handler(RequestValidationError)(validation_exception_handler)
31
+
32
+ # 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
33
+ app.exception_handler(VLLMError)(vllm_error_handler)
34
+
35
+ # 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
36
+ # TODO(zqzten): remove these fallback handlers after migration to VLLMError
37
+ app.exception_handler(ValueError)(exception_handler)
38
+ app.exception_handler(TypeError)(exception_handler)
39
+ app.exception_handler(OverflowError)(exception_handler)
40
+ app.exception_handler(NotImplementedError)(exception_handler)
41
+
42
+ # 4. the raw ``Exception`` handler as a safety net
43
+ app.exception_handler(Exception)(exception_handler)
vllm/entrypoints/serve/exception_handling/utils.py
new
+15
@@ -0,0 +1,15 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import regex as re
5
+
6
+
7
+def sanitize_message(message: str) -> str:
8
+ """Strip memory addresses, tracebacks, and file paths from error messages."""
9
+ message = re.sub(r" at 0x[0-9a-f]+>", ">", message)
10
+ message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message)
11
+ message = re.sub(
12
+ r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "<path>", message
13
+ )
14
+ message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "<path>", message)
15
+ return message.strip()
vllm/entrypoints/serve/utils/api_utils.py
-12
@@ -10,7 +10,6 @@ from logging import Logger
10
from string import Template
11
from typing import Any
12
13
-import regex as re
13
from fastapi import Request
14
from fastapi.exceptions import RequestValidationError
15
from fastapi.responses import JSONResponse, StreamingResponse
@@ -309,17 +308,6 @@ def process_lora_modules(
308
return lora_modules
309
310
312
-def sanitize_message(message: str) -> str:
313
- """Strip memory addresses, tracebacks, and file paths from error messages."""
314
- message = re.sub(r" at 0x[0-9a-f]+>", ">", message)
315
- message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message)
316
- message = re.sub(
317
- r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "<path>", message
318
- )
319
- message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "<path>", message)
320
- return message.strip()
321
-
322
-
311
def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
312
if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None:
313
message = "vLLM server version %s, serving model %s"
vllm/entrypoints/serve/utils/server_utils.py
+1
-232
@@ -8,12 +8,9 @@ import uuid
8
from argparse import Namespace
9
from collections.abc import Awaitable
10
from contextlib import asynccontextmanager
11
-from http import HTTPStatus
11
12
import pydantic
14
-import regex as re
15
-from fastapi import FastAPI, HTTPException, Request
16
-from fastapi.exceptions import RequestValidationError
13
+from fastapi import FastAPI, Request
14
from fastapi.responses import JSONResponse
15
from starlette.concurrency import iterate_in_threadpool
16
from starlette.datastructures import Headers, MutableHeaders
@@ -21,20 +18,8 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
18
19
from vllm import envs
20
from vllm.engine.protocol import EngineClient
24
-from vllm.entrypoints.launcher import terminate_if_errored
25
-from vllm.entrypoints.openai.engine.protocol import (
26
- ErrorInfo,
27
- ErrorResponse,
28
- GenerationError,
29
-)
30
-from vllm.entrypoints.serve.utils.error_response import (
31
- create_error_response,
32
- sanitize_message,
33
-)
34
-from vllm.exceptions import VLLMError, VLLMValidationError
21
from vllm.logger import init_logger
22
from vllm.utils.gc_utils import freeze_gc_heap
37
-from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
23
24
logger = init_logger("vllm.entrypoints.openai.server_utils")
25
@@ -325,222 +310,6 @@ async def log_response(request: Request, call_next):
310
return response
311
312
328
-async def vllm_error_handler(req: Request, exc: VLLMError):
329
- """Dispatch a vLLM-specific error to the appropriate handler."""
330
- if isinstance(exc, (EngineGenerateError, EngineDeadError)):
331
- return await engine_error_handler(req, exc)
332
- elif isinstance(exc, GenerationError):
333
- return await generation_error_handler(req, exc)
334
- else:
335
- return await exception_handler(req, exc)
336
-
337
-
338
-async def engine_error_handler(
339
- req: Request, exc: EngineDeadError | EngineGenerateError
340
-):
341
- """
342
- VLLM V1 AsyncLLM catches exceptions and returns
343
- only two types: EngineGenerateError and EngineDeadError.
344
-
345
- EngineGenerateError is raised by the per request generate()
346
- method. This error could be request specific (and therefore
347
- recoverable - e.g. if there is an error in input processing).
348
-
349
- EngineDeadError is raised by the background output_handler
350
- method. This error is global and therefore not recoverable.
351
-
352
- We register these @app.exception_handlers to return nice
353
- responses to the end user if they occur and shut down if needed.
354
- See https://fastapi.tiangolo.com/tutorial/handling-errors/
355
- for more details on how exception handlers work.
356
-
357
- If an exception is encountered in a StreamingResponse
358
- generator, the exception is not raised, since we already sent
359
- a 200 status. Rather, we send an error message as the next chunk.
360
- Since the exception is not raised, this means that the server
361
- will not automatically shut down. Instead, we use the watchdog
362
- background task for check for errored state.
363
- """
364
-
365
- if req.app.state.args.log_error_stack:
366
- logger.exception(
367
- "Engine Exception caught. Request id: %s",
368
- req.state.request_metadata.request_id
369
- if hasattr(req.state, "request_metadata")
370
- else None,
371
- )
372
-
373
- terminate_if_errored(
374
- server=req.app.state.server,
375
- engine=req.app.state.engine_client,
376
- )
377
- err = create_error_response(exc)
378
- return JSONResponse(err.model_dump(), status_code=err.error.code)
379
-
380
-
381
-async def generation_error_handler(req: Request, exc: GenerationError):
382
- """Handle GenerationError without logging stack traces.
383
-
384
- GenerationError is a known, expected error (e.g. KV cache load failure)
385
- that should be returned to the client as a 500 response without polluting
386
- server logs with stack traces.
387
- """
388
- err = create_error_response(exc)
389
- return JSONResponse(err.model_dump(), status_code=err.error.code)
390
-
391
-
392
-async def exception_handler(req: Request, exc: Exception):
393
- if req.app.state.args.log_error_stack:
394
- logger.error(
395
- "Exception caught. Request id: %s",
396
- req.state.request_metadata.request_id
397
- if hasattr(req.state, "request_metadata")
398
- else None,
399
- )
400
-
401
- err = create_error_response(exc)
402
- return JSONResponse(err.model_dump(), status_code=err.error.code)
403
-
404
-
405
-async def http_exception_handler(req: Request, exc: HTTPException):
406
- if req.app.state.args.log_error_stack:
407
- logger.exception(
408
- "HTTPException caught. Request id: %s",
409
- req.state.request_metadata.request_id
410
- if hasattr(req.state, "request_metadata")
411
- else None,
412
- )
413
- err = ErrorResponse(
414
- error=ErrorInfo(
415
- message=sanitize_message(exc.detail),
416
- type=HTTPStatus(exc.status_code).phrase,
417
- code=exc.status_code,
418
- )
419
- )
420
- return JSONResponse(err.model_dump(), status_code=exc.status_code)
421
-
422
-
423
-_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
424
-
425
-# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
426
-# not a stable public API -- it can grow when pydantic-core adds new
427
-# wrapper/validator kinds. To refresh it after a pydantic upgrade:
428
-# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
429
-# /v1/completions, /v1/chat/completions) with deliberately
430
-# malformed values for union-typed and wrapped fields (e.g. `stop`,
431
-# `prompt`), and inspect the raw `loc` tuples in the response.
432
-# 2. Any *unbracketed* segment that isn't a real field name or list
433
-# index is a new internal marker -- add it here. Bracketed/
434
-# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
435
-# are already caught structurally by _BRACKETED_INTERNAL_RE and
436
-# don't need a list entry.
437
-# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
438
-# in the pydantic-core Rust crate) is the canonical reference if
439
-# you want to check before it shows up in a live fuzz run.
440
-_INTERNAL_LOC_MARKERS = frozenset(
441
- {
442
- "function-wrap",
443
- "function-after",
444
- "function-before",
445
- "function-plain",
446
- "json-or-python",
447
- "lax-or-strict",
448
- "chain",
449
- "default",
450
- "nullable",
451
- "tagged-union",
452
- "union",
453
- "call",
454
- "arguments",
455
- "is-instance",
456
- "is-subclass",
457
- "callable",
458
- "str",
459
- "int",
460
- "float",
461
- "bool",
462
- "bytes",
463
- "bytearray",
464
- "list",
465
- "tuple",
466
- "dict",
467
- "set",
468
- "frozenset",
469
- "complex",
470
- "none",
471
- "nonetype",
472
- }
473
-)
474
-
475
-
476
-def _is_internal_loc_segment(segment: str) -> bool:
477
- """True if `segment` is a Pydantic-internal wrapper/union-branch
478
- marker rather than a user-meaningful field name or list index."""
479
- if _BRACKETED_INTERNAL_RE.search(segment):
480
- return True
481
- return segment.lower() in _INTERNAL_LOC_MARKERS
482
-
483
-
484
-def clean_loc_for_param(loc: tuple) -> str:
485
- """Join a Pydantic error `loc` tuple into a clean dotted `param`
486
- path, dropping internal wrapper/union-branch markers that don't
487
- correspond to a real field name an API consumer would recognize.
488
-
489
- E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
490
- -> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
491
- """
492
- parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
493
- if not parts:
494
- return ".".join(str(p) for p in loc)
495
- return ".".join(parts)
496
-
497
-
498
-async def validation_exception_handler(req: Request, exc: RequestValidationError):
499
- if req.app.state.args.log_error_stack:
500
- logger.exception(
501
- "RequestValidationError caught. Request id: %s",
502
- req.state.request_metadata.request_id
503
- if hasattr(req.state, "request_metadata")
504
- else None,
505
- )
506
-
507
- param = None
508
- errors = exc.errors()
509
- for error in errors:
510
- if "ctx" in error and "error" in error["ctx"]:
511
- ctx_error = error["ctx"]["error"]
512
- if isinstance(ctx_error, VLLMValidationError):
513
- param = ctx_error.parameter
514
- break
515
-
516
- if param is None and errors:
517
- first_error = errors[0]
518
- loc = first_error.get("loc") if isinstance(first_error, dict) else None
519
- if loc:
520
- param = clean_loc_for_param(loc)
521
-
522
- # Build the message from exc.errors() instead of str(exc) - str(exc)
523
- # leaks the server's file path via FastAPI's endpoint context.
524
- if errors:
525
- count = len(errors)
526
- label = "error" if count == 1 else "errors"
527
- message = f"{count} validation {label}:\n"
528
- message += "".join(f" {err}\n" for err in errors)
529
- message = message.rstrip()
530
- else:
531
- message = "Validation error"
532
-
533
- err = ErrorResponse(
534
- error=ErrorInfo(
535
- message=sanitize_message(message),
536
- type=HTTPStatus.BAD_REQUEST.phrase,
537
- code=HTTPStatus.BAD_REQUEST,
538
- param=param,
539
- )
540
- )
541
- return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
542
-
543
-
313
_running_tasks: set[asyncio.Task] = set()
314
315
vllm/entrypoints/speech_to_text/realtime/connection.py
+1
-1
@@ -14,7 +14,7 @@ from starlette.websockets import WebSocketDisconnect
14
15
from vllm import envs
16
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
17
-from vllm.entrypoints.serve.utils.api_utils import sanitize_message
17
+from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
18
from vllm.exceptions import VLLMValidationError
19
from vllm.logger import init_logger
20
vllm/renderers/online_renderer.py
+1
-1
@@ -26,7 +26,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
26
render_for_completion,
27
)
28
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
29
-from vllm.entrypoints.serve.utils.error_response import create_error_response
29
+from vllm.entrypoints.serve import create_error_response
30
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
31
from vllm.inputs import (
32
EngineInput,