A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 23:00 UTC
[openapi server] log exception in exception handler(2/N) (#36201)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Mar 11, 2026 at 11:16 UTC
fe714dd5071d1e1f829ecfe4ee10d0d7e6144b5f
16 files changed
+63
-113
tests/entrypoints/openai/test_lora_adapters.py
+3
-3
@@ -196,7 +196,7 @@ async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
196
invalid_files.mkdir()
197
(invalid_files / "adapter_config.json").write_text("this is not json")
198
199
- with pytest.raises(openai.BadRequestError):
199
+ with pytest.raises(openai.InternalServerError):
200
await client.post(
201
"load_lora_adapter",
202
cast_to=str,
@@ -232,7 +232,7 @@ async def test_dynamic_lora_badrequests(
232
json.dump(adapter_config, f)
233
234
# Test loading the adapter
235
- with pytest.raises(openai.BadRequestError, match=expected_error):
235
+ with pytest.raises(openai.InternalServerError, match=expected_error):
236
await client.post(
237
"load_lora_adapter",
238
cast_to=str,
@@ -312,7 +312,7 @@ async def test_loading_invalid_adapters_does_not_break_others(
312
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
313
)
314
for _ in range(25):
315
- with suppress(openai.BadRequestError):
315
+ with suppress(openai.InternalServerError):
316
await client.post(
317
"load_lora_adapter",
318
cast_to=str,
tests/entrypoints/sagemaker/test_sagemaker_lora_adapters.py
+1
-1
@@ -88,7 +88,7 @@ async def test_sagemaker_load_adapter_invalid_files(
88
basic_server_with_lora.url_for("adapters"),
89
json={"name": "invalid-adapter", "src": str(invalid_files)},
90
)
91
- assert load_response.status_code == 400
91
+ assert load_response.status_code == 500
92
93
94
@pytest.mark.asyncio
vllm/entrypoints/anthropic/api_router.py
+2
-2
@@ -62,7 +62,7 @@ async def create_messages(request: AnthropicMessagesRequest, raw_request: Reques
62
if handler is None:
63
base_server = raw_request.app.state.openai_serving_tokenization
64
error = base_server.create_error_response(
65
- message="The model does not support Messages API"
65
+ NotImplementedError("The model does not support Messages API")
66
)
67
return translate_error_response(error)
68
@@ -108,7 +108,7 @@ async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Reques
108
if handler is None:
109
base_server = raw_request.app.state.openai_serving_tokenization
110
error = base_server.create_error_response(
111
- message="The model does not support Messages API"
111
+ NotImplementedError("The model does not support Messages API")
112
)
113
return translate_error_response(error)
114
vllm/entrypoints/openai/chat_completion/api_router.py
+1
-4
@@ -50,10 +50,7 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
50
)
51
handler = chat(raw_request)
52
if handler is None:
53
- base_server = raw_request.app.state.openai_serving_tokenization
54
- return base_server.create_error_response(
55
- message="The model does not support Chat Completions API"
56
- )
53
+ raise NotImplementedError("The model does not support Chat Completions API")
54
55
generator = await handler.create_chat_completion(request, raw_request)
56
vllm/entrypoints/openai/completion/api_router.py
+1
-4
@@ -49,10 +49,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
49
)
50
handler = completion(raw_request)
51
if handler is None:
52
- base_server = raw_request.app.state.openai_serving_tokenization
53
- return base_server.create_error_response(
54
- message="The model does not support Completions API"
55
- )
52
+ raise NotImplementedError("The model does not support Completions API")
53
54
generator = await handler.create_completion(request, raw_request)
55
vllm/entrypoints/openai/models/serving.py
+11
-25
@@ -7,7 +7,6 @@ from http import HTTPStatus
7
8
from vllm.engine.protocol import EngineClient
9
from vllm.entrypoints.openai.engine.protocol import (
10
- ErrorInfo,
10
ErrorResponse,
11
ModelCard,
12
ModelList,
@@ -18,7 +17,8 @@ from vllm.entrypoints.serve.lora.protocol import (
17
LoadLoRAAdapterRequest,
18
UnloadLoRAAdapterRequest,
19
)
21
-from vllm.entrypoints.utils import sanitize_message
20
+from vllm.entrypoints.utils import create_error_response
21
+from vllm.exceptions import LoRAAdapterNotFoundError
22
from vllm.logger import init_logger
23
from vllm.lora.request import LoRARequest
24
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
@@ -152,15 +152,15 @@ class OpenAIServingModels:
152
try:
153
await self.engine_client.add_lora(lora_request)
154
except Exception as e:
155
- error_type = "BadRequestError"
156
- status_code = HTTPStatus.BAD_REQUEST
157
- if "No adapter found" in str(e):
158
- error_type = "NotFoundError"
159
- status_code = HTTPStatus.NOT_FOUND
160
-
161
- return create_error_response(
162
- message=str(e), err_type=error_type, status_code=status_code
163
- )
155
+ if str(
156
+ LoRAAdapterNotFoundError(
157
+ lora_request.lora_name, lora_request.lora_path
158
+ )
159
+ ) in str(e):
160
+ raise LoRAAdapterNotFoundError(
161
+ lora_request.lora_name, lora_request.lora_path
162
+ ) from e
163
+ raise
164
165
self.lora_requests[lora_name] = lora_request
166
logger.info(
@@ -292,17 +292,3 @@ class OpenAIServingModels:
292
err_type="NotFoundError",
293
status_code=HTTPStatus.NOT_FOUND,
294
)
295
-
296
-
297
-def create_error_response(
298
- message: str,
299
- err_type: str = "BadRequestError",
300
- status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
301
-) -> ErrorResponse:
302
- return ErrorResponse(
303
- error=ErrorInfo(
304
- message=sanitize_message(message),
305
- type=err_type,
306
- code=status_code.value,
307
- )
308
- )
vllm/entrypoints/openai/responses/api_router.py
+3
-12
@@ -59,10 +59,7 @@ async def _convert_stream_to_sse_events(
59
async def create_responses(request: ResponsesRequest, raw_request: Request):
60
handler = responses(raw_request)
61
if handler is None:
62
- base_server = raw_request.app.state.openai_serving_tokenization
63
- return base_server.create_error_response(
64
- message="The model does not support Responses API"
65
- )
62
+ raise NotImplementedError("The model does not support Responses API")
63
64
generator = await handler.create_responses(request, raw_request)
65
@@ -88,10 +85,7 @@ async def retrieve_responses(
85
):
86
handler = responses(raw_request)
87
if handler is None:
91
- base_server = raw_request.app.state.openai_serving_tokenization
92
- return base_server.create_error_response(
93
- message="The model does not support Responses API"
94
- )
88
+ raise NotImplementedError("The model does not support Responses API")
89
90
response = await handler.retrieve_responses(
91
response_id,
@@ -115,10 +109,7 @@ async def retrieve_responses(
109
async def cancel_responses(response_id: str, raw_request: Request):
110
handler = responses(raw_request)
111
if handler is None:
118
- base_server = raw_request.app.state.openai_serving_tokenization
119
- return base_server.create_error_response(
120
- message="The model does not support Responses API"
121
- )
112
+ raise NotImplementedError("The model does not support Responses API")
113
114
response = await handler.cancel_responses(response_id)
115
vllm/entrypoints/openai/speech_to_text/api_router.py
+2
-8
@@ -65,10 +65,7 @@ async def create_transcriptions(
65
):
66
handler = transcription(raw_request)
67
if handler is None:
68
- base_server = raw_request.app.state.openai_serving_tokenization
69
- return base_server.create_error_response(
70
- message="The model does not support Transcriptions API"
71
- )
68
+ raise NotImplementedError("The model does not support Transcriptions API")
69
70
audio_data = await request.file.read()
71
@@ -101,10 +98,7 @@ async def create_translations(
98
):
99
handler = translation(raw_request)
100
if handler is None:
104
- base_server = raw_request.app.state.openai_serving_tokenization
105
- return base_server.create_error_response(
106
- message="The model does not support Translations API"
107
- )
101
+ raise NotImplementedError("The model does not support Translations API")
102
103
audio_data = await request.file.read()
104
vllm/entrypoints/pooling/classify/api_router.py
+2
-9
@@ -2,13 +2,12 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from fastapi import APIRouter, Depends, Request
5
-from fastapi.responses import JSONResponse, Response
5
+from fastapi.responses import Response
6
7
from vllm.entrypoints.openai.utils import validate_json_request
8
from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
9
from vllm.entrypoints.pooling.classify.serving import ServingClassification
10
from vllm.entrypoints.utils import (
11
- create_error_response,
11
load_aware_call,
12
with_cancellation,
13
)
@@ -28,12 +27,6 @@ async def create_classify(
27
) -> Response:
28
handler = classify(raw_request)
29
if handler is None:
31
- error_response = create_error_response(
32
- message="The model does not support Classification API"
33
- )
34
- return JSONResponse(
35
- content=error_response.model_dump(),
36
- status_code=error_response.error.code,
37
- )
30
+ raise NotImplementedError("The model does not support Classification API")
31
32
return await handler(request, raw_request)
vllm/entrypoints/pooling/embed/api_router.py
+2
-9
@@ -4,14 +4,12 @@
4
from http import HTTPStatus
5
6
from fastapi import APIRouter, Depends, Request
7
-from fastapi.responses import JSONResponse
7
8
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
9
from vllm.entrypoints.openai.utils import validate_json_request
10
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest
11
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding
12
from vllm.entrypoints.utils import (
14
- create_error_response,
13
load_aware_call,
14
with_cancellation,
15
)
@@ -39,11 +37,6 @@ async def create_embedding(
37
):
38
handler = embedding(raw_request)
39
if handler is None:
42
- error_response = create_error_response(
43
- message="The model does not support Embeddings API"
44
- )
45
- return JSONResponse(
46
- content=error_response.model_dump(),
47
- status_code=error_response.error.code,
48
- )
40
+ raise NotImplementedError("The model does not support Embeddings API")
41
+
42
return await handler(request, raw_request)
vllm/entrypoints/pooling/pooling/api_router.py
+1
-4
@@ -37,10 +37,7 @@ def pooling(request: Request) -> OpenAIServingPooling | None:
37
async def create_pooling(request: PoolingRequest, raw_request: Request):
38
handler = pooling(raw_request)
39
if handler is None:
40
- base_server = raw_request.app.state.openai_serving_tokenization
41
- return base_server.create_error_response(
42
- message="The model does not support Pooling API"
43
- )
40
+ raise NotImplementedError("The model does not support Pooling API")
41
42
generator = await handler.create_pooling(request, raw_request)
43
vllm/entrypoints/pooling/score/api_router.py
+2
-8
@@ -44,10 +44,7 @@ def rerank(request: Request) -> ServingScores | None:
44
async def create_score(request: ScoreRequest, raw_request: Request):
45
handler = score(raw_request)
46
if handler is None:
47
- base_server = raw_request.app.state.openai_serving_tokenization
48
- return base_server.create_error_response(
49
- message="The model does not support Score API"
50
- )
47
+ raise NotImplementedError("The model does not support Score API")
48
49
generator = await handler.create_score(request, raw_request)
50
@@ -93,10 +90,7 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
90
async def do_rerank(request: RerankRequest, raw_request: Request):
91
handler = rerank(raw_request)
92
if handler is None:
96
- base_server = raw_request.app.state.openai_serving_tokenization
97
- return base_server.create_error_response(
98
- message="The model does not support Rerank (Score) API"
99
- )
93
+ raise NotImplementedError("The model does not support Rerank (Score) API")
94
95
generator = await handler.do_rerank(request, raw_request)
96
vllm/entrypoints/serve/disagg/api_router.py
+1
-3
@@ -61,9 +61,7 @@ router = APIRouter()
61
async def generate(request: GenerateRequest, raw_request: Request):
62
handler = generate_tokens(raw_request)
63
if handler is None:
64
- return tokenization(raw_request).create_error_response(
65
- message="The model does not support generate tokens API"
66
- )
64
+ raise NotImplementedError("The model does not support generate tokens API")
65
66
generator = await handler.serve_tokens(request, raw_request)
67
vllm/entrypoints/serve/render/api_router.py
+3
-16
@@ -10,7 +10,6 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest
10
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11
from vllm.entrypoints.openai.utils import validate_json_request
12
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
13
-from vllm.entrypoints.utils import create_error_response
13
from vllm.logger import init_logger
14
15
logger = init_logger(__name__)
@@ -36,13 +35,8 @@ def render(request: Request) -> OpenAIServingRender | None:
35
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
36
handler = render(raw_request)
37
if handler is None:
39
- error = create_error_response(
40
- message="The model does not support Chat Completions Render API",
41
- err_type="NotFoundError",
42
- status_code=HTTPStatus.NOT_FOUND,
43
- )
44
- return JSONResponse(
45
- status_code=HTTPStatus.NOT_FOUND, content=error.model_dump()
38
+ raise NotImplementedError(
39
+ "The model does not support Chat Completions Render API"
40
)
41
42
result = await handler.render_chat_request(request)
@@ -66,14 +60,7 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re
60
async def render_completion(request: CompletionRequest, raw_request: Request):
61
handler = render(raw_request)
62
if handler is None:
69
- error = create_error_response(
70
- message="The model does not support Completions Render API",
71
- err_type="NotFoundError",
72
- status_code=HTTPStatus.NOT_FOUND,
73
- )
74
- return JSONResponse(
75
- status_code=HTTPStatus.NOT_FOUND, content=error.model_dump()
76
- )
63
+ raise NotImplementedError("The model does not support Completions Render API")
64
65
result = await handler.render_completion_request(request)
66
vllm/exceptions.py
+25
-1
@@ -36,7 +36,31 @@ class VLLMValidationError(ValueError):
36
return f"{base} ({', '.join(extras)})" if extras else base
37
38
39
-class VLLMNotFoundError(ValueError):
39
+class VLLMNotFoundError(Exception):
40
"""vLLM-specific NotFoundError"""
41
42
pass
43
+
44
+
45
+class LoRAAdapterNotFoundError(VLLMNotFoundError):
46
+ """Exception raised when a LoRA adapter is not found.
47
+
48
+ This exception is thrown when a requested LoRA adapter does not exist
49
+ in the system.
50
+
51
+ Attributes:
52
+ message: The error message string describing the exception
53
+ """
54
+
55
+ message: str
56
+
57
+ def __init__(
58
+ self,
59
+ lora_name: str,
60
+ lora_path: str,
61
+ ) -> None:
62
+ message = f"Loading lora {lora_name} failed: No adapter found for {lora_path}"
63
+ self.message = message
64
+
65
+ def __str__(self):
66
+ return self.message
vllm/lora/worker_manager.py
+3
-4
@@ -7,6 +7,7 @@ from typing import Any, Literal
7
import torch
8
9
from vllm.config import VllmConfig
10
+from vllm.exceptions import LoRAAdapterNotFoundError
11
from vllm.logger import init_logger
12
from vllm.lora.lora_model import LoRAModel
13
from vllm.lora.model_manager import (
@@ -147,12 +148,10 @@ class WorkerLoRAManager:
148
# offline mode)
149
# - No local adapter files found at `lora_request.lora_path`
150
# For NotFoundError
150
- raise ValueError(
151
- f"Loading lora {lora_request.lora_name} failed: No adapter "
152
- f"found for {lora_request.lora_path}"
151
+ raise LoRAAdapterNotFoundError(
152
+ lora_request.lora_name, lora_request.lora_path
153
) from e
154
except Exception as e:
155
- # For BadRequestError
155
raise e
156
157
return lora