A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 15:45 UTC
[Bugfix] Add error handling for FINISHED_ERROR in OpenAIServing (#37148)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Mar 17, 2026 at 00:27 UTC
6682c231fa97f33d3b3f4d788da4e14959989a67
2 files changed
+19
-1
vllm/entrypoints/openai/api_server.py
+3
@@ -29,11 +29,13 @@ from vllm.entrypoints.chat_utils import load_chat_template
29
from vllm.entrypoints.launcher import serve_http
30
from vllm.entrypoints.logger import RequestLogger
31
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
32
+from vllm.entrypoints.openai.engine.protocol import GenerationError
33
from vllm.entrypoints.openai.models.protocol import BaseModelPath
34
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
35
from vllm.entrypoints.openai.server_utils import (
36
engine_error_handler,
37
exception_handler,
38
+ generation_error_handler,
39
get_uvicorn_log_config,
40
http_exception_handler,
41
lifespan,
@@ -263,6 +265,7 @@ def build_app(
265
app.exception_handler(RequestValidationError)(validation_exception_handler)
266
app.exception_handler(EngineGenerateError)(engine_error_handler)
267
app.exception_handler(EngineDeadError)(engine_error_handler)
268
+ app.exception_handler(GenerationError)(generation_error_handler)
269
app.exception_handler(Exception)(exception_handler)
270
271
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
vllm/entrypoints/openai/server_utils.py
+16
-1
@@ -21,7 +21,11 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
21
from vllm import envs
22
from vllm.engine.protocol import EngineClient
23
from vllm.entrypoints.launcher import terminate_if_errored
24
-from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
24
+from vllm.entrypoints.openai.engine.protocol import (
25
+ ErrorInfo,
26
+ ErrorResponse,
27
+ GenerationError,
28
+)
29
from vllm.entrypoints.utils import create_error_response, sanitize_message
30
from vllm.exceptions import VLLMValidationError
31
from vllm.logger import init_logger
@@ -354,6 +358,17 @@ async def engine_error_handler(
358
return JSONResponse(err.model_dump(), status_code=err.error.code)
359
360
361
+async def generation_error_handler(req: Request, exc: GenerationError):
362
+ """Handle GenerationError without logging stack traces.
363
+
364
+ GenerationError is a known, expected error (e.g. KV cache load failure)
365
+ that should be returned to the client as a 500 response without polluting
366
+ server logs with stack traces.
367
+ """
368
+ err = create_error_response(exc)
369
+ return JSONResponse(err.model_dump(), status_code=err.error.code)
370
+
371
+
372
async def exception_handler(req: Request, exc: Exception):
373
if req.app.state.args.log_error_stack:
374
logger.exception(