| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from fastapi import Request |
| 5 | from starlette.responses import JSONResponse |
| 6 | |
| 7 | from vllm.entrypoints.launchers.launcher import terminate_if_errored |
| 8 | from vllm.exceptions import GenerationError, VLLMError |
| 9 | from vllm.logger import init_logger |
| 10 | from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError |
| 11 | |
| 12 | from ..error_response import create_error_response |
| 13 | from .exception import exception_handler |
| 14 | |
| 15 | logger = init_logger(__name__) |
| 16 | |
| 17 | |
| 18 | async def vllm_error_handler(req: Request, exc: VLLMError): |
| 19 | """Dispatch a vLLM-specific error to the appropriate handler.""" |
| 20 | if isinstance(exc, (EngineGenerateError, EngineDeadError)): |
| 21 | return await engine_error_handler(req, exc) |
| 22 | elif isinstance(exc, GenerationError): |
| 23 | return await generation_error_handler(req, exc) |
| 24 | else: |
| 25 | return await exception_handler(req, exc) |
| 26 | |
| 27 | |
| 28 | async def engine_error_handler( |
| 29 | req: Request, exc: EngineDeadError | EngineGenerateError |
| 30 | ): |
| 31 | """VLLM V1 AsyncLLM catches exceptions and returns |
| 32 | only two types: EngineGenerateError and EngineDeadError. |
| 33 | |
| 34 | EngineGenerateError is raised by the per request generate() |
| 35 | method. This error could be request specific (and therefore |
| 36 | recoverable - e.g. if there is an error in input processing). |
| 37 | |
| 38 | EngineDeadError is raised by the background output_handler |
| 39 | method. This error is global and therefore not recoverable. |
| 40 | |
| 41 | We register these @app.exception_handlers to return nice |
| 42 | responses to the end user if they occur and shut down if needed. |
| 43 | See https://fastapi.tiangolo.com/tutorial/handling-errors/ |
| 44 | for more details on how exception handlers work. |
| 45 | |
| 46 | If an exception is encountered in a StreamingResponse |
| 47 | generator, the exception is not raised, since we already sent |
| 48 | a 200 status. Rather, we send an error message as the next chunk. |
| 49 | Since the exception is not raised, this means that the server |
| 50 | will not automatically shut down. Instead, we use the watchdog |
| 51 | background task for check for errored state. |
| 52 | """ |
| 53 | if req.app.state.args.log_error_stack: |
| 54 | logger.exception( |
| 55 | "Engine Exception caught. Request id: %s", |
| 56 | req.state.request_metadata.request_id |
| 57 | if hasattr(req.state, "request_metadata") |
| 58 | else None, |
| 59 | ) |
| 60 | |
| 61 | terminate_if_errored( |
| 62 | server=req.app.state.server, |
| 63 | engine=req.app.state.engine_client, |
| 64 | ) |
| 65 | err = create_error_response(exc) |
| 66 | return JSONResponse(err.model_dump(), status_code=err.error.code) |
| 67 | |
| 68 | |
| 69 | async def generation_error_handler(req: Request, exc: GenerationError): |
| 70 | """Handle GenerationError without logging stack traces. |
| 71 | |
| 72 | GenerationError is a known, expected error (e.g. KV cache load failure) |
| 73 | that should be returned to the client without polluting server logs with |
| 74 | stack traces. |
| 75 | """ |
| 76 | err = create_error_response(exc) |
| 77 | return JSONResponse(err.model_dump(), status_code=err.error.code) |