| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from http import HTTPStatus |
| 5 | |
| 6 | import regex as re |
| 7 | from fastapi import Request |
| 8 | from fastapi.exceptions import RequestValidationError |
| 9 | from starlette.responses import JSONResponse |
| 10 | |
| 11 | from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse |
| 12 | from vllm.exceptions import VLLMValidationError |
| 13 | from vllm.logger import init_logger |
| 14 | |
| 15 | from ..utils import sanitize_message |
| 16 | |
| 17 | logger = init_logger(__name__) |
| 18 | |
| 19 | |
| 20 | _BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]") |
| 21 | |
| 22 | # NOTE: this list is pydantic-core's internal schema-kind vocabulary, |
| 23 | # not a stable public API -- it can grow when pydantic-core adds new |
| 24 | # wrapper/validator kinds. To refresh it after a pydantic upgrade: |
| 25 | # 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize, |
| 26 | # /v1/completions, /v1/chat/completions) with deliberately |
| 27 | # malformed values for union-typed and wrapped fields (e.g. `stop`, |
| 28 | # `prompt`), and inspect the raw `loc` tuples in the response. |
| 29 | # 2. Any *unbracketed* segment that isn't a real field name or list |
| 30 | # index is a new internal marker -- add it here. Bracketed/ |
| 31 | # parenthesized markers (e.g. "list[...]", "function-wrap[...]") |
| 32 | # are already caught structurally by _BRACKETED_INTERNAL_RE and |
| 33 | # don't need a list entry. |
| 34 | # 3. pydantic-core's source (the `error.rs`/schema-kind definitions |
| 35 | # in the pydantic-core Rust crate) is the canonical reference if |
| 36 | # you want to check before it shows up in a live fuzz run. |
| 37 | _INTERNAL_LOC_MARKERS = frozenset( |
| 38 | { |
| 39 | "function-wrap", |
| 40 | "function-after", |
| 41 | "function-before", |
| 42 | "function-plain", |
| 43 | "json-or-python", |
| 44 | "lax-or-strict", |
| 45 | "chain", |
| 46 | "default", |
| 47 | "nullable", |
| 48 | "tagged-union", |
| 49 | "union", |
| 50 | "call", |
| 51 | "arguments", |
| 52 | "is-instance", |
| 53 | "is-subclass", |
| 54 | "callable", |
| 55 | "str", |
| 56 | "int", |
| 57 | "float", |
| 58 | "bool", |
| 59 | "bytes", |
| 60 | "bytearray", |
| 61 | "list", |
| 62 | "tuple", |
| 63 | "dict", |
| 64 | "set", |
| 65 | "frozenset", |
| 66 | "complex", |
| 67 | "none", |
| 68 | "nonetype", |
| 69 | } |
| 70 | ) |
| 71 | |
| 72 | |
| 73 | # Pydantic repeats the whole offending value under `input` in *every* |
| 74 | # error entry, so one malformed request whose input fails per-element |
| 75 | # echoes that input once per error. Measured on `/v1/responses`: a |
| 76 | # 4,475-byte body produced 12,001 errors and a 23.6 MB response, an |
| 77 | # amplification of ~5,300x that scales linearly with the request size. |
| 78 | # Bound both the number of entries reported and the value echoed in each. |
| 79 | _MAX_REPORTED_ERRORS = 10 |
| 80 | _MAX_ERROR_INPUT_CHARS = 200 |
| 81 | _MAX_ERROR_CHARS = 1000 |
| 82 | |
| 83 | |
| 84 | def _summarize_error_input(value: object) -> object: |
| 85 | """A size-bounded stand-in for a validation error's `input` value. |
| 86 | |
| 87 | Containers are described rather than rendered: `repr()` of a large |
| 88 | list materializes the whole string first, which is the cost this is |
| 89 | meant to avoid. |
| 90 | """ |
| 91 | if isinstance(value, str): |
| 92 | if len(value) > _MAX_ERROR_INPUT_CHARS: |
| 93 | return value[:_MAX_ERROR_INPUT_CHARS] + "...[truncated]" |
| 94 | return value |
| 95 | if isinstance(value, (bytes, bytearray)): |
| 96 | return f"<{type(value).__name__} of {len(value)} bytes>" |
| 97 | if isinstance(value, (list, tuple, set, frozenset, dict)): |
| 98 | return f"<{type(value).__name__} of {len(value)} items>" |
| 99 | try: |
| 100 | text = str(value) |
| 101 | except Exception: |
| 102 | # e.g. an int past `sys.get_int_max_str_digits()`. |
| 103 | return f"<{type(value).__name__}>" |
| 104 | if len(text) > _MAX_ERROR_INPUT_CHARS: |
| 105 | return text[:_MAX_ERROR_INPUT_CHARS] + "...[truncated]" |
| 106 | return text |
| 107 | |
| 108 | |
| 109 | def _format_error(err: object) -> str: |
| 110 | """Render one validation error, bounded in size.""" |
| 111 | if isinstance(err, dict): |
| 112 | err = dict(err) |
| 113 | if "input" in err: |
| 114 | err["input"] = _summarize_error_input(err["input"]) |
| 115 | loc = err.get("loc") |
| 116 | if isinstance(loc, (tuple, list)): |
| 117 | # For a union-typed field pydantic spells every branch out in |
| 118 | # `loc`, which is ~800 characters of type names per entry here. |
| 119 | # `param` is already cleaned this way. |
| 120 | err["loc"] = clean_loc_for_param(tuple(loc)) |
| 121 | text = str(err) |
| 122 | if len(text) > _MAX_ERROR_CHARS: |
| 123 | text = text[:_MAX_ERROR_CHARS] + "...[truncated]" |
| 124 | return text |
| 125 | |
| 126 | |
| 127 | def _is_internal_loc_segment(segment: str) -> bool: |
| 128 | """True if `segment` is a Pydantic-internal wrapper/union-branch |
| 129 | marker rather than a user-meaningful field name or list index.""" |
| 130 | if _BRACKETED_INTERNAL_RE.search(segment): |
| 131 | return True |
| 132 | return segment.lower() in _INTERNAL_LOC_MARKERS |
| 133 | |
| 134 | |
| 135 | def clean_loc_for_param(loc: tuple) -> str: |
| 136 | """Join a Pydantic error `loc` tuple into a clean dotted `param` |
| 137 | path, dropping internal wrapper/union-branch markers that don't |
| 138 | correspond to a real field name an API consumer would recognize. |
| 139 | |
| 140 | E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt') |
| 141 | -> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt". |
| 142 | """ |
| 143 | parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))] |
| 144 | if not parts: |
| 145 | return ".".join(str(p) for p in loc) |
| 146 | return ".".join(parts) |
| 147 | |
| 148 | |
| 149 | async def validation_exception_handler(req: Request, exc: RequestValidationError): |
| 150 | if req.app.state.args.log_error_stack: |
| 151 | logger.exception( |
| 152 | "RequestValidationError caught. Request id: %s", |
| 153 | req.state.request_metadata.request_id |
| 154 | if hasattr(req.state, "request_metadata") |
| 155 | else None, |
| 156 | ) |
| 157 | |
| 158 | param = None |
| 159 | errors = exc.errors() |
| 160 | for error in errors: |
| 161 | if "ctx" in error and "error" in error["ctx"]: |
| 162 | ctx_error = error["ctx"]["error"] |
| 163 | if isinstance(ctx_error, VLLMValidationError): |
| 164 | param = ctx_error.parameter |
| 165 | break |
| 166 | |
| 167 | if param is None and errors: |
| 168 | first_error = errors[0] |
| 169 | loc = first_error.get("loc") if isinstance(first_error, dict) else None |
| 170 | if loc: |
| 171 | param = clean_loc_for_param(loc) |
| 172 | |
| 173 | # Build the message from exc.errors() instead of str(exc) - str(exc) |
| 174 | # leaks the server's file path via FastAPI's endpoint context. |
| 175 | if errors: |
| 176 | count = len(errors) |
| 177 | label = "error" if count == 1 else "errors" |
| 178 | reported = errors[:_MAX_REPORTED_ERRORS] |
| 179 | message = f"{count} validation {label}:\n" |
| 180 | message += "".join(f" {_format_error(err)}\n" for err in reported) |
| 181 | if count > len(reported): |
| 182 | message += f" ...and {count - len(reported)} more {label}\n" |
| 183 | message = message.rstrip() |
| 184 | else: |
| 185 | message = "Validation error" |
| 186 | |
| 187 | err = ErrorResponse( |
| 188 | error=ErrorInfo( |
| 189 | message=sanitize_message(message), |
| 190 | type=HTTPStatus.BAD_REQUEST.phrase, |
| 191 | code=HTTPStatus.BAD_REQUEST, |
| 192 | param=param, |
| 193 | ) |
| 194 | ) |
| 195 | return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST) |