| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests that validation_exception_handler populates the `param` field |
| 4 | in its error response using the Pydantic error's `loc`, even when no |
| 5 | custom VLLMValidationError context is present. |
| 6 | |
| 7 | Previously, `param` was only populated for errors carrying a custom |
| 8 | VLLMValidationError in their Pydantic `ctx`. Plain validation failures |
| 9 | (missing fields, wrong types) left `param` as None, even though the |
| 10 | field name was readily available from `error['loc']`. |
| 11 | """ |
| 12 | |
| 13 | import json |
| 14 | from types import SimpleNamespace |
| 15 | |
| 16 | import pytest |
| 17 | from fastapi.exceptions import RequestValidationError |
| 18 | |
| 19 | from vllm.entrypoints.serve.exception_handling.handlers.validation import ( |
| 20 | clean_loc_for_param, |
| 21 | validation_exception_handler, |
| 22 | ) |
| 23 | |
| 24 | |
| 25 | def _fake_request(log_error_stack: bool = False) -> SimpleNamespace: |
| 26 | """Minimal stand-in for a FastAPI Request - just enough for the |
| 27 | handler to read req.app.state.args.log_error_stack.""" |
| 28 | return SimpleNamespace( |
| 29 | app=SimpleNamespace( |
| 30 | state=SimpleNamespace(args=SimpleNamespace(log_error_stack=log_error_stack)) |
| 31 | ), |
| 32 | state=SimpleNamespace(), # no request_metadata -> hasattr(...) is False |
| 33 | ) |
| 34 | |
| 35 | |
| 36 | class TestValidationErrorParamFallback: |
| 37 | """Ensure `param` falls back to the Pydantic error's `loc` when no |
| 38 | custom VLLMValidationError context is present.""" |
| 39 | |
| 40 | @pytest.mark.parametrize( |
| 41 | ("error_type", "msg"), |
| 42 | [ |
| 43 | ("missing", "Field required"), |
| 44 | ("list_type", "Input should be a valid list"), |
| 45 | ], |
| 46 | ids=["missing-field", "wrong-type"], |
| 47 | ) |
| 48 | @pytest.mark.asyncio |
| 49 | async def test_param_falls_back_to_loc(self, error_type: str, msg: str): |
| 50 | errors = [{"type": error_type, "loc": ("body", "messages"), "msg": msg}] |
| 51 | exc = RequestValidationError(errors) |
| 52 | |
| 53 | response = await validation_exception_handler(_fake_request(), exc) |
| 54 | body = json.loads(response.body) |
| 55 | |
| 56 | assert body["error"]["param"] == "body.messages" |
| 57 | |
| 58 | @pytest.mark.asyncio |
| 59 | async def test_param_fallback_does_not_crash_on_non_dict_error(self): |
| 60 | """Schemathesis fuzzing found that errors[0] isn't always a dict. |
| 61 | The fallback must not crash in that case - it should just leave |
| 62 | param as None instead of raising.""" |
| 63 | exc = RequestValidationError(["some unexpected non-dict error"]) |
| 64 | |
| 65 | response = await validation_exception_handler(_fake_request(), exc) |
| 66 | body = json.loads(response.body) |
| 67 | |
| 68 | assert body["error"]["param"] is None |
| 69 | |
| 70 | |
| 71 | class TestCleanLocForParam: |
| 72 | """Guards against PR #1's naive dot-joined `loc` fallback leaking |
| 73 | Pydantic-internal wrapper/union-branch markers into `param`, e.g. |
| 74 | 'body.function-wrap[__log_extra_fields__()].prompt' instead of the |
| 75 | clean 'body.prompt' an API consumer would recognize. |
| 76 | """ |
| 77 | |
| 78 | @pytest.mark.parametrize( |
| 79 | "loc,expected", |
| 80 | [ |
| 81 | (("body", "prompt"), "body.prompt"), |
| 82 | (("body", "messages", 2, "content"), "body.messages.2.content"), |
| 83 | ( |
| 84 | ("body", "function-wrap[__log_extra_fields__()]", "prompt"), |
| 85 | "body.prompt", |
| 86 | ), |
| 87 | (("body", "stop", "str"), "body.stop"), |
| 88 | (("body", "stop", "list[str]"), "body.stop"), |
| 89 | ( |
| 90 | ("body", "prompt", "list[constrained-int]"), |
| 91 | "body.prompt", |
| 92 | ), |
| 93 | ], |
| 94 | ) |
| 95 | def test_strips_internal_markers(self, loc, expected): |
| 96 | assert clean_loc_for_param(loc) == expected |
| 97 | |
| 98 | def test_all_internal_falls_back_to_raw_join(self): |
| 99 | """If every loc segment looks internal, fall back to the raw |
| 100 | dot-join rather than returning an empty string.""" |
| 101 | loc = ("function-wrap[__log_extra_fields__()]",) |
| 102 | assert clean_loc_for_param(loc) == "function-wrap[__log_extra_fields__()]" |
| 103 | |
| 104 | |
| 105 | class TestValidationErrorDoesNotLeakServerPaths: |
| 106 | """FastAPI sets endpoint_file/line/function/path on the exception |
| 107 | during routing, not in the constructor - so we set them manually |
| 108 | here to reproduce the real leak. See #31683.""" |
| 109 | |
| 110 | @pytest.mark.asyncio |
| 111 | async def test_handler_strips_endpoint_file_context(self): |
| 112 | errors = [ |
| 113 | { |
| 114 | "type": "list_type", |
| 115 | "loc": ("body", "messages"), |
| 116 | "msg": "Input should be a valid list", |
| 117 | "input": "not-a-list", |
| 118 | } |
| 119 | ] |
| 120 | exc = RequestValidationError(errors) |
| 121 | exc.endpoint_file = ( |
| 122 | "/usr/local/lib/python3.12/dist-packages/vllm/" |
| 123 | "entrypoints/serve/utils/api_utils.py" |
| 124 | ) |
| 125 | exc.endpoint_line = 40 |
| 126 | exc.endpoint_function = "create_chat_completion" |
| 127 | exc.endpoint_path = "POST /v1/chat/completions" |
| 128 | |
| 129 | response = await validation_exception_handler(_fake_request(), exc) |
| 130 | body = json.loads(response.body) |
| 131 | message = body["error"]["message"] |
| 132 | |
| 133 | assert "/usr/local/" not in message |
| 134 | assert "api_utils.py" not in message |
| 135 | assert "create_chat_completion" not in message |
| 136 | assert "list_type" in message |
| 137 | assert "Input should be a valid list" in message |
| 138 | |
| 139 | @pytest.mark.asyncio |
| 140 | async def test_handler_strips_endpoint_path_only_variant(self): |
| 141 | """Covers the branch where only endpoint_path is set.""" |
| 142 | errors = [ |
| 143 | {"type": "missing", "loc": ("body", "messages"), "msg": "Field required"} |
| 144 | ] |
| 145 | exc = RequestValidationError(errors) |
| 146 | exc.endpoint_path = "POST /v1/chat/completions" |
| 147 | |
| 148 | response = await validation_exception_handler(_fake_request(), exc) |
| 149 | body = json.loads(response.body) |
| 150 | message = body["error"]["message"] |
| 151 | |
| 152 | assert "missing" in message |
| 153 | assert "Field required" in message |
| 154 | |
| 155 | |
| 156 | class TestValidationErrorBodyIsBounded: |
| 157 | """A malformed request must not produce an unbounded 400 body. |
| 158 | |
| 159 | Pydantic repeats the whole offending value under `input` in every |
| 160 | error entry, so an input that fails per-element echoes itself once |
| 161 | per error. On `/v1/responses` a 4,475-byte body produced 12,001 |
| 162 | errors and a 23.6 MB response before this was bounded. See #49239. |
| 163 | """ |
| 164 | |
| 165 | @pytest.mark.asyncio |
| 166 | async def test_many_errors_are_capped(self): |
| 167 | big_input = [{"type": "input_text", "text": 1} for _ in range(500)] |
| 168 | errors = [ |
| 169 | { |
| 170 | "type": "string_type", |
| 171 | "loc": ("body", "input", i), |
| 172 | "msg": "Input should be a valid string", |
| 173 | "input": big_input, |
| 174 | } |
| 175 | for i in range(5000) |
| 176 | ] |
| 177 | exc = RequestValidationError(errors) |
| 178 | |
| 179 | response = await validation_exception_handler(_fake_request(), exc) |
| 180 | body = json.loads(response.body) |
| 181 | message = body["error"]["message"] |
| 182 | |
| 183 | assert len(response.body) < 32_000, f"body was {len(response.body)} bytes" |
| 184 | # The true count is still reported, and the remainder acknowledged. |
| 185 | assert message.startswith("5000 validation errors:") |
| 186 | assert "more errors" in message |
| 187 | |
| 188 | @pytest.mark.asyncio |
| 189 | async def test_container_input_is_described_not_echoed(self): |
| 190 | errors = [ |
| 191 | { |
| 192 | "type": "string_type", |
| 193 | "loc": ("body", "input"), |
| 194 | "msg": "Input should be a valid string", |
| 195 | "input": [{"text": "payload"} for _ in range(300)], |
| 196 | } |
| 197 | ] |
| 198 | exc = RequestValidationError(errors) |
| 199 | |
| 200 | response = await validation_exception_handler(_fake_request(), exc) |
| 201 | message = json.loads(response.body)["error"]["message"] |
| 202 | |
| 203 | assert "<list of 300 items>" in message |
| 204 | assert "payload" not in message |
| 205 | |
| 206 | @pytest.mark.asyncio |
| 207 | async def test_long_string_input_is_truncated(self): |
| 208 | errors = [ |
| 209 | { |
| 210 | "type": "int_type", |
| 211 | "loc": ("body", "max_tokens"), |
| 212 | "msg": "Input should be a valid integer", |
| 213 | "input": "A" * 100_000, |
| 214 | } |
| 215 | ] |
| 216 | exc = RequestValidationError(errors) |
| 217 | |
| 218 | response = await validation_exception_handler(_fake_request(), exc) |
| 219 | message = json.loads(response.body)["error"]["message"] |
| 220 | |
| 221 | assert "[truncated]" in message |
| 222 | assert len(message) < 4_000 |
| 223 | |
| 224 | @pytest.mark.asyncio |
| 225 | async def test_small_input_is_still_reported_verbatim(self): |
| 226 | """Bounding must not cost legitimate clients their diagnostics.""" |
| 227 | errors = [ |
| 228 | { |
| 229 | "type": "list_type", |
| 230 | "loc": ("body", "messages"), |
| 231 | "msg": "Input should be a valid list", |
| 232 | "input": "not-a-list", |
| 233 | } |
| 234 | ] |
| 235 | exc = RequestValidationError(errors) |
| 236 | |
| 237 | response = await validation_exception_handler(_fake_request(), exc) |
| 238 | message = json.loads(response.body)["error"]["message"] |
| 239 | |
| 240 | assert "not-a-list" in message |
| 241 | assert "[truncated]" not in message |
| 242 | |
| 243 | @pytest.mark.asyncio |
| 244 | async def test_union_loc_is_cleaned_in_the_message(self): |
| 245 | """A union-typed field spells every branch out in `loc`. |
| 246 | |
| 247 | Left raw, that is ~800 characters of type names per entry and it |
| 248 | dominates the bounded message. `param` is already cleaned this way. |
| 249 | """ |
| 250 | loc = ( |
| 251 | "body", |
| 252 | "input", |
| 253 | "list[union[EasyInputMessageParam,Message,ResponseOutputMessageParam]]", |
| 254 | 0, |
| 255 | "content", |
| 256 | ) |
| 257 | errors = [ |
| 258 | { |
| 259 | "type": "string_type", |
| 260 | "loc": loc, |
| 261 | "msg": "Input should be a valid string", |
| 262 | "input": 12345, |
| 263 | } |
| 264 | ] |
| 265 | exc = RequestValidationError(errors) |
| 266 | |
| 267 | response = await validation_exception_handler(_fake_request(), exc) |
| 268 | message = json.loads(response.body)["error"]["message"] |
| 269 | |
| 270 | assert "'loc': 'body.input.0.content'" in message |
| 271 | assert "EasyInputMessageParam" not in message |