[Bugfix][Frontend] Fix http_requests_total metric recording some 4xx errors as 5xx (#44303)
Signed-off-by: Zach Zhu <zzqshu@126.com>
Zach Zhu committed
Jul 8, 2026 at 13:33 UTC
5d5fab0061e2c880035ff479f929710e790fb6cc
2 files changed
+219
-1
tests/entrypoints/serve/instrumentator/test_http_status_metrics.py
new
+204
@@ -0,0 +1,204 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Test that http_requests_total metric records correct status codes.
4
+
5
+Regression test for: Prometheus http_requests_total records 4xx exceptions
6
+(ValueError, TypeError, etc.) as 5xx because they propagate through the
7
+PrometheusInstrumentatorMiddleware before being caught by ServerErrorMiddleware.
8
+"""
9
+
10
+from argparse import Namespace
11
+from http import HTTPStatus
12
+
13
+import httpx
14
+import pytest
15
+from fastapi import FastAPI, HTTPException, Request
16
+from fastapi.exceptions import RequestValidationError
17
+from fastapi.responses import JSONResponse
18
+from prometheus_client import CollectorRegistry
19
+from prometheus_fastapi_instrumentator import Instrumentator
20
+
21
+from vllm.entrypoints.serve.utils.server_utils import exception_handler
22
+from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
23
+
24
+
25
+@pytest.fixture
26
+def registry():
27
+ """Create a fresh Prometheus registry for each test."""
28
+ return CollectorRegistry()
29
+
30
+
31
+@pytest.fixture
32
+def app(registry):
33
+ """Create a minimal FastAPI app that mirrors vLLM's exception handler
34
+ and Prometheus middleware setup."""
35
+
36
+ app = FastAPI()
37
+
38
+ # Mock app state that exception_handler needs
39
+ app.state.args = Namespace(log_error_stack=False)
40
+
41
+ # Register exception handlers exactly as vLLM does in build_app()
42
+ app.exception_handler(HTTPException)(_http_exception_handler)
43
+ app.exception_handler(RequestValidationError)(_validation_exception_handler)
44
+ app.exception_handler(ValueError)(exception_handler)
45
+ app.exception_handler(TypeError)(exception_handler)
46
+ app.exception_handler(OverflowError)(exception_handler)
47
+ app.exception_handler(NotImplementedError)(exception_handler)
48
+ app.exception_handler(VLLMValidationError)(exception_handler)
49
+ app.exception_handler(VLLMNotFoundError)(exception_handler)
50
+ app.exception_handler(Exception)(exception_handler)
51
+
52
+ # Instrument with Prometheus (same as vLLM's attach_router)
53
+ Instrumentator(
54
+ excluded_handlers=["/metrics"],
55
+ registry=registry,
56
+ ).add().instrument(app)
57
+
58
+ # Test routes that raise different exception types
59
+ @app.get("/raise_value_error")
60
+ async def raise_value_error():
61
+ raise ValueError("invalid input value")
62
+
63
+ @app.get("/raise_type_error")
64
+ async def raise_type_error():
65
+ raise TypeError("wrong type")
66
+
67
+ @app.get("/raise_overflow_error")
68
+ async def raise_overflow_error():
69
+ raise OverflowError("number too large")
70
+
71
+ @app.get("/raise_not_implemented_error")
72
+ async def raise_not_implemented_error():
73
+ raise NotImplementedError("feature not supported")
74
+
75
+ @app.get("/raise_vllm_validation_error")
76
+ async def raise_vllm_validation_error():
77
+ raise VLLMValidationError("bad parameter", parameter="temperature")
78
+
79
+ @app.get("/raise_vllm_not_found_error")
80
+ async def raise_vllm_not_found_error():
81
+ raise VLLMNotFoundError("model not found")
82
+
83
+ @app.get("/raise_http_exception_400")
84
+ async def raise_http_exception_400():
85
+ raise HTTPException(status_code=400, detail="bad request")
86
+
87
+ @app.get("/raise_http_exception_404")
88
+ async def raise_http_exception_404():
89
+ raise HTTPException(status_code=404, detail="not found")
90
+
91
+ @app.get("/raise_runtime_error")
92
+ async def raise_runtime_error():
93
+ raise RuntimeError("unexpected server error")
94
+
95
+ @app.get("/success")
96
+ async def success():
97
+ return {"status": "ok"}
98
+
99
+ return app
100
+
101
+
102
+async def _http_exception_handler(req: Request, exc: HTTPException):
103
+ return JSONResponse({"error": exc.detail}, status_code=exc.status_code)
104
+
105
+
106
+async def _validation_exception_handler(req: Request, exc: RequestValidationError):
107
+ return JSONResponse({"error": str(exc)}, status_code=HTTPStatus.BAD_REQUEST)
108
+
109
+
110
+def _get_http_requests_total(registry, method: str, handler: str):
111
+ """Extract the http_requests_total metric values grouped by status.
112
+
113
+ Returns a dict like {"2xx": 1.0, "5xx": 1.0} for the given handler.
114
+ """
115
+ results: dict[str, float] = {}
116
+ for metric in registry.collect():
117
+ if metric.name == "http_requests":
118
+ for sample in metric.samples:
119
+ if (
120
+ sample.name == "http_requests_total"
121
+ and sample.labels.get("method") == method
122
+ and sample.labels.get("handler") == handler
123
+ ):
124
+ status = sample.labels.get("status")
125
+ results[status] = results.get(status, 0) + sample.value
126
+ return results
127
+
128
+
129
+@pytest.mark.asyncio
130
+@pytest.mark.parametrize(
131
+ "endpoint,expected_status_group,expected_http_code",
132
+ [
133
+ # These should record as 4xx in Prometheus
134
+ ("/raise_value_error", "4xx", 400),
135
+ ("/raise_type_error", "4xx", 400),
136
+ ("/raise_overflow_error", "4xx", 400),
137
+ ("/raise_vllm_validation_error", "4xx", 400),
138
+ ("/raise_vllm_not_found_error", "4xx", 404),
139
+ ("/raise_http_exception_400", "4xx", 400),
140
+ ("/raise_http_exception_404", "4xx", 404),
141
+ # NotImplementedError returns 501 which is still 5xx group
142
+ ("/raise_not_implemented_error", "5xx", 501),
143
+ # These should record as 5xx in Prometheus (genuine server errors)
144
+ ("/raise_runtime_error", "5xx", 500),
145
+ # Successful requests should record as 2xx
146
+ ("/success", "2xx", 200),
147
+ ],
148
+ ids=[
149
+ "ValueError->4xx",
150
+ "TypeError->4xx",
151
+ "OverflowError->4xx",
152
+ "VLLMValidationError->4xx",
153
+ "VLLMNotFoundError->4xx",
154
+ "HTTPException(400)->4xx",
155
+ "HTTPException(404)->4xx",
156
+ "NotImplementedError->5xx",
157
+ "RuntimeError->5xx",
158
+ "success->2xx",
159
+ ],
160
+)
161
+async def test_http_requests_total_records_correct_status(
162
+ app,
163
+ registry,
164
+ endpoint,
165
+ expected_status_group,
166
+ expected_http_code,
167
+):
168
+ """Verify that http_requests_total records the correct status group.
169
+
170
+ The Prometheus metric should reflect the actual HTTP status code returned
171
+ to the client, not a default 500 for all exceptions.
172
+ """
173
+ # raise_app_exceptions=False allows the full ASGI middleware stack
174
+ # (including ServerErrorMiddleware) to handle exceptions and generate
175
+ # proper HTTP responses, just like a real server would.
176
+ transport = httpx.ASGITransport(app=app, raise_app_exceptions=False)
177
+ async with httpx.AsyncClient(
178
+ transport=transport, base_url="http://testserver"
179
+ ) as client:
180
+ response = await client.get(endpoint)
181
+
182
+ # Verify the HTTP response code returned to the client is correct
183
+ assert response.status_code == expected_http_code, (
184
+ f"Expected HTTP {expected_http_code} for {endpoint}, got {response.status_code}"
185
+ )
186
+
187
+ # Verify Prometheus recorded the correct status group
188
+ metrics = _get_http_requests_total(registry, "GET", endpoint)
189
+ assert expected_status_group in metrics, (
190
+ f"Expected Prometheus to record '{expected_status_group}' for "
191
+ f"{endpoint}, but got: {metrics}"
192
+ )
193
+ assert metrics[expected_status_group] == 1.0, (
194
+ f"Expected 1 request recorded as '{expected_status_group}' for "
195
+ f"{endpoint}, but got {metrics[expected_status_group]}"
196
+ )
197
+
198
+ # For endpoints that should be recorded as 4xx, verify they are NOT
199
+ # incorrectly recorded as 5xx
200
+ if expected_status_group == "4xx":
201
+ assert "5xx" not in metrics, (
202
+ f"Expected NO '5xx' recording for {endpoint} "
203
+ f"(should be '4xx'), but found: {metrics}"
204
+ )
vllm/entrypoints/openai/api_server.py
+15
-1
@@ -51,7 +51,11 @@ from vllm.entrypoints.serve.utils.server_utils import (
51
log_response,
52
validation_exception_handler,
53
)
54
-from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
54
+from vllm.exceptions import (
55
+ VLLMNotFoundError,
56
+ VLLMUnprocessableEntityError,
57
+ VLLMValidationError,
58
+)
59
from vllm.logger import init_logger
60
from vllm.reasoning import ReasoningParserManager
61
from vllm.renderers.online_derenderer import OnlineDerenderer
@@ -285,8 +289,18 @@ def build_app(
289
app.exception_handler(EngineGenerateError)(engine_error_handler)
290
app.exception_handler(EngineDeadError)(engine_error_handler)
291
app.exception_handler(GenerationError)(generation_error_handler)
292
+ # Register specific exception types so they are handled by
293
+ # ExceptionMiddleware (inside the Prometheus middleware) rather than
294
+ # ServerErrorMiddleware (outside it). Without this, these exceptions
295
+ # propagate through Prometheus as unhandled and get recorded as 5xx
296
+ # even though they result in 4xx responses to the client.
297
app.exception_handler(VLLMValidationError)(exception_handler)
298
app.exception_handler(VLLMUnprocessableEntityError)(exception_handler)
299
+ app.exception_handler(VLLMNotFoundError)(exception_handler)
300
+ app.exception_handler(ValueError)(exception_handler)
301
+ app.exception_handler(TypeError)(exception_handler)
302
+ app.exception_handler(OverflowError)(exception_handler)
303
+ app.exception_handler(NotImplementedError)(exception_handler)
304
app.exception_handler(Exception)(exception_handler)
305
306
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY