@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 222 lines 7.7 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Test that http_requests_total metric records correct status codes.
4
5 Regression test for: Prometheus http_requests_total records 4xx exceptions
6 (ValueError, TypeError, etc.) as 5xx because they propagate through the
7 PrometheusInstrumentatorMiddleware before being caught by ServerErrorMiddleware.
8 """
9
10 from argparse import Namespace
11
12 import httpx
13 import pytest
14 from fastapi import HTTPException
15 from prometheus_client import CollectorRegistry
16
17 from vllm.entrypoints.launchers.api_server.entry import build_app
18 from vllm.exceptions import (
19 VLLMNotFoundError,
20 VLLMServerError,
21 VLLMValidationError,
22 )
23
24
25 @pytest.fixture(scope="module")
26 def should_do_global_cleanup_after_test() -> bool:
27 # This suite never initializes distributed/accelerator state.
28 return False
29
30
31 def _build_args() -> Namespace:
32 """Minimal args for ``build_app``; avoids ``make_arg_parser`` device probing."""
33 return Namespace(
34 disable_fastapi_docs=True,
35 enable_offline_docs=False,
36 root_path=None,
37 allowed_origins=["*"],
38 allow_credentials=False,
39 allowed_methods=["*"],
40 allowed_headers=["*"],
41 api_key=None,
42 enable_request_id_headers=False,
43 enable_fault_tolerance=False,
44 middleware=[],
45 log_error_stack=False,
46 )
47
48
49 @pytest.fixture(scope="module")
50 def registry():
51 """Shared Prometheus registry for the module-scoped app."""
52 return CollectorRegistry()
53
54
55 @pytest.fixture(scope="module")
56 def app(registry):
57 """Build the real vLLM FastAPI app once and attach probe routes that raise.
58
59 Patch the name used by ``attach_router`` (imported into the instrumentator
60 metrics module), not ``vllm.v1.metrics.prometheus`` alone — that binding is
61 captured at import time.
62 """
63 import vllm.entrypoints.serve.instrumentator.metrics as metrics_mod
64
65 original = metrics_mod.get_prometheus_registry
66 metrics_mod.get_prometheus_registry = lambda: registry
67 try:
68 app = build_app(_build_args(), supported_tasks=())
69 finally:
70 metrics_mod.get_prometheus_registry = original
71
72 @app.get("/raise_http_exception_400")
73 async def raise_http_exception_400():
74 raise HTTPException(status_code=400, detail="bad request")
75
76 @app.get("/raise_http_exception_404")
77 async def raise_http_exception_404():
78 raise HTTPException(status_code=404, detail="not found")
79
80 @app.get("/raise_request_validation_error")
81 async def raise_request_validation_error(n: int):
82 # Invalid ``n`` triggers FastAPI's RequestValidationError.
83 return {"n": n}
84
85 @app.get("/raise_vllm_validation_error")
86 async def raise_vllm_validation_error():
87 raise VLLMValidationError("bad parameter", parameter="temperature")
88
89 @app.get("/raise_vllm_not_found_error")
90 async def raise_vllm_not_found_error():
91 raise VLLMNotFoundError("model not found")
92
93 @app.get("/raise_vllm_server_error")
94 async def raise_vllm_server_error():
95 # Bare VLLMServerError goes through vllm_error_handler → 500.
96 # EngineGenerateError / EngineDeadError are not used here: they call
97 # terminate_if_errored and need engine/server state.
98 raise VLLMServerError("internal server failure")
99
100 @app.get("/raise_value_error")
101 async def raise_value_error():
102 raise ValueError("invalid input value")
103
104 @app.get("/raise_type_error")
105 async def raise_type_error():
106 raise TypeError("wrong type")
107
108 @app.get("/raise_overflow_error")
109 async def raise_overflow_error():
110 raise OverflowError("number too large")
111
112 @app.get("/raise_not_implemented_error")
113 async def raise_not_implemented_error():
114 raise NotImplementedError("feature not supported")
115
116 @app.get("/raise_runtime_error")
117 async def raise_runtime_error():
118 raise RuntimeError("unexpected server error")
119
120 @app.get("/success")
121 async def success():
122 return {"status": "ok"}
123
124 return app
125
126
127 def _get_http_requests_total(registry, method: str, handler: str):
128 """Extract the http_requests_total metric values grouped by status.
129
130 Returns a dict like {"2xx": 1.0, "5xx": 1.0} for the given handler.
131 """
132 results: dict[str, float] = {}
133 for metric in registry.collect():
134 if metric.name == "http_requests":
135 for sample in metric.samples:
136 if (
137 sample.name == "http_requests_total"
138 and sample.labels.get("method") == method
139 and sample.labels.get("handler") == handler
140 ):
141 status = sample.labels.get("status")
142 results[status] = results.get(status, 0) + sample.value
143 return results
144
145
146 @pytest.mark.asyncio
147 @pytest.mark.parametrize(
148 "endpoint,expected_status_group,expected_http_code,request_kwargs",
149 [
150 ("/raise_http_exception_400", "4xx", 400, {}),
151 ("/raise_http_exception_404", "4xx", 404, {}),
152 ("/raise_request_validation_error", "4xx", 400, {"params": {"n": "x"}}),
153 ("/raise_vllm_validation_error", "4xx", 400, {}),
154 ("/raise_vllm_not_found_error", "4xx", 404, {}),
155 ("/raise_vllm_server_error", "5xx", 500, {}),
156 ("/raise_value_error", "4xx", 400, {}),
157 ("/raise_type_error", "4xx", 400, {}),
158 ("/raise_overflow_error", "4xx", 400, {}),
159 ("/raise_not_implemented_error", "5xx", 501, {}),
160 ("/raise_runtime_error", "5xx", 500, {}),
161 ("/success", "2xx", 200, {}),
162 ],
163 ids=[
164 "HTTPException(400)->4xx",
165 "HTTPException(404)->4xx",
166 "RequestValidationError->4xx",
167 "VLLMValidationError->4xx",
168 "VLLMNotFoundError->4xx",
169 "VLLMServerError->5xx",
170 "ValueError->4xx",
171 "TypeError->4xx",
172 "OverflowError->4xx",
173 "NotImplementedError->5xx",
174 "RuntimeError->5xx",
175 "success->2xx",
176 ],
177 )
178 async def test_http_requests_total_records_correct_status(
179 app,
180 registry,
181 endpoint,
182 expected_status_group,
183 expected_http_code,
184 request_kwargs,
185 ):
186 """Verify that http_requests_total records the correct status group.
187
188 The Prometheus metric should reflect the actual HTTP status code returned
189 to the client, not a default 500 for all exceptions.
190 """
191 # raise_app_exceptions=False allows the full ASGI middleware stack
192 # (including ServerErrorMiddleware) to handle exceptions and generate
193 # proper HTTP responses, just like a real server would.
194 transport = httpx.ASGITransport(app=app, raise_app_exceptions=False)
195 async with httpx.AsyncClient(
196 transport=transport, base_url="http://testserver"
197 ) as client:
198 response = await client.get(endpoint, **request_kwargs)
199
200 # Verify the HTTP response code returned to the client is correct
201 assert response.status_code == expected_http_code, (
202 f"Expected HTTP {expected_http_code} for {endpoint}, got {response.status_code}"
203 )
204
205 # Verify Prometheus recorded the correct status group
206 metrics = _get_http_requests_total(registry, "GET", endpoint)
207 assert expected_status_group in metrics, (
208 f"Expected Prometheus to record '{expected_status_group}' for "
209 f"{endpoint}, but got: {metrics}"
210 )
211 assert metrics[expected_status_group] == 1.0, (
212 f"Expected 1 request recorded as '{expected_status_group}' for "
213 f"{endpoint}, but got {metrics[expected_status_group]}"
214 )
215
216 # For endpoints that should be recorded as 4xx, verify they are NOT
217 # incorrectly recorded as 5xx
218 if expected_status_group == "4xx":
219 assert "5xx" not in metrics, (
220 f"Expected NO '5xx' recording for {endpoint} "
221 f"(should be '4xx'), but found: {metrics}"
222 )