A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix][Frontend] Fix missing `/metrics` endpoint (#6463)
Cyrus Leung committed
Jul 19, 2024 at 11:55 UTC
6366efc67b0aedd2c1721c14385370e50b297fb3
2 files changed
+70
-5
tests/entrypoints/openai/test_basic.py
new
+61
@@ -0,0 +1,61 @@
1
+from http import HTTPStatus
2
+
3
+import openai
4
+import pytest
5
+import requests
6
+
7
+from vllm.version import __version__ as VLLM_VERSION
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
12
+
13
+
14
+@pytest.fixture(scope="module")
15
+def server():
16
+ args = [
17
+ # use half precision for speed and memory savings in CI environment
18
+ "--dtype",
19
+ "bfloat16",
20
+ "--max-model-len",
21
+ "8192",
22
+ "--enforce-eager",
23
+ "--max-num-seqs",
24
+ "128",
25
+ ]
26
+
27
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
28
+ yield remote_server
29
+
30
+
31
+@pytest.fixture(scope="module")
32
+def client(server):
33
+ return server.get_async_client()
34
+
35
+
36
+@pytest.mark.asyncio
37
+async def test_show_version(client: openai.AsyncOpenAI):
38
+ base_url = str(client.base_url)[:-3].strip("/")
39
+
40
+ response = requests.get(base_url + "/version")
41
+ response.raise_for_status()
42
+
43
+ assert response.json() == {"version": VLLM_VERSION}
44
+
45
+
46
+@pytest.mark.asyncio
47
+async def test_check_health(client: openai.AsyncOpenAI):
48
+ base_url = str(client.base_url)[:-3].strip("/")
49
+
50
+ response = requests.get(base_url + "/health")
51
+
52
+ assert response.status_code == HTTPStatus.OK
53
+
54
+
55
+@pytest.mark.asyncio
56
+async def test_log_metrics(client: openai.AsyncOpenAI):
57
+ base_url = str(client.base_url)[:-3].strip("/")
58
+
59
+ response = requests.get(base_url + "/metrics")
60
+
61
+ assert response.status_code == HTTPStatus.OK
vllm/entrypoints/openai/api_server.py
+9
-5
@@ -73,11 +73,13 @@ async def lifespan(app: fastapi.FastAPI):
73
74
router = APIRouter()
75
76
-# Add prometheus asgi middleware to route /metrics requests
77
-route = Mount("/metrics", make_asgi_app())
78
-# Workaround for 307 Redirect for /metrics
79
-route.path_regex = re.compile('^/metrics(?P<path>.*)$')
80
-router.routes.append(route)
76
+
77
+def mount_metrics(app: fastapi.FastAPI):
78
+ # Add prometheus asgi middleware to route /metrics requests
79
+ metrics_route = Mount("/metrics", make_asgi_app())
80
+ # Workaround for 307 Redirect for /metrics
81
+ metrics_route.path_regex = re.compile('^/metrics(?P<path>.*)$')
82
+ app.routes.append(metrics_route)
83
84
85
@router.get("/health")
@@ -167,6 +169,8 @@ def build_app(args):
169
app.include_router(router)
170
app.root_path = args.root_path
171
172
+ mount_metrics(app)
173
+
174
app.add_middleware(
175
CORSMiddleware,
176
allow_origins=args.allowed_origins,