@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6366efc67b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix][Frontend] Fix missing `/metrics` endpoint (#6463)

Cyrus Leung committed Jul 19, 2024 at 11:55 UTC 6366efc67b0aedd2c1721c14385370e50b297fb3
2 files changed +70 -5
tests/entrypoints/openai/test_basic.py new
+61
@@ -0,0 +1,61 @@
1 +from http import HTTPStatus
2 +
3 +import openai
4 +import pytest
5 +import requests
6 +
7 +from vllm.version import __version__ as VLLM_VERSION
8 +
9 +from ...utils import RemoteOpenAIServer
10 +
11 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
12 +
13 +
14 +@pytest.fixture(scope="module")
15 +def server():
16 + args = [
17 + # use half precision for speed and memory savings in CI environment
18 + "--dtype",
19 + "bfloat16",
20 + "--max-model-len",
21 + "8192",
22 + "--enforce-eager",
23 + "--max-num-seqs",
24 + "128",
25 + ]
26 +
27 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
28 + yield remote_server
29 +
30 +
31 +@pytest.fixture(scope="module")
32 +def client(server):
33 + return server.get_async_client()
34 +
35 +
36 +@pytest.mark.asyncio
37 +async def test_show_version(client: openai.AsyncOpenAI):
38 + base_url = str(client.base_url)[:-3].strip("/")
39 +
40 + response = requests.get(base_url + "/version")
41 + response.raise_for_status()
42 +
43 + assert response.json() == {"version": VLLM_VERSION}
44 +
45 +
46 +@pytest.mark.asyncio
47 +async def test_check_health(client: openai.AsyncOpenAI):
48 + base_url = str(client.base_url)[:-3].strip("/")
49 +
50 + response = requests.get(base_url + "/health")
51 +
52 + assert response.status_code == HTTPStatus.OK
53 +
54 +
55 +@pytest.mark.asyncio
56 +async def test_log_metrics(client: openai.AsyncOpenAI):
57 + base_url = str(client.base_url)[:-3].strip("/")
58 +
59 + response = requests.get(base_url + "/metrics")
60 +
61 + assert response.status_code == HTTPStatus.OK
vllm/entrypoints/openai/api_server.py
+9 -5
@@ -73,11 +73,13 @@ async def lifespan(app: fastapi.FastAPI):
73
74 router = APIRouter()
75
76 -# Add prometheus asgi middleware to route /metrics requests
77 -route = Mount("/metrics", make_asgi_app())
78 -# Workaround for 307 Redirect for /metrics
79 -route.path_regex = re.compile('^/metrics(?P<path>.*)$')
80 -router.routes.append(route)
76 +
77 +def mount_metrics(app: fastapi.FastAPI):
78 + # Add prometheus asgi middleware to route /metrics requests
79 + metrics_route = Mount("/metrics", make_asgi_app())
80 + # Workaround for 307 Redirect for /metrics
81 + metrics_route.path_regex = re.compile('^/metrics(?P<path>.*)$')
82 + app.routes.append(metrics_route)
83
84
85 @router.get("/health")
@@ -167,6 +169,8 @@ def build_app(args):
169 app.include_router(router)
170 app.root_path = args.root_path
171
172 + mount_metrics(app)
173 +
174 app.add_middleware(
175 CORSMiddleware,
176 allow_origins=args.allowed_origins,