A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fix 307 Redirect for `/metrics` (#4523)
Robert Shaw committed
May 1, 2024 at 12:14 UTC
4dc8026d8614185ece28dd3fcd82aa0dabb4f79c
2 files changed
+7
-3
vllm/engine/metrics.py
+1
-1
@@ -119,7 +119,7 @@ class Metrics:
119
buckets=[1, 2, 5, 10, 20],
120
)
121
self.counter_request_success = Counter(
122
- name="vllm:request_success",
122
+ name="vllm:request_success_total",
123
documentation="Count of successfully processed requests.",
124
labelnames=labelnames + [Metrics.labelname_finish_reason])
125
vllm/entrypoints/openai/api_server.py
+6
-2
@@ -2,6 +2,7 @@ import asyncio
2
import importlib
3
import inspect
4
import os
5
+import re
6
from contextlib import asynccontextmanager
7
from http import HTTPStatus
8
@@ -12,6 +13,7 @@ from fastapi.exceptions import RequestValidationError
13
from fastapi.middleware.cors import CORSMiddleware
14
from fastapi.responses import JSONResponse, Response, StreamingResponse
15
from prometheus_client import make_asgi_app
16
+from starlette.routing import Mount
17
18
import vllm
19
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -55,8 +57,10 @@ def parse_args():
57
58
59
# Add prometheus asgi middleware to route /metrics requests
58
-metrics_app = make_asgi_app()
59
-app.mount("/metrics", metrics_app)
60
+route = Mount("/metrics", make_asgi_app())
61
+# Workaround for 307 Redirect for /metrics
62
+route.path_regex = re.compile('^/metrics(?P<path>.*)$')
63
+app.routes.append(route)
64
65
66
@app.exception_handler(RequestValidationError)