@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4dc8026d86
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fix 307 Redirect for `/metrics` (#4523)

Robert Shaw committed May 1, 2024 at 12:14 UTC 4dc8026d8614185ece28dd3fcd82aa0dabb4f79c
2 files changed +7 -3
vllm/engine/metrics.py
+1 -1
@@ -119,7 +119,7 @@ class Metrics:
119 buckets=[1, 2, 5, 10, 20],
120 )
121 self.counter_request_success = Counter(
122 - name="vllm:request_success",
122 + name="vllm:request_success_total",
123 documentation="Count of successfully processed requests.",
124 labelnames=labelnames + [Metrics.labelname_finish_reason])
125
vllm/entrypoints/openai/api_server.py
+6 -2
@@ -2,6 +2,7 @@ import asyncio
2 import importlib
3 import inspect
4 import os
5 +import re
6 from contextlib import asynccontextmanager
7 from http import HTTPStatus
8
@@ -12,6 +13,7 @@ from fastapi.exceptions import RequestValidationError
13 from fastapi.middleware.cors import CORSMiddleware
14 from fastapi.responses import JSONResponse, Response, StreamingResponse
15 from prometheus_client import make_asgi_app
16 +from starlette.routing import Mount
17
18 import vllm
19 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -55,8 +57,10 @@ def parse_args():
57
58
59 # Add prometheus asgi middleware to route /metrics requests
58 -metrics_app = make_asgi_app()
59 -app.mount("/metrics", metrics_app)
60 +route = Mount("/metrics", make_asgi_app())
61 +# Workaround for 307 Redirect for /metrics
62 +route.path_regex = re.compile('^/metrics(?P<path>.*)$')
63 +app.routes.append(route)
64
65
66 @app.exception_handler(RequestValidationError)