@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8da48e4d95
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Frontend] Publish Prometheus metrics in run_batch API (#7641)

Pooya Davoodi committed Aug 23, 2024 at 23:04 UTC 8da48e4d95421cbd96fbdecdffed89a3d1aab218
2 files changed +76
tests/entrypoints/openai/test_metrics.py
+49
@@ -1,3 +1,7 @@
1 +import subprocess
2 +import sys
3 +import tempfile
4 +import time
5 from http import HTTPStatus
6
7 import openai
@@ -177,3 +181,48 @@ async def test_metrics_exist(client: openai.AsyncOpenAI):
181
182 for metric in EXPECTED_METRICS:
183 assert metric in response.text
184 +
185 +
186 +def test_metrics_exist_run_batch():
187 + input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/e5-mistral-7b-instruct", "input": "You are a helpful assistant."}}""" # noqa: E501
188 +
189 + base_url = "0.0.0.0"
190 + port = "8001"
191 + server_url = f"http://{base_url}:{port}"
192 +
193 + with tempfile.NamedTemporaryFile(
194 + "w") as input_file, tempfile.NamedTemporaryFile(
195 + "r") as output_file:
196 + input_file.write(input_batch)
197 + input_file.flush()
198 + proc = subprocess.Popen([
199 + sys.executable,
200 + "-m",
201 + "vllm.entrypoints.openai.run_batch",
202 + "-i",
203 + input_file.name,
204 + "-o",
205 + output_file.name,
206 + "--model",
207 + "intfloat/e5-mistral-7b-instruct",
208 + "--enable-metrics",
209 + "--url",
210 + base_url,
211 + "--port",
212 + port,
213 + ], )
214 +
215 + def is_server_up(url):
216 + try:
217 + response = requests.get(url)
218 + return response.status_code == 200
219 + except requests.ConnectionError:
220 + return False
221 +
222 + while not is_server_up(server_url):
223 + time.sleep(1)
224 +
225 + response = requests.get(server_url + "/metrics")
226 + assert response.status_code == HTTPStatus.OK
227 +
228 + proc.wait()
vllm/entrypoints/openai/run_batch.py
+27
@@ -3,6 +3,7 @@ from io import StringIO
3 from typing import Awaitable, Callable, List
4
5 import aiohttp
6 +from prometheus_client import start_http_server
7
8 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
9 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -56,6 +57,24 @@ def parse_args():
57 'ID numbers being printed in log.'
58 '\n\nDefault: Unlimited')
59
60 + parser.add_argument("--enable-metrics",
61 + action="store_true",
62 + help="Enable Prometheus metrics")
63 + parser.add_argument(
64 + "--url",
65 + type=str,
66 + default="0.0.0.0",
67 + help="URL to the Prometheus metrics server "
68 + "(only needed if enable-metrics is set).",
69 + )
70 + parser.add_argument(
71 + "--port",
72 + type=int,
73 + default=8000,
74 + help="Port number for the Prometheus metrics server "
75 + "(only needed if enable-metrics is set).",
76 + )
77 +
78 return parser.parse_args()
79
80
@@ -184,4 +203,12 @@ if __name__ == "__main__":
203 logger.info("vLLM batch processing API version %s", VLLM_VERSION)
204 logger.info("args: %s", args)
205
206 + # Start the Prometheus metrics server. LLMEngine uses the Prometheus client
207 + # to publish metrics at the /metrics endpoint.
208 + if args.enable_metrics:
209 + logger.info("Prometheus metrics enabled")
210 + start_http_server(port=args.port, addr=args.url)
211 + else:
212 + logger.info("Prometheus metrics disabled")
213 +
214 asyncio.run(main(args))