A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Metrics] Add initial Prometheus logger (#12416)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Jan 27, 2025 at 17:26 UTC
01ba927040d0b6f7d8daf6bfbf32fde562d2f8a6
3 files changed
+78
-10
tests/entrypoints/openai/test_metrics.py
+35
-6
@@ -16,6 +16,24 @@ from ...utils import RemoteOpenAIServer
16
MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
17
18
19
+@pytest.fixture(scope="module", params=[True, False])
20
+def use_v1(request):
21
+ # Module-scoped variant of run_with_both_engines
22
+ #
23
+ # Use this fixture to run a test with both v0 and v1, and
24
+ # also to conditionalize the test logic e.g.
25
+ #
26
+ # def test_metrics_exist(use_v1, server, client):
27
+ # ...
28
+ # expected = EXPECTED_V1_METRICS if use_v1 else EXPECTED_METRICS
29
+ # for metric in expected:
30
+ # assert metric in response.text
31
+ #
32
+ # @skip_v1 wouldn't work here because this is a module-level
33
+ # fixture - per-function decorators would have no effect
34
+ yield request.param
35
+
36
+
37
@pytest.fixture(scope="module")
38
def default_server_args():
39
return [
@@ -36,10 +54,12 @@ def default_server_args():
54
"--enable-chunked-prefill",
55
"--disable-frontend-multiprocessing",
56
])
39
-def server(default_server_args, request):
57
+def server(use_v1, default_server_args, request):
58
if request.param:
59
default_server_args.append(request.param)
42
- with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
60
+ env_dict = dict(VLLM_USE_V1='1' if use_v1 else '0')
61
+ with RemoteOpenAIServer(MODEL_NAME, default_server_args,
62
+ env_dict=env_dict) as remote_server:
63
yield remote_server
64
65
@@ -84,7 +104,9 @@ EXPECTED_VALUES = {
104
105
@pytest.mark.asyncio
106
async def test_metrics_counts(server: RemoteOpenAIServer,
87
- client: openai.AsyncClient):
107
+ client: openai.AsyncClient, use_v1: bool):
108
+ if use_v1:
109
+ pytest.skip("Skipping test on vllm V1")
110
for _ in range(_NUM_REQUESTS):
111
# sending a request triggers the metrics to be logged.
112
await client.completions.create(
@@ -174,10 +196,15 @@ EXPECTED_METRICS = [
196
"swap_space_bytes",
197
]
198
199
+EXPECTED_METRICS_V1 = [
200
+ "vllm:num_requests_running",
201
+ "vllm:num_requests_waiting",
202
+]
203
+
204
205
@pytest.mark.asyncio
206
async def test_metrics_exist(server: RemoteOpenAIServer,
180
- client: openai.AsyncClient):
207
+ client: openai.AsyncClient, use_v1: bool):
208
# sending a request triggers the metrics to be logged.
209
await client.completions.create(model=MODEL_NAME,
210
prompt="Hello, my name is",
@@ -187,11 +214,13 @@ async def test_metrics_exist(server: RemoteOpenAIServer,
214
response = requests.get(server.url_for("metrics"))
215
assert response.status_code == HTTPStatus.OK
216
190
- for metric in EXPECTED_METRICS:
217
+ for metric in (EXPECTED_METRICS_V1 if use_v1 else EXPECTED_METRICS):
218
assert metric in response.text
219
220
194
-def test_metrics_exist_run_batch():
221
+def test_metrics_exist_run_batch(use_v1: bool):
222
+ if use_v1:
223
+ pytest.skip("Skipping test on vllm V1")
224
input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/e5-mistral-7b-instruct", "input": "You are a helpful assistant."}}""" # noqa: E501
225
226
base_url = "0.0.0.0"
vllm/v1/engine/async_llm.py
+7
-4
@@ -24,7 +24,8 @@ from vllm.v1.engine.core_client import EngineCoreClient
24
from vllm.v1.engine.output_processor import OutputProcessor
25
from vllm.v1.engine.processor import Processor
26
from vllm.v1.executor.abstract import Executor
27
-from vllm.v1.metrics.loggers import LoggingStatLogger, StatLoggerBase
27
+from vllm.v1.metrics.loggers import (LoggingStatLogger, PrometheusStatLogger,
28
+ StatLoggerBase)
29
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
30
31
logger = init_logger(__name__)
@@ -46,13 +47,15 @@ class AsyncLLM(EngineClient):
47
48
assert start_engine_loop
49
50
+ self.model_config = vllm_config.model_config
51
+
52
self.log_requests = log_requests
53
self.log_stats = log_stats
54
self.stat_loggers: List[StatLoggerBase] = [
55
LoggingStatLogger(),
53
- # TODO(rob): PrometheusStatLogger(),
56
+ PrometheusStatLogger(labels=dict(
57
+ model_name=self.model_config.served_model_name)),
58
]
55
- self.model_config = vllm_config.model_config
59
60
# Tokenizer (+ ensure liveness if running in another process).
61
self.tokenizer = init_tokenizer_from_configs(
@@ -272,7 +275,7 @@ class AsyncLLM(EngineClient):
275
276
# 4) Logging.
277
# TODO(rob): make into a coroutine and launch it in
275
- # background thread once we add Prometheus.
278
+ # background thread once Prometheus overhead is non-trivial.
279
assert iteration_stats is not None
280
self._log_stats(
281
scheduler_stats=outputs.scheduler_stats,
vllm/v1/metrics/loggers.py
+36
@@ -1,5 +1,8 @@
1
import time
2
from abc import ABC, abstractmethod
3
+from typing import Dict
4
+
5
+import prometheus_client
6
7
from vllm.logger import init_logger
8
from vllm.v1.metrics.stats import SchedulerStats
@@ -36,3 +39,36 @@ class LoggingStatLogger(StatLoggerBase):
39
scheduler_stats.num_running_reqs,
40
scheduler_stats.num_waiting_reqs,
41
)
42
+
43
+
44
+class PrometheusStatLogger(StatLoggerBase):
45
+
46
+ def __init__(self, labels: Dict[str, str]):
47
+ self.labels = labels
48
+
49
+ labelnames = self.labels.keys()
50
+ labelvalues = self.labels.values()
51
+
52
+ self._unregister_vllm_metrics()
53
+
54
+ self.gauge_scheduler_running = prometheus_client.Gauge(
55
+ name="vllm:num_requests_running",
56
+ documentation="Number of requests in model execution batches.",
57
+ labelnames=labelnames).labels(*labelvalues)
58
+
59
+ self.gauge_scheduler_waiting = prometheus_client.Gauge(
60
+ name="vllm:num_requests_waiting",
61
+ documentation="Number of requests waiting to be processed.",
62
+ labelnames=labelnames).labels(*labelvalues)
63
+
64
+ def log(self, scheduler_stats: SchedulerStats):
65
+ """Log to prometheus."""
66
+ self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
67
+ self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
68
+
69
+ @staticmethod
70
+ def _unregister_vllm_metrics():
71
+ # Unregister any existing vLLM collectors (for CI/CD
72
+ for collector in list(prometheus_client.REGISTRY._collector_to_names):
73
+ if hasattr(collector, "_name") and "vllm" in collector._name:
74
+ prometheus_client.REGISTRY.unregister(collector)