@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01ba927040
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Metrics] Add initial Prometheus logger (#12416)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Jan 27, 2025 at 17:26 UTC 01ba927040d0b6f7d8daf6bfbf32fde562d2f8a6
3 files changed +78 -10
tests/entrypoints/openai/test_metrics.py
+35 -6
@@ -16,6 +16,24 @@ from ...utils import RemoteOpenAIServer
16 MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
17
18
19 +@pytest.fixture(scope="module", params=[True, False])
20 +def use_v1(request):
21 + # Module-scoped variant of run_with_both_engines
22 + #
23 + # Use this fixture to run a test with both v0 and v1, and
24 + # also to conditionalize the test logic e.g.
25 + #
26 + # def test_metrics_exist(use_v1, server, client):
27 + # ...
28 + # expected = EXPECTED_V1_METRICS if use_v1 else EXPECTED_METRICS
29 + # for metric in expected:
30 + # assert metric in response.text
31 + #
32 + # @skip_v1 wouldn't work here because this is a module-level
33 + # fixture - per-function decorators would have no effect
34 + yield request.param
35 +
36 +
37 @pytest.fixture(scope="module")
38 def default_server_args():
39 return [
@@ -36,10 +54,12 @@ def default_server_args():
54 "--enable-chunked-prefill",
55 "--disable-frontend-multiprocessing",
56 ])
39 -def server(default_server_args, request):
57 +def server(use_v1, default_server_args, request):
58 if request.param:
59 default_server_args.append(request.param)
42 - with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
60 + env_dict = dict(VLLM_USE_V1='1' if use_v1 else '0')
61 + with RemoteOpenAIServer(MODEL_NAME, default_server_args,
62 + env_dict=env_dict) as remote_server:
63 yield remote_server
64
65
@@ -84,7 +104,9 @@ EXPECTED_VALUES = {
104
105 @pytest.mark.asyncio
106 async def test_metrics_counts(server: RemoteOpenAIServer,
87 - client: openai.AsyncClient):
107 + client: openai.AsyncClient, use_v1: bool):
108 + if use_v1:
109 + pytest.skip("Skipping test on vllm V1")
110 for _ in range(_NUM_REQUESTS):
111 # sending a request triggers the metrics to be logged.
112 await client.completions.create(
@@ -174,10 +196,15 @@ EXPECTED_METRICS = [
196 "swap_space_bytes",
197 ]
198
199 +EXPECTED_METRICS_V1 = [
200 + "vllm:num_requests_running",
201 + "vllm:num_requests_waiting",
202 +]
203 +
204
205 @pytest.mark.asyncio
206 async def test_metrics_exist(server: RemoteOpenAIServer,
180 - client: openai.AsyncClient):
207 + client: openai.AsyncClient, use_v1: bool):
208 # sending a request triggers the metrics to be logged.
209 await client.completions.create(model=MODEL_NAME,
210 prompt="Hello, my name is",
@@ -187,11 +214,13 @@ async def test_metrics_exist(server: RemoteOpenAIServer,
214 response = requests.get(server.url_for("metrics"))
215 assert response.status_code == HTTPStatus.OK
216
190 - for metric in EXPECTED_METRICS:
217 + for metric in (EXPECTED_METRICS_V1 if use_v1 else EXPECTED_METRICS):
218 assert metric in response.text
219
220
194 -def test_metrics_exist_run_batch():
221 +def test_metrics_exist_run_batch(use_v1: bool):
222 + if use_v1:
223 + pytest.skip("Skipping test on vllm V1")
224 input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/e5-mistral-7b-instruct", "input": "You are a helpful assistant."}}""" # noqa: E501
225
226 base_url = "0.0.0.0"
vllm/v1/engine/async_llm.py
+7 -4
@@ -24,7 +24,8 @@ from vllm.v1.engine.core_client import EngineCoreClient
24 from vllm.v1.engine.output_processor import OutputProcessor
25 from vllm.v1.engine.processor import Processor
26 from vllm.v1.executor.abstract import Executor
27 -from vllm.v1.metrics.loggers import LoggingStatLogger, StatLoggerBase
27 +from vllm.v1.metrics.loggers import (LoggingStatLogger, PrometheusStatLogger,
28 + StatLoggerBase)
29 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
30
31 logger = init_logger(__name__)
@@ -46,13 +47,15 @@ class AsyncLLM(EngineClient):
47
48 assert start_engine_loop
49
50 + self.model_config = vllm_config.model_config
51 +
52 self.log_requests = log_requests
53 self.log_stats = log_stats
54 self.stat_loggers: List[StatLoggerBase] = [
55 LoggingStatLogger(),
53 - # TODO(rob): PrometheusStatLogger(),
56 + PrometheusStatLogger(labels=dict(
57 + model_name=self.model_config.served_model_name)),
58 ]
55 - self.model_config = vllm_config.model_config
59
60 # Tokenizer (+ ensure liveness if running in another process).
61 self.tokenizer = init_tokenizer_from_configs(
@@ -272,7 +275,7 @@ class AsyncLLM(EngineClient):
275
276 # 4) Logging.
277 # TODO(rob): make into a coroutine and launch it in
275 - # background thread once we add Prometheus.
278 + # background thread once Prometheus overhead is non-trivial.
279 assert iteration_stats is not None
280 self._log_stats(
281 scheduler_stats=outputs.scheduler_stats,
vllm/v1/metrics/loggers.py
+36
@@ -1,5 +1,8 @@
1 import time
2 from abc import ABC, abstractmethod
3 +from typing import Dict
4 +
5 +import prometheus_client
6
7 from vllm.logger import init_logger
8 from vllm.v1.metrics.stats import SchedulerStats
@@ -36,3 +39,36 @@ class LoggingStatLogger(StatLoggerBase):
39 scheduler_stats.num_running_reqs,
40 scheduler_stats.num_waiting_reqs,
41 )
42 +
43 +
44 +class PrometheusStatLogger(StatLoggerBase):
45 +
46 + def __init__(self, labels: Dict[str, str]):
47 + self.labels = labels
48 +
49 + labelnames = self.labels.keys()
50 + labelvalues = self.labels.values()
51 +
52 + self._unregister_vllm_metrics()
53 +
54 + self.gauge_scheduler_running = prometheus_client.Gauge(
55 + name="vllm:num_requests_running",
56 + documentation="Number of requests in model execution batches.",
57 + labelnames=labelnames).labels(*labelvalues)
58 +
59 + self.gauge_scheduler_waiting = prometheus_client.Gauge(
60 + name="vllm:num_requests_waiting",
61 + documentation="Number of requests waiting to be processed.",
62 + labelnames=labelnames).labels(*labelvalues)
63 +
64 + def log(self, scheduler_stats: SchedulerStats):
65 + """Log to prometheus."""
66 + self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
67 + self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
68 +
69 + @staticmethod
70 + def _unregister_vllm_metrics():
71 + # Unregister any existing vLLM collectors (for CI/CD
72 + for collector in list(prometheus_client.REGISTRY._collector_to_names):
73 + if hasattr(collector, "_name") and "vllm" in collector._name:
74 + prometheus_client.REGISTRY.unregister(collector)