A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Port metrics from `aioprometheus` to `prometheus_client` (#2730)
Harry Mellor committed
Feb 25, 2024 at 19:54 UTC
ef978fe4111b0eb91c81eceba4d9791b94c7ffbf
9 files changed
+133
-87
docs/source/conf.py
+1
-1
@@ -72,7 +72,7 @@ html_theme_options = {
72
73
# Mock out external dependencies here.
74
autodoc_mock_imports = [
75
- "torch", "transformers", "psutil", "aioprometheus", "sentencepiece",
75
+ "torch", "transformers", "psutil", "prometheus_client", "sentencepiece",
76
"vllm.cuda_utils", "vllm._C"
77
]
78
requirements-neuron.txt
+1
-1
@@ -6,4 +6,4 @@ neuronx-cc
6
fastapi
7
uvicorn[standard]
8
pydantic >= 2.0 # Required for OpenAI server.
9
-aioprometheus[starlette]
9
+prometheus_client
requirements-rocm.txt
+1
-1
@@ -10,4 +10,4 @@ transformers >= 4.38.0 # Required for Gemma.
10
fastapi
11
uvicorn[standard]
12
pydantic >= 2.0 # Required for OpenAI server.
13
-aioprometheus[starlette]
13
+prometheus_client
requirements.txt
+1
-1
@@ -9,7 +9,7 @@ xformers == 0.0.23.post1 # Required for CUDA 12.1.
9
fastapi
10
uvicorn[standard]
11
pydantic >= 2.0 # Required for OpenAI server.
12
-aioprometheus[starlette]
12
+prometheus_client
13
pynvml == 11.5.0
14
triton >= 2.1.0
15
cupy-cuda12x == 12.1.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
tests/conftest.py
+2
@@ -165,6 +165,7 @@ class VllmRunner:
165
dtype: str = "half",
166
disable_log_stats: bool = True,
167
tensor_parallel_size: int = 1,
168
+ **kwargs,
169
) -> None:
170
self.model = LLM(
171
model=model_name,
@@ -174,6 +175,7 @@ class VllmRunner:
175
swap_space=0,
176
disable_log_stats=disable_log_stats,
177
tensor_parallel_size=tensor_parallel_size,
178
+ **kwargs,
179
)
180
181
def generate(
tests/metrics/test_metrics.py
+14
-11
@@ -1,5 +1,4 @@
1
import pytest
2
-import vllm.engine.metrics
2
3
MODELS = [
4
"facebook/opt-125m",
@@ -16,10 +15,10 @@ def test_metric_counter_prompt_tokens(
15
dtype: str,
16
max_tokens: int,
17
) -> None:
19
- # Reset metric
20
- vllm.engine.metrics.counter_prompt_tokens.set_value({}, 0)
21
-
22
- vllm_model = vllm_runner(model, dtype=dtype, disable_log_stats=False)
18
+ vllm_model = vllm_runner(model,
19
+ dtype=dtype,
20
+ disable_log_stats=False,
21
+ gpu_memory_utilization=0.4)
22
tokenizer = vllm_model.model.get_tokenizer()
23
prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
24
# This test needs at least 2 prompts in a batch of different lengths to verify their token count is correct despite padding.
@@ -29,7 +28,9 @@ def test_metric_counter_prompt_tokens(
28
vllm_prompt_token_count = sum(prompt_token_counts)
29
30
_ = vllm_model.generate_greedy(example_prompts, max_tokens)
32
- metric_count = vllm.engine.metrics.counter_prompt_tokens.get_value({})
31
+ stat_logger = vllm_model.model.llm_engine.stat_logger
32
+ metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
33
+ **stat_logger.labels)._value.get()
34
35
assert vllm_prompt_token_count == metric_count, (
36
f"prompt token count: {vllm_prompt_token_count!r}\nmetric: {metric_count!r}"
@@ -46,13 +47,15 @@ def test_metric_counter_generation_tokens(
47
dtype: str,
48
max_tokens: int,
49
) -> None:
49
- # Reset metric
50
- vllm.engine.metrics.counter_generation_tokens.set_value({}, 0)
51
-
52
- vllm_model = vllm_runner(model, dtype=dtype, disable_log_stats=False)
50
+ vllm_model = vllm_runner(model,
51
+ dtype=dtype,
52
+ disable_log_stats=False,
53
+ gpu_memory_utilization=0.4)
54
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
55
tokenizer = vllm_model.model.get_tokenizer()
55
- metric_count = vllm.engine.metrics.counter_generation_tokens.get_value({})
56
+ stat_logger = vllm_model.model.llm_engine.stat_logger
57
+ metric_count = stat_logger.metrics.counter_generation_tokens.labels(
58
+ **stat_logger.labels)._value.get()
59
vllm_generation_count = 0
60
for i in range(len(example_prompts)):
61
vllm_output_ids, vllm_output_str = vllm_outputs[i]
vllm/engine/llm_engine.py
+2
-1
@@ -128,7 +128,8 @@ class LLMEngine:
128
# Metric Logging.
129
if self.log_stats:
130
self.stat_logger = StatLogger(
131
- local_interval=_LOCAL_LOGGING_INTERVAL_SEC)
131
+ local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
132
+ labels=dict(model_name=model_config.model))
133
134
self.forward_dag = None
135
if USE_RAY_COMPILED_DAG:
vllm/engine/metrics.py
+107
-63
@@ -1,66 +1,94 @@
1
from vllm.logger import init_logger
2
-from aioprometheus import Counter, Gauge, Histogram
2
+from prometheus_client import Counter, Gauge, Histogram, REGISTRY, disable_created_metrics
3
4
import time
5
import numpy as np
6
-from typing import List
6
+from typing import Dict, List
7
from dataclasses import dataclass
8
9
logger = init_logger(__name__)
10
11
-labels = {}
12
-
13
-
14
-def add_global_metrics_labels(**kwargs):
15
- labels.update(kwargs)
16
-
11
+disable_created_metrics()
12
13
# The begin-* and end* here are used by the documentation generator
14
# to extract the metrics definitions.
15
16
+
17
# begin-metrics-definitions
22
-gauge_avg_prompt_throughput = Gauge("vllm:avg_prompt_throughput_toks_per_s",
23
- "Average prefill throughput in tokens/s.")
24
-gauge_avg_generation_throughput = Gauge(
25
- "vllm:avg_generation_throughput_toks_per_s",
26
- "Average generation throughput in tokens/s.")
27
-counter_prompt_tokens = Counter("vllm:prompt_tokens_total",
28
- "Number of prefill tokens processed.")
29
-counter_generation_tokens = Counter("vllm:generation_tokens_total",
30
- "Number of generation tokens processed.")
31
-
32
-gauge_scheduler_running = Gauge(
33
- "vllm:num_requests_running",
34
- "Number of requests currently running on GPU.")
35
-gauge_scheduler_swapped = Gauge("vllm:num_requests_swapped",
36
- "Number of requests swapped to CPU.")
37
-gauge_scheduler_waiting = Gauge("vllm:num_requests_waiting",
38
- "Number of requests waiting to be processed.")
39
-
40
-gauge_gpu_cache_usage = Gauge(
41
- "vllm:gpu_cache_usage_perc",
42
- "GPU KV-cache usage. 1 means 100 percent usage.")
43
-gauge_cpu_cache_usage = Gauge(
44
- "vllm:cpu_cache_usage_perc",
45
- "CPU KV-cache usage. 1 means 100 percent usage.")
46
-
47
-histogram_time_to_first_token = Histogram(
48
- "vllm:time_to_first_token_seconds",
49
- "Histogram of time to first token in seconds.",
50
- buckets=[
51
- 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0,
52
- 2.5, 5.0, 7.5, 10.0
53
- ])
54
-histogram_time_per_output_tokens = Histogram(
55
- "vllm:time_per_output_token_seconds",
56
- "Histogram of time per output token in seconds.",
57
- buckets=[
58
- 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5
59
- ])
60
-histogram_e2e_request_latency = Histogram(
61
- "vllm:e2e_request_latency_seconds",
62
- "Histogram of end to end request latency in seconds.",
63
- buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
18
+class Metrics:
19
+
20
+ def __init__(self, labelnames: List[str]):
21
+ # Unregister any existing vLLM collectors
22
+ for collector in list(REGISTRY._collector_to_names):
23
+ if hasattr(collector, "_name") and "vllm" in collector._name:
24
+ REGISTRY.unregister(collector)
25
+
26
+ # System stats
27
+ self.gauge_scheduler_running = Gauge(
28
+ name="vllm:num_requests_running",
29
+ documentation="Number of requests currently running on GPU.",
30
+ labelnames=labelnames)
31
+ self.gauge_scheduler_swapped = Gauge(
32
+ name="vllm:num_requests_swapped",
33
+ documentation="Number of requests swapped to CPU.",
34
+ labelnames=labelnames)
35
+ self.gauge_scheduler_waiting = Gauge(
36
+ name="vllm:num_requests_waiting",
37
+ documentation="Number of requests waiting to be processed.",
38
+ labelnames=labelnames)
39
+ self.gauge_gpu_cache_usage = Gauge(
40
+ name="vllm:gpu_cache_usage_perc",
41
+ documentation="GPU KV-cache usage. 1 means 100 percent usage.",
42
+ labelnames=labelnames)
43
+ self.gauge_cpu_cache_usage = Gauge(
44
+ name="vllm:cpu_cache_usage_perc",
45
+ documentation="CPU KV-cache usage. 1 means 100 percent usage.",
46
+ labelnames=labelnames)
47
+
48
+ # Raw stats from last model iteration
49
+ self.counter_prompt_tokens = Counter(
50
+ name="vllm:prompt_tokens_total",
51
+ documentation="Number of prefill tokens processed.",
52
+ labelnames=labelnames)
53
+ self.counter_generation_tokens = Counter(
54
+ name="vllm:generation_tokens_total",
55
+ documentation="Number of generation tokens processed.",
56
+ labelnames=labelnames)
57
+ self.histogram_time_to_first_token = Histogram(
58
+ name="vllm:time_to_first_token_seconds",
59
+ documentation="Histogram of time to first token in seconds.",
60
+ labelnames=labelnames,
61
+ buckets=[
62
+ 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
63
+ 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
64
+ ])
65
+ self.histogram_time_per_output_token = Histogram(
66
+ name="vllm:time_per_output_token_seconds",
67
+ documentation="Histogram of time per output token in seconds.",
68
+ labelnames=labelnames,
69
+ buckets=[
70
+ 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
71
+ 1.0, 2.5
72
+ ])
73
+ self.histogram_e2e_request_latency = Histogram(
74
+ name="vllm:e2e_request_latency_seconds",
75
+ documentation="Histogram of end to end request latency in seconds.",
76
+ labelnames=labelnames,
77
+ buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
78
+
79
+ # Legacy metrics
80
+ self.gauge_avg_prompt_throughput = Gauge(
81
+ name="vllm:avg_prompt_throughput_toks_per_s",
82
+ documentation="Average prefill throughput in tokens/s.",
83
+ labelnames=labelnames,
84
+ )
85
+ self.gauge_avg_generation_throughput = Gauge(
86
+ name="vllm:avg_generation_throughput_toks_per_s",
87
+ documentation="Average generation throughput in tokens/s.",
88
+ labelnames=labelnames,
89
+ )
90
+
91
+
92
# end-metrics-definitions
93
94
@@ -87,7 +115,7 @@ class Stats:
115
class StatLogger:
116
"""StatLogger is used LLMEngine to log to Promethus and Stdout."""
117
90
- def __init__(self, local_interval: float) -> None:
118
+ def __init__(self, local_interval: float, labels: Dict[str, str]) -> None:
119
# Metadata for logging locally.
120
self.last_local_log = time.monotonic()
121
self.local_interval = local_interval
@@ -96,6 +124,10 @@ class StatLogger:
124
self.num_prompt_tokens: List[int] = []
125
self.num_generation_tokens: List[int] = []
126
127
+ # Prometheus metrics
128
+ self.labels = labels
129
+ self.metrics = Metrics(labelnames=list(labels.keys()))
130
+
131
def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
132
return float(np.sum(tracked_stats) / (now - self.last_local_log))
133
@@ -105,23 +137,33 @@ class StatLogger:
137
138
def _log_prometheus(self, stats: Stats) -> None:
139
# Set system stat gauges.
108
- gauge_scheduler_running.set(labels, stats.num_running)
109
- gauge_scheduler_swapped.set(labels, stats.num_swapped)
110
- gauge_scheduler_waiting.set(labels, stats.num_waiting)
111
- gauge_gpu_cache_usage.set(labels, stats.gpu_cache_usage)
112
- gauge_cpu_cache_usage.set(labels, stats.cpu_cache_usage)
140
+ self.metrics.gauge_scheduler_running.labels(**self.labels).set(
141
+ stats.num_running)
142
+ self.metrics.gauge_scheduler_swapped.labels(**self.labels).set(
143
+ stats.num_swapped)
144
+ self.metrics.gauge_scheduler_waiting.labels(**self.labels).set(
145
+ stats.num_waiting)
146
+ self.metrics.gauge_gpu_cache_usage.labels(**self.labels).set(
147
+ stats.gpu_cache_usage)
148
+ self.metrics.gauge_cpu_cache_usage.labels(**self.labels).set(
149
+ stats.cpu_cache_usage)
150
151
# Add to token counters.
115
- counter_prompt_tokens.add(labels, stats.num_prompt_tokens)
116
- counter_generation_tokens.add(labels, stats.num_generation_tokens)
152
+ self.metrics.counter_prompt_tokens.labels(**self.labels).inc(
153
+ stats.num_prompt_tokens)
154
+ self.metrics.counter_generation_tokens.labels(**self.labels).inc(
155
+ stats.num_generation_tokens)
156
157
# Observe request level latencies in histograms.
158
for ttft in stats.time_to_first_tokens:
120
- histogram_time_to_first_token.observe(labels, ttft)
159
+ self.metrics.histogram_time_to_first_token.labels(
160
+ **self.labels).observe(ttft)
161
for tpot in stats.time_per_output_tokens:
122
- histogram_time_per_output_tokens.observe(labels, tpot)
162
+ self.metrics.histogram_time_per_output_token.labels(
163
+ **self.labels).observe(tpot)
164
for e2e in stats.time_e2e_requests:
124
- histogram_e2e_request_latency.observe(labels, e2e)
165
+ self.metrics.histogram_e2e_request_latency.labels(
166
+ **self.labels).observe(e2e)
167
168
def _log_prometheus_interval(self, prompt_throughput: float,
169
generation_throughput: float) -> None:
@@ -130,8 +172,10 @@ class StatLogger:
172
# Moving forward, we should use counters like counter_prompt_tokens, counter_generation_tokens
173
# Which log raw data and calculate summaries using rate() on the grafana/prometheus side.
174
# See https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
133
- gauge_avg_prompt_throughput.set(labels, prompt_throughput)
134
- gauge_avg_generation_throughput.set(labels, generation_throughput)
175
+ self.metrics.gauge_avg_prompt_throughput.labels(
176
+ **self.labels).set(prompt_throughput)
177
+ self.metrics.gauge_avg_generation_throughput.labels(
178
+ **self.labels).set(generation_throughput)
179
180
def log(self, stats: Stats) -> None:
181
"""Called by LLMEngine.
vllm/entrypoints/openai/api_server.py
+4
-8
@@ -6,8 +6,7 @@ import os
6
import importlib
7
import inspect
8
9
-from aioprometheus import MetricsMiddleware
10
-from aioprometheus.asgi.starlette import metrics
9
+from prometheus_client import make_asgi_app
10
import fastapi
11
import uvicorn
12
from http import HTTPStatus
@@ -18,7 +17,6 @@ from fastapi.responses import JSONResponse, StreamingResponse, Response
17
18
from vllm.engine.arg_utils import AsyncEngineArgs
19
from vllm.engine.async_llm_engine import AsyncLLMEngine
21
-from vllm.engine.metrics import add_global_metrics_labels
20
from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
21
from vllm.logger import init_logger
22
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
@@ -141,8 +139,9 @@ def parse_args():
139
return parser.parse_args()
140
141
144
-app.add_middleware(MetricsMiddleware) # Trace HTTP server metrics
145
-app.add_route("/metrics", metrics) # Exposes HTTP metrics
142
+# Add prometheus asgi middleware to route /metrics requests
143
+metrics_app = make_asgi_app()
144
+app.mount("/metrics", metrics_app)
145
146
147
@app.exception_handler(RequestValidationError)
@@ -242,9 +241,6 @@ if __name__ == "__main__":
241
openai_serving_completion = OpenAIServingCompletion(
242
engine, served_model, args.lora_modules)
243
245
- # Register labels for metrics
246
- add_global_metrics_labels(model_name=engine_args.model)
247
-
244
app.root_path = args.root_path
245
uvicorn.run(app,
246
host=args.host,