A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Metrics] Hide deprecated metrics (#15458)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Apr 2, 2025 at 15:37 UTC
98d7367b614bf4bb2189e849723df24156de67d0
4 files changed
+114
-82
tests/entrypoints/openai/test_metrics.py
+17
-4
@@ -13,9 +13,12 @@ import requests
13
from prometheus_client.parser import text_string_to_metric_families
14
from transformers import AutoTokenizer
15
16
+from vllm import version
17
+
18
from ...utils import RemoteOpenAIServer
19
20
MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
21
+PREV_MINOR_VERSION = version._prev_minor_version()
22
23
24
@pytest.fixture(scope="module", params=[True, False])
@@ -55,6 +58,7 @@ def default_server_args():
58
"",
59
"--enable-chunked-prefill",
60
"--disable-frontend-multiprocessing",
61
+ f"--show-hidden-metrics-for-version={PREV_MINOR_VERSION}",
62
])
63
def server(use_v1, default_server_args, request):
64
if request.param:
@@ -129,7 +133,9 @@ async def test_metrics_counts(server: RemoteOpenAIServer,
133
134
# Loop over all expected metric_families
135
for metric_family, suffix_values_list in EXPECTED_VALUES.items():
132
- if use_v1 and metric_family not in EXPECTED_METRICS_V1:
136
+ if ((use_v1 and metric_family not in EXPECTED_METRICS_V1)
137
+ or (not server.show_hidden_metrics
138
+ and metric_family in HIDDEN_DEPRECATED_METRICS)):
139
continue
140
141
found_metric = False
@@ -165,10 +171,10 @@ async def test_metrics_counts(server: RemoteOpenAIServer,
171
172
EXPECTED_METRICS = [
173
"vllm:num_requests_running",
168
- "vllm:num_requests_swapped",
174
+ "vllm:num_requests_swapped", # deprecated
175
"vllm:num_requests_waiting",
176
"vllm:gpu_cache_usage_perc",
171
- "vllm:cpu_cache_usage_perc",
177
+ "vllm:cpu_cache_usage_perc", # deprecated
178
"vllm:time_to_first_token_seconds_sum",
179
"vllm:time_to_first_token_seconds_bucket",
180
"vllm:time_to_first_token_seconds_count",
@@ -268,6 +274,11 @@ EXPECTED_METRICS_V1 = [
274
"vllm:request_decode_time_seconds_count",
275
]
276
277
+HIDDEN_DEPRECATED_METRICS = [
278
+ "vllm:num_requests_swapped",
279
+ "vllm:cpu_cache_usage_perc",
280
+]
281
+
282
283
@pytest.mark.asyncio
284
async def test_metrics_exist(server: RemoteOpenAIServer,
@@ -282,7 +293,9 @@ async def test_metrics_exist(server: RemoteOpenAIServer,
293
assert response.status_code == HTTPStatus.OK
294
295
for metric in (EXPECTED_METRICS_V1 if use_v1 else EXPECTED_METRICS):
285
- assert metric in response.text
296
+ if (not server.show_hidden_metrics
297
+ and metric not in HIDDEN_DEPRECATED_METRICS):
298
+ assert metric in response.text
299
300
301
def test_metrics_exist_run_batch(use_v1: bool):
tests/utils.py
+3
@@ -104,6 +104,9 @@ class RemoteOpenAIServer:
104
self.host = str(args.host or 'localhost')
105
self.port = int(args.port)
106
107
+ self.show_hidden_metrics = \
108
+ args.show_hidden_metrics_for_version is not None
109
+
110
# download the model before starting the server to avoid timeout
111
is_local = os.path.isdir(model)
112
if not is_local:
vllm/engine/metrics.py
+85
-78
@@ -52,6 +52,11 @@ class Metrics:
52
53
max_model_len = vllm_config.model_config.max_model_len
54
55
+ # Use this flag to hide metrics that were deprecated in
56
+ # a previous release and which will be removed future
57
+ self.show_hidden_metrics = \
58
+ vllm_config.observability_config.show_hidden_metrics
59
+
60
# System stats
61
# Scheduler State
62
self.gauge_scheduler_running = self._gauge_cls(
@@ -76,14 +81,15 @@ class Metrics:
81
)
82
83
# Deprecated in 0.8 - KV cache offloading is not used in V1
79
- # TODO: in 0.9, only enable if show_hidden_metrics=True
80
- self.gauge_scheduler_swapped = self._gauge_cls(
81
- name="vllm:num_requests_swapped",
82
- documentation=(
83
- "Number of requests swapped to CPU. "
84
- "DEPRECATED: KV cache offloading is not used in V1"),
85
- labelnames=labelnames,
86
- multiprocess_mode="sum")
84
+ # Hidden in 0.9, due to be removed in 0.10
85
+ if self.show_hidden_metrics:
86
+ self.gauge_scheduler_swapped = self._gauge_cls(
87
+ name="vllm:num_requests_swapped",
88
+ documentation=(
89
+ "Number of requests swapped to CPU. "
90
+ "DEPRECATED: KV cache offloading is not used in V1"),
91
+ labelnames=labelnames,
92
+ multiprocess_mode="sum")
93
94
# KV Cache Usage in %
95
self.gauge_gpu_cache_usage = self._gauge_cls(
@@ -93,34 +99,33 @@ class Metrics:
99
multiprocess_mode="sum")
100
101
# Deprecated in 0.8 - KV cache offloading is not used in V1
96
- # TODO: in 0.9, only enable if show_hidden_metrics=True
97
- self.gauge_cpu_cache_usage = self._gauge_cls(
98
- name="vllm:cpu_cache_usage_perc",
99
- documentation=(
100
- "CPU KV-cache usage. 1 means 100 percent usage. "
101
- "DEPRECATED: KV cache offloading is not used in V1"),
102
- labelnames=labelnames,
103
- multiprocess_mode="sum")
104
-
105
- # Deprecated in 0.8 - KV cache offloading is not used in V1
106
- # TODO: in 0.9, only enable if show_hidden_metrics=True
107
- self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
108
- name="vllm:cpu_prefix_cache_hit_rate",
109
- documentation=(
110
- "CPU prefix cache block hit rate. "
111
- "DEPRECATED: KV cache offloading is not used in V1"),
112
- labelnames=labelnames,
113
- multiprocess_mode="sum")
102
+ # Hidden in 0.9, due to be removed in 0.10
103
+ if self.show_hidden_metrics:
104
+ self.gauge_cpu_cache_usage = self._gauge_cls(
105
+ name="vllm:cpu_cache_usage_perc",
106
+ documentation=(
107
+ "CPU KV-cache usage. 1 means 100 percent usage. "
108
+ "DEPRECATED: KV cache offloading is not used in V1"),
109
+ labelnames=labelnames,
110
+ multiprocess_mode="sum")
111
+ self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
112
+ name="vllm:cpu_prefix_cache_hit_rate",
113
+ documentation=(
114
+ "CPU prefix cache block hit rate. "
115
+ "DEPRECATED: KV cache offloading is not used in V1"),
116
+ labelnames=labelnames,
117
+ multiprocess_mode="sum")
118
119
# Deprecated in 0.8 - replaced by queries+hits counters in V1
116
- # TODO: in 0.9, only enable if show_hidden_metrics=True
117
- self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
118
- name="vllm:gpu_prefix_cache_hit_rate",
119
- documentation=("GPU prefix cache block hit rate. "
120
- "DEPRECATED: use vllm:gpu_prefix_cache_queries and "
121
- "vllm:gpu_prefix_cache_queries in V1"),
122
- labelnames=labelnames,
123
- multiprocess_mode="sum")
120
+ # Hidden in 0.9, due to be removed in 0.10
121
+ if self.show_hidden_metrics:
122
+ self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
123
+ name="vllm:gpu_prefix_cache_hit_rate",
124
+ documentation=("GPU prefix cache block hit rate. "
125
+ "DEPRECATED: use vllm:gpu_prefix_cache_queries "
126
+ "and vllm:gpu_prefix_cache_queries in V1"),
127
+ labelnames=labelnames,
128
+ multiprocess_mode="sum")
129
130
# Iteration stats
131
self.counter_num_preemption = self._counter_cls(
@@ -198,33 +203,35 @@ class Metrics:
203
labelnames=labelnames,
204
buckets=request_latency_buckets)
205
# Deprecated in 0.8 - duplicates vllm:request_queue_time_seconds:
201
- # TODO: in 0.9, only enable if show_hidden_metrics=True
202
- self.histogram_time_in_queue_request = self._histogram_cls(
203
- name="vllm:time_in_queue_requests",
204
- documentation=(
205
- "Histogram of time the request spent in the queue in seconds. "
206
- "DEPRECATED: use vllm:request_queue_time_seconds instead."),
207
- labelnames=labelnames,
208
- buckets=request_latency_buckets)
206
+ # Hidden in 0.9, due to be removed in 0.10
207
+ if self.show_hidden_metrics:
208
+ self.histogram_time_in_queue_request = self._histogram_cls(
209
+ name="vllm:time_in_queue_requests",
210
+ documentation=
211
+ ("Histogram of time the request spent in the queue in seconds. "
212
+ "DEPRECATED: use vllm:request_queue_time_seconds instead."),
213
+ labelnames=labelnames,
214
+ buckets=request_latency_buckets)
215
216
# Deprecated in 0.8 - use prefill/decode/inference time metrics
211
- # TODO: in 0.9, only enable if show_hidden_metrics=True
212
- self.histogram_model_forward_time_request = self._histogram_cls(
213
- name="vllm:model_forward_time_milliseconds",
214
- documentation=(
215
- "Histogram of time spent in the model forward pass in ms. "
216
- "DEPRECATED: use prefill/decode/inference time metrics instead."
217
- ),
218
- labelnames=labelnames,
219
- buckets=build_1_2_3_5_8_buckets(3000))
220
- self.histogram_model_execute_time_request = self._histogram_cls(
221
- name="vllm:model_execute_time_milliseconds",
222
- documentation=(
223
- "Histogram of time spent in the model execute function in ms."
224
- "DEPRECATED: use prefill/decode/inference time metrics instead."
225
- ),
226
- labelnames=labelnames,
227
- buckets=build_1_2_3_5_8_buckets(3000))
217
+ # Hidden in 0.9, due to be removed in 0.10
218
+ if self.show_hidden_metrics:
219
+ self.histogram_model_forward_time_request = self._histogram_cls(
220
+ name="vllm:model_forward_time_milliseconds",
221
+ documentation=
222
+ ("Histogram of time spent in the model forward pass in ms. "
223
+ "DEPRECATED: use prefill/decode/inference time metrics instead"
224
+ ),
225
+ labelnames=labelnames,
226
+ buckets=build_1_2_3_5_8_buckets(3000))
227
+ self.histogram_model_execute_time_request = self._histogram_cls(
228
+ name="vllm:model_execute_time_milliseconds",
229
+ documentation=
230
+ ("Histogram of time spent in the model execute function in ms."
231
+ "DEPRECATED: use prefill/decode/inference time metrics instead"
232
+ ),
233
+ labelnames=labelnames,
234
+ buckets=build_1_2_3_5_8_buckets(3000))
235
236
# Metadata
237
self.histogram_num_prompt_tokens_request = self._histogram_cls(
@@ -543,11 +550,6 @@ class PrometheusStatLogger(StatLoggerBase):
550
self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
551
vllm_config=vllm_config)
552
546
- # Use this flag to hide metrics that were deprecated in
547
- # a previous release and which will be removed future
548
- self.show_hidden_metrics = \
549
- vllm_config.observability_config.show_hidden_metrics
550
-
553
def _log_gauge(self, gauge, data: Union[int, float]) -> None:
554
# Convenience function for logging to gauge.
555
gauge.labels(**self.labels).set(data)
@@ -580,18 +582,20 @@ class PrometheusStatLogger(StatLoggerBase):
582
# System state data
583
self._log_gauge(self.metrics.gauge_scheduler_running,
584
stats.num_running_sys)
583
- self._log_gauge(self.metrics.gauge_scheduler_swapped,
584
- stats.num_swapped_sys)
585
+ if self.metrics.show_hidden_metrics:
586
+ self._log_gauge(self.metrics.gauge_scheduler_swapped,
587
+ stats.num_swapped_sys)
588
self._log_gauge(self.metrics.gauge_scheduler_waiting,
589
stats.num_waiting_sys)
590
self._log_gauge(self.metrics.gauge_gpu_cache_usage,
591
stats.gpu_cache_usage_sys)
589
- self._log_gauge(self.metrics.gauge_cpu_cache_usage,
590
- stats.cpu_cache_usage_sys)
591
- self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
592
- stats.cpu_prefix_cache_hit_rate)
593
- self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
594
- stats.gpu_prefix_cache_hit_rate)
592
+ if self.metrics.show_hidden_metrics:
593
+ self._log_gauge(self.metrics.gauge_cpu_cache_usage,
594
+ stats.cpu_cache_usage_sys)
595
+ self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
596
+ stats.cpu_prefix_cache_hit_rate)
597
+ self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
598
+ stats.gpu_prefix_cache_hit_rate)
599
# Including max-lora in metric, in future this property of lora
600
# config maybe extended to be dynamic.
601
lora_info = {
@@ -629,12 +633,15 @@ class PrometheusStatLogger(StatLoggerBase):
633
stats.time_prefill_requests)
634
self._log_histogram(self.metrics.histogram_decode_time_request,
635
stats.time_decode_requests)
632
- self._log_histogram(self.metrics.histogram_time_in_queue_request,
633
- stats.time_in_queue_requests)
634
- self._log_histogram(self.metrics.histogram_model_forward_time_request,
635
- stats.model_forward_time_requests)
636
- self._log_histogram(self.metrics.histogram_model_execute_time_request,
637
- stats.model_execute_time_requests)
636
+ if self.metrics.show_hidden_metrics:
637
+ self._log_histogram(self.metrics.histogram_time_in_queue_request,
638
+ stats.time_in_queue_requests)
639
+ self._log_histogram(
640
+ self.metrics.histogram_model_forward_time_request,
641
+ stats.model_forward_time_requests)
642
+ self._log_histogram(
643
+ self.metrics.histogram_model_execute_time_request,
644
+ stats.model_execute_time_requests)
645
# Metadata
646
finished_reason_counter = CollectionsCounter(
647
stats.finished_reason_requests)
vllm/version.py
+9
@@ -28,4 +28,13 @@ def _prev_minor_version_was(version_str):
28
return True
29
30
# Note - this won't do the right thing when we release 1.0!
31
+ assert __version_tuple__[0] == 0
32
+ assert isinstance(__version_tuple__[1], int)
33
return version_str == f"{__version_tuple__[0]}.{__version_tuple__[1] - 1}"
34
+
35
+
36
+def _prev_minor_version():
37
+ """For the purpose of testing, return a previous minor version number."""
38
+ # In dev tree, this will return "0.-1", but that will work fine"
39
+ assert isinstance(__version_tuple__[1], int)
40
+ return f"{__version_tuple__[0]}.{__version_tuple__[1] - 1}"