A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0 Deprecation] Remove V0 metrics code (#27215)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Oct 20, 2025 at 19:02 UTC
a03cf9bc704ca32a1fc5c946866f5382d5b73d5a
2 files changed
-772
vllm/engine/metrics.py
deleted
-688
@@ -1,688 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import time
5
-from collections import Counter as CollectionsCounter
6
-from typing import cast
7
-
8
-import numpy as np
9
-import prometheus_client
10
-
11
-from vllm.config import SupportsMetricsInfo, VllmConfig
12
-from vllm.engine.metrics_types import StatLoggerBase, Stats
13
-from vllm.executor.ray_utils import ray
14
-from vllm.logger import init_logger
15
-
16
-if ray is not None:
17
- from ray.util import metrics as ray_metrics
18
-else:
19
- ray_metrics = None
20
-
21
-logger = init_logger(__name__)
22
-
23
-prometheus_client.disable_created_metrics()
24
-
25
-# The begin-* and end* here are used by the documentation generator
26
-# to extract the metrics definitions.
27
-
28
-
29
-# --8<-- [start:metrics-definitions]
30
-class Metrics:
31
- """
32
- vLLM uses a multiprocessing-based frontend for the OpenAI server.
33
- This means that we need to run prometheus_client in multiprocessing mode
34
- See https://prometheus.github.io/client_python/multiprocess/ for more
35
- details on limitations.
36
- """
37
-
38
- labelname_finish_reason = "finished_reason"
39
- labelname_waiting_lora_adapters = "waiting_lora_adapters"
40
- labelname_running_lora_adapters = "running_lora_adapters"
41
- labelname_max_lora = "max_lora"
42
- _gauge_cls = prometheus_client.Gauge
43
- _counter_cls = prometheus_client.Counter
44
- _histogram_cls = prometheus_client.Histogram
45
-
46
- def __init__(self, labelnames: list[str], vllm_config: VllmConfig):
47
- # Unregister any existing vLLM collectors (for CI/CD)
48
- self._unregister_vllm_metrics()
49
-
50
- max_model_len = vllm_config.model_config.max_model_len
51
-
52
- # Use this flag to hide metrics that were deprecated in
53
- # a previous release and which will be removed future
54
- self.show_hidden_metrics = vllm_config.observability_config.show_hidden_metrics
55
-
56
- # System stats
57
- # Scheduler State
58
- self.gauge_scheduler_running = self._gauge_cls(
59
- name="vllm:num_requests_running",
60
- documentation="Number of requests currently running on GPU.",
61
- labelnames=labelnames,
62
- multiprocess_mode="sum",
63
- )
64
- self.gauge_scheduler_waiting = self._gauge_cls(
65
- name="vllm:num_requests_waiting",
66
- documentation="Number of requests waiting to be processed.",
67
- labelnames=labelnames,
68
- multiprocess_mode="sum",
69
- )
70
- self.gauge_lora_info = self._gauge_cls(
71
- name="vllm:lora_requests_info",
72
- documentation="Running stats on lora requests.",
73
- labelnames=[
74
- self.labelname_running_lora_adapters,
75
- self.labelname_max_lora,
76
- self.labelname_waiting_lora_adapters,
77
- ],
78
- multiprocess_mode="livemostrecent",
79
- )
80
-
81
- # KV Cache Usage in %
82
- self.gauge_gpu_cache_usage = self._gauge_cls(
83
- name="vllm:gpu_cache_usage_perc",
84
- documentation="GPU KV-cache usage. 1 means 100 percent usage.",
85
- labelnames=labelnames,
86
- multiprocess_mode="sum",
87
- )
88
-
89
- # Iteration stats
90
- self.counter_num_preemption = self._counter_cls(
91
- name="vllm:num_preemptions_total",
92
- documentation="Cumulative number of preemption from the engine.",
93
- labelnames=labelnames,
94
- )
95
- self.counter_prompt_tokens = self._counter_cls(
96
- name="vllm:prompt_tokens_total",
97
- documentation="Number of prefill tokens processed.",
98
- labelnames=labelnames,
99
- )
100
- self.counter_generation_tokens = self._counter_cls(
101
- name="vllm:generation_tokens_total",
102
- documentation="Number of generation tokens processed.",
103
- labelnames=labelnames,
104
- )
105
- self.histogram_iteration_tokens = self._histogram_cls(
106
- name="vllm:iteration_tokens_total",
107
- documentation="Histogram of number of tokens per engine_step.",
108
- labelnames=labelnames,
109
- buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
110
- )
111
- self.histogram_time_to_first_token = self._histogram_cls(
112
- name="vllm:time_to_first_token_seconds",
113
- documentation="Histogram of time to first token in seconds.",
114
- labelnames=labelnames,
115
- buckets=[
116
- 0.001,
117
- 0.005,
118
- 0.01,
119
- 0.02,
120
- 0.04,
121
- 0.06,
122
- 0.08,
123
- 0.1,
124
- 0.25,
125
- 0.5,
126
- 0.75,
127
- 1.0,
128
- 2.5,
129
- 5.0,
130
- 7.5,
131
- 10.0,
132
- 20.0,
133
- 40.0,
134
- 80.0,
135
- 160.0,
136
- 640.0,
137
- 2560.0,
138
- ],
139
- )
140
- # Deprecated in 0.11 - Renamed as vllm:inter_token_latency_seconds
141
- # TODO: in 0.12, only enable if show_hidden_metrics=True
142
- self.histogram_time_per_output_token = self._histogram_cls(
143
- name="vllm:time_per_output_token_seconds",
144
- documentation=(
145
- "Histogram of time per output token in seconds."
146
- "DEPRECATED: Use vllm:inter_token_latency_seconds instead."
147
- ),
148
- labelnames=labelnames,
149
- buckets=[
150
- 0.01,
151
- 0.025,
152
- 0.05,
153
- 0.075,
154
- 0.1,
155
- 0.15,
156
- 0.2,
157
- 0.3,
158
- 0.4,
159
- 0.5,
160
- 0.75,
161
- 1.0,
162
- 2.5,
163
- 5.0,
164
- 7.5,
165
- 10.0,
166
- 20.0,
167
- 40.0,
168
- 80.0,
169
- ],
170
- )
171
- self.histogram_inter_token_latency = self._histogram_cls(
172
- name="vllm:inter_token_latency_seconds",
173
- documentation="Histogram of inter token latency in seconds.",
174
- labelnames=labelnames,
175
- buckets=[
176
- 0.01,
177
- 0.025,
178
- 0.05,
179
- 0.075,
180
- 0.1,
181
- 0.15,
182
- 0.2,
183
- 0.3,
184
- 0.4,
185
- 0.5,
186
- 0.75,
187
- 1.0,
188
- 2.5,
189
- 5.0,
190
- 7.5,
191
- 10.0,
192
- 20.0,
193
- 40.0,
194
- 80.0,
195
- ],
196
- )
197
-
198
- # Request stats
199
- # Latency
200
- request_latency_buckets = [
201
- 0.3,
202
- 0.5,
203
- 0.8,
204
- 1.0,
205
- 1.5,
206
- 2.0,
207
- 2.5,
208
- 5.0,
209
- 10.0,
210
- 15.0,
211
- 20.0,
212
- 30.0,
213
- 40.0,
214
- 50.0,
215
- 60.0,
216
- 120.0,
217
- 240.0,
218
- 480.0,
219
- 960.0,
220
- 1920.0,
221
- 7680.0,
222
- ]
223
- self.histogram_e2e_time_request = self._histogram_cls(
224
- name="vllm:e2e_request_latency_seconds",
225
- documentation="Histogram of end to end request latency in seconds.",
226
- labelnames=labelnames,
227
- buckets=request_latency_buckets,
228
- )
229
- self.histogram_queue_time_request = self._histogram_cls(
230
- name="vllm:request_queue_time_seconds",
231
- documentation="Histogram of time spent in WAITING phase for request.",
232
- labelnames=labelnames,
233
- buckets=request_latency_buckets,
234
- )
235
- self.histogram_inference_time_request = self._histogram_cls(
236
- name="vllm:request_inference_time_seconds",
237
- documentation="Histogram of time spent in RUNNING phase for request.",
238
- labelnames=labelnames,
239
- buckets=request_latency_buckets,
240
- )
241
- self.histogram_prefill_time_request = self._histogram_cls(
242
- name="vllm:request_prefill_time_seconds",
243
- documentation="Histogram of time spent in PREFILL phase for request.",
244
- labelnames=labelnames,
245
- buckets=request_latency_buckets,
246
- )
247
- self.histogram_decode_time_request = self._histogram_cls(
248
- name="vllm:request_decode_time_seconds",
249
- documentation="Histogram of time spent in DECODE phase for request.",
250
- labelnames=labelnames,
251
- buckets=request_latency_buckets,
252
- )
253
-
254
- # Metadata
255
- self.histogram_num_prompt_tokens_request = self._histogram_cls(
256
- name="vllm:request_prompt_tokens",
257
- documentation="Number of prefill tokens processed.",
258
- labelnames=labelnames,
259
- buckets=build_1_2_5_buckets(max_model_len),
260
- )
261
- self.histogram_num_generation_tokens_request = self._histogram_cls(
262
- name="vllm:request_generation_tokens",
263
- documentation="Number of generation tokens processed.",
264
- labelnames=labelnames,
265
- buckets=build_1_2_5_buckets(max_model_len),
266
- )
267
- self.histogram_max_num_generation_tokens_request = self._histogram_cls(
268
- name="vllm:request_max_num_generation_tokens",
269
- documentation="Histogram of maximum number of requested generation tokens.",
270
- labelnames=labelnames,
271
- buckets=build_1_2_5_buckets(max_model_len),
272
- )
273
- self.histogram_n_request = self._histogram_cls(
274
- name="vllm:request_params_n",
275
- documentation="Histogram of the n request parameter.",
276
- labelnames=labelnames,
277
- buckets=[1, 2, 5, 10, 20],
278
- )
279
- self.histogram_max_tokens_request = self._histogram_cls(
280
- name="vllm:request_params_max_tokens",
281
- documentation="Histogram of the max_tokens request parameter.",
282
- labelnames=labelnames,
283
- buckets=build_1_2_5_buckets(max_model_len),
284
- )
285
- self.counter_request_success = self._counter_cls(
286
- name="vllm:request_success_total",
287
- documentation="Count of successfully processed requests.",
288
- labelnames=labelnames + [Metrics.labelname_finish_reason],
289
- )
290
-
291
- # --8<-- [end:metrics-definitions]
292
-
293
- def _unregister_vllm_metrics(self) -> None:
294
- for collector in list(prometheus_client.REGISTRY._collector_to_names):
295
- if hasattr(collector, "_name") and "vllm" in collector._name:
296
- prometheus_client.REGISTRY.unregister(collector)
297
-
298
-
299
-class _RayGaugeWrapper:
300
- """Wraps around ray.util.metrics.Gauge to provide same API as
301
- prometheus_client.Gauge"""
302
-
303
- def __init__(
304
- self,
305
- name: str,
306
- documentation: str = "",
307
- labelnames: list[str] | None = None,
308
- multiprocess_mode: str = "",
309
- ):
310
- del multiprocess_mode
311
- labelnames_tuple = tuple(labelnames) if labelnames else None
312
- self._gauge = ray_metrics.Gauge(
313
- name=name, description=documentation, tag_keys=labelnames_tuple
314
- )
315
-
316
- def labels(self, **labels):
317
- self._gauge.set_default_tags(labels)
318
- return self
319
-
320
- def set(self, value: int | float):
321
- return self._gauge.set(value)
322
-
323
- def set_to_current_time(self):
324
- # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
325
- return self._gauge.set(time.time())
326
-
327
-
328
-class _RayCounterWrapper:
329
- """Wraps around ray.util.metrics.Counter to provide same API as
330
- prometheus_client.Counter"""
331
-
332
- def __init__(
333
- self, name: str, documentation: str = "", labelnames: list[str] | None = None
334
- ):
335
- labelnames_tuple = tuple(labelnames) if labelnames else None
336
- self._counter = ray_metrics.Counter(
337
- name=name, description=documentation, tag_keys=labelnames_tuple
338
- )
339
-
340
- def labels(self, **labels):
341
- self._counter.set_default_tags(labels)
342
- return self
343
-
344
- def inc(self, value: int | float = 1.0):
345
- if value == 0:
346
- return
347
- return self._counter.inc(value)
348
-
349
-
350
-class _RayHistogramWrapper:
351
- """Wraps around ray.util.metrics.Histogram to provide same API as
352
- prometheus_client.Histogram"""
353
-
354
- def __init__(
355
- self,
356
- name: str,
357
- documentation: str = "",
358
- labelnames: list[str] | None = None,
359
- buckets: list[float] | None = None,
360
- ):
361
- labelnames_tuple = tuple(labelnames) if labelnames else None
362
- boundaries = buckets if buckets else []
363
- self._histogram = ray_metrics.Histogram(
364
- name=name,
365
- description=documentation,
366
- tag_keys=labelnames_tuple,
367
- boundaries=boundaries,
368
- )
369
-
370
- def labels(self, **labels):
371
- self._histogram.set_default_tags(labels)
372
- return self
373
-
374
- def observe(self, value: int | float):
375
- return self._histogram.observe(value)
376
-
377
-
378
-class RayMetrics(Metrics):
379
- """
380
- RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
381
- Provides the same metrics as Metrics but uses Ray's util.metrics library.
382
- """
383
-
384
- _gauge_cls: type[prometheus_client.Gauge] = cast(
385
- type[prometheus_client.Gauge], _RayGaugeWrapper
386
- )
387
- _counter_cls: type[prometheus_client.Counter] = cast(
388
- type[prometheus_client.Counter], _RayCounterWrapper
389
- )
390
- _histogram_cls: type[prometheus_client.Histogram] = cast(
391
- type[prometheus_client.Histogram], _RayHistogramWrapper
392
- )
393
-
394
- def __init__(self, labelnames: list[str], vllm_config: VllmConfig):
395
- if ray_metrics is None:
396
- raise ImportError("RayMetrics requires Ray to be installed.")
397
- super().__init__(labelnames, vllm_config)
398
-
399
- def _unregister_vllm_metrics(self) -> None:
400
- # No-op on purpose
401
- pass
402
-
403
-
404
-def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
405
- """
406
- Builds a list of buckets with increasing powers of 10 multiplied by
407
- mantissa values until the value exceeds the specified maximum.
408
-
409
- """
410
- exponent = 0
411
- buckets: list[int] = []
412
- while True:
413
- for m in mantissa_lst:
414
- value = m * 10**exponent
415
- if value <= max_value:
416
- buckets.append(value)
417
- else:
418
- return buckets
419
- exponent += 1
420
-
421
-
422
-def build_1_2_5_buckets(max_value: int) -> list[int]:
423
- """
424
- Example:
425
- >>> build_1_2_5_buckets(100)
426
- [1, 2, 5, 10, 20, 50, 100]
427
- """
428
- return build_buckets([1, 2, 5], max_value)
429
-
430
-
431
-def build_1_2_3_5_8_buckets(max_value: int) -> list[int]:
432
- """
433
- Example:
434
- >>> build_1_2_3_5_8_buckets(100)
435
- [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
436
- """
437
- return build_buckets([1, 2, 3, 5, 8], max_value)
438
-
439
-
440
-def local_interval_elapsed(now: float, last_log: float, local_interval: float) -> bool:
441
- elapsed_time = now - last_log
442
- return elapsed_time > local_interval
443
-
444
-
445
-def get_throughput(tracked_stats: list[int], now: float, last_log: float) -> float:
446
- return float(np.sum(tracked_stats) / (now - last_log))
447
-
448
-
449
-class LoggingStatLogger(StatLoggerBase):
450
- """LoggingStatLogger is used in LLMEngine to log to Stdout."""
451
-
452
- def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
453
- super().__init__(local_interval, vllm_config)
454
- self.last_prompt_throughput: float | None = None
455
- self.last_generation_throughput: float | None = None
456
-
457
- def log(self, stats: Stats) -> None:
458
- """Called by LLMEngine.
459
- Logs to Stdout every self.local_interval seconds."""
460
-
461
- # Save tracked stats for token counters.
462
- self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
463
- self.num_generation_tokens.append(stats.num_generation_tokens_iter)
464
-
465
- # Log locally every local_interval seconds.
466
- if local_interval_elapsed(stats.now, self.last_local_log, self.local_interval):
467
- # Compute summary metrics for tracked stats (and log them
468
- # to prometheus if applicable).
469
- prompt_throughput = get_throughput(
470
- self.num_prompt_tokens, now=stats.now, last_log=self.last_local_log
471
- )
472
- generation_throughput = get_throughput(
473
- self.num_generation_tokens, now=stats.now, last_log=self.last_local_log
474
- )
475
-
476
- log_fn = logger.info
477
- if not any(
478
- (
479
- prompt_throughput,
480
- generation_throughput,
481
- self.last_prompt_throughput,
482
- self.last_generation_throughput,
483
- )
484
- ):
485
- # Avoid log noise on an idle production system
486
- log_fn = logger.debug
487
-
488
- log_fn(
489
- "Avg prompt throughput: %.1f tokens/s, "
490
- "Avg generation throughput: %.1f tokens/s, "
491
- "Running: %d reqs, Swapped: %d reqs, "
492
- "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
493
- "CPU KV cache usage: %.1f%%.",
494
- prompt_throughput,
495
- generation_throughput,
496
- stats.num_running_sys,
497
- stats.num_swapped_sys,
498
- stats.num_waiting_sys,
499
- stats.gpu_cache_usage_sys * 100,
500
- stats.cpu_cache_usage_sys * 100,
501
- )
502
- if (
503
- stats.cpu_prefix_cache_hit_rate >= 0
504
- or stats.gpu_prefix_cache_hit_rate >= 0
505
- ):
506
- log_fn(
507
- "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
508
- stats.gpu_prefix_cache_hit_rate * 100,
509
- stats.cpu_prefix_cache_hit_rate * 100,
510
- )
511
-
512
- self._reset(stats, prompt_throughput, generation_throughput)
513
-
514
- def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
515
- # Reset tracked stats for next interval.
516
- self.num_prompt_tokens = []
517
- self.num_generation_tokens = []
518
- self.last_local_log = stats.now
519
- self.last_prompt_throughput = prompt_throughput
520
- self.last_generation_throughput = generation_throughput
521
-
522
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
523
- raise NotImplementedError
524
-
525
-
526
-class PrometheusStatLogger(StatLoggerBase):
527
- """PrometheusStatLogger is used LLMEngine to log to Prometheus."""
528
-
529
- _metrics_cls = Metrics
530
- _gauge_cls = prometheus_client.Gauge
531
-
532
- def __init__(
533
- self, local_interval: float, labels: dict[str, str], vllm_config: VllmConfig
534
- ) -> None:
535
- super().__init__(local_interval, vllm_config)
536
- # Prometheus metrics
537
- self.labels = labels
538
- self.metrics = self._metrics_cls(
539
- labelnames=list(labels.keys()), vllm_config=vllm_config
540
- )
541
-
542
- def _log_gauge(self, gauge, data: int | float) -> None:
543
- # Convenience function for logging to gauge.
544
- gauge.labels(**self.labels).set(data)
545
-
546
- def _log_counter(self, counter, data: int | float) -> None:
547
- # Convenience function for logging to counter.
548
- # Prevent ValueError from negative increment
549
- if data < 0:
550
- logger.warning("Skipping negative increment of %g to %s", data, counter)
551
- return
552
- counter.labels(**self.labels).inc(data)
553
-
554
- def _log_counter_labels(
555
- self, counter, data: CollectionsCounter, label_key: str
556
- ) -> None:
557
- # Convenience function for collection counter of labels.
558
- for label, count in data.items():
559
- counter.labels(**{**self.labels, label_key: label}).inc(count)
560
-
561
- def _log_histogram(self, histogram, data: list[int] | list[float]) -> None:
562
- # Convenience function for logging list to histogram.
563
- for datum in data:
564
- histogram.labels(**self.labels).observe(datum)
565
-
566
- def _log_gauge_string(self, gauge, data: dict[str, str]) -> None:
567
- gauge.labels(**data).set_to_current_time()
568
-
569
- def _log_prometheus(self, stats: Stats) -> None:
570
- # System state data
571
- self._log_gauge(self.metrics.gauge_scheduler_running, stats.num_running_sys)
572
- self._log_gauge(self.metrics.gauge_scheduler_waiting, stats.num_waiting_sys)
573
- self._log_gauge(self.metrics.gauge_gpu_cache_usage, stats.gpu_cache_usage_sys)
574
- # Including max-lora in metric, in future this property of lora
575
- # config maybe extended to be dynamic.
576
- lora_info = {
577
- self.metrics.labelname_running_lora_adapters: ",".join(
578
- stats.running_lora_adapters
579
- ),
580
- self.metrics.labelname_waiting_lora_adapters: ",".join(
581
- stats.waiting_lora_adapters
582
- ),
583
- self.metrics.labelname_max_lora: stats.max_lora,
584
- }
585
- self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
586
- # Iteration level data
587
- self._log_counter(
588
- self.metrics.counter_num_preemption, stats.num_preemption_iter
589
- )
590
- self._log_counter(
591
- self.metrics.counter_prompt_tokens, stats.num_prompt_tokens_iter
592
- )
593
- self._log_counter(
594
- self.metrics.counter_generation_tokens, stats.num_generation_tokens_iter
595
- )
596
- self._log_histogram(
597
- self.metrics.histogram_iteration_tokens, [stats.num_tokens_iter]
598
- )
599
- self._log_histogram(
600
- self.metrics.histogram_time_to_first_token, stats.time_to_first_tokens_iter
601
- )
602
- self._log_histogram(
603
- self.metrics.histogram_time_per_output_token,
604
- stats.inter_token_latencies_iter,
605
- )
606
- self._log_histogram(
607
- self.metrics.histogram_inter_token_latency, stats.inter_token_latencies_iter
608
- )
609
-
610
- # Request level data
611
- # Latency
612
- self._log_histogram(
613
- self.metrics.histogram_e2e_time_request, stats.time_e2e_requests
614
- )
615
- self._log_histogram(
616
- self.metrics.histogram_queue_time_request, stats.time_queue_requests
617
- )
618
- self._log_histogram(
619
- self.metrics.histogram_inference_time_request, stats.time_inference_requests
620
- )
621
- self._log_histogram(
622
- self.metrics.histogram_prefill_time_request, stats.time_prefill_requests
623
- )
624
- self._log_histogram(
625
- self.metrics.histogram_decode_time_request, stats.time_decode_requests
626
- )
627
- # Metadata
628
- finished_reason_counter = CollectionsCounter(stats.finished_reason_requests)
629
- self._log_counter_labels(
630
- self.metrics.counter_request_success,
631
- finished_reason_counter,
632
- Metrics.labelname_finish_reason,
633
- )
634
- self._log_histogram(
635
- self.metrics.histogram_num_prompt_tokens_request,
636
- stats.num_prompt_tokens_requests,
637
- )
638
- self._log_histogram(
639
- self.metrics.histogram_num_generation_tokens_request,
640
- stats.num_generation_tokens_requests,
641
- )
642
- self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
643
- self._log_histogram(
644
- self.metrics.histogram_max_num_generation_tokens_request,
645
- stats.max_num_generation_tokens_requests,
646
- )
647
- self._log_histogram(
648
- self.metrics.histogram_max_tokens_request, stats.max_tokens_requests
649
- )
650
-
651
- def log(self, stats: Stats):
652
- """Logs to prometheus and tracked stats every iteration."""
653
- # Log to prometheus.
654
- self._log_prometheus(stats)
655
-
656
- # Save tracked stats for token counters.
657
- self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
658
- self.num_generation_tokens.append(stats.num_generation_tokens_iter)
659
-
660
- # Log locally every local_interval seconds.
661
- if local_interval_elapsed(stats.now, self.last_local_log, self.local_interval):
662
- # Reset tracked stats for next interval.
663
- self.num_prompt_tokens = []
664
- self.num_generation_tokens = []
665
- self.last_local_log = stats.now
666
-
667
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
668
- # Info type metrics are syntactic sugar for a gauge permanently set to 1
669
- # Since prometheus multiprocessing mode does not support Info, emulate
670
- # info here with a gauge.
671
- if type == "cache_config":
672
- metrics_info = obj.metrics_info()
673
- info_gauge = self._gauge_cls(
674
- name="vllm:cache_config_info",
675
- documentation="Information of the LLMEngine CacheConfig",
676
- labelnames=metrics_info.keys(),
677
- multiprocess_mode="mostrecent",
678
- )
679
- info_gauge.labels(**metrics_info).set(1)
680
-
681
-
682
-class RayPrometheusStatLogger(PrometheusStatLogger):
683
- """RayPrometheusStatLogger uses Ray metrics instead."""
684
-
685
- _metrics_cls = RayMetrics
686
-
687
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
688
- return None
vllm/engine/metrics_types.py
deleted
-84
@@ -1,84 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-These types are defined in this file to avoid importing vllm.engine.metrics
5
-and therefore importing prometheus_client.
6
-
7
-This is required due to usage of Prometheus multiprocess mode to enable
8
-metrics after splitting out the uvicorn process from the engine process.
9
-
10
-Prometheus multiprocess mode requires setting PROMETHEUS_MULTIPROC_DIR
11
-before prometheus_client is imported. Typically, this is done by setting
12
-the env variable before launch, but since we are a library, we need to
13
-do this in Python code and lazily import prometheus_client.
14
-"""
15
-
16
-import time
17
-from abc import ABC, abstractmethod
18
-from dataclasses import dataclass
19
-
20
-from vllm.config import SupportsMetricsInfo, VllmConfig
21
-
22
-
23
-@dataclass
24
-class Stats:
25
- """Created by LLMEngine for use by StatLogger."""
26
-
27
- now: float
28
-
29
- # System stats (should have _sys suffix)
30
- # Scheduler State
31
- num_running_sys: int
32
- num_waiting_sys: int
33
- num_swapped_sys: int
34
- # KV Cache Usage in %
35
- gpu_cache_usage_sys: float
36
- cpu_cache_usage_sys: float
37
- # Prefix caching block hit rate
38
- cpu_prefix_cache_hit_rate: float
39
- gpu_prefix_cache_hit_rate: float
40
-
41
- # Iteration stats (should have _iter suffix)
42
- num_prompt_tokens_iter: int
43
- num_generation_tokens_iter: int
44
- num_tokens_iter: int
45
- time_to_first_tokens_iter: list[float]
46
- inter_token_latencies_iter: list[float]
47
- num_preemption_iter: int
48
-
49
- # Request stats (should have _requests suffix)
50
- # Latency
51
- time_e2e_requests: list[float]
52
- time_queue_requests: list[float]
53
- time_inference_requests: list[float]
54
- time_prefill_requests: list[float]
55
- time_decode_requests: list[float]
56
- # Metadata
57
- num_prompt_tokens_requests: list[int]
58
- num_generation_tokens_requests: list[int]
59
- n_requests: list[int]
60
- max_num_generation_tokens_requests: list[int]
61
- max_tokens_requests: list[int]
62
- finished_reason_requests: list[str]
63
- waiting_lora_adapters: list[str]
64
- running_lora_adapters: list[str]
65
- max_lora: str
66
-
67
-
68
-class StatLoggerBase(ABC):
69
- """Base class for StatLogger."""
70
-
71
- def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
72
- # Tracked stats over current local logging interval.
73
- self.num_prompt_tokens: list[int] = []
74
- self.num_generation_tokens: list[int] = []
75
- self.last_local_log = time.time()
76
- self.local_interval = local_interval
77
-
78
- @abstractmethod
79
- def log(self, stats: Stats) -> None:
80
- raise NotImplementedError
81
-
82
- @abstractmethod
83
- def info(self, type: str, obj: SupportsMetricsInfo) -> None:
84
- raise NotImplementedError