A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 08:15 UTC
[Metrics] Some small refactoring for better maintainability (#33898)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Martin Hickey committed
Mar 20, 2026 at 16:11 UTC
880be2b1b80fb2d18c32b0ee5a95174cf2e37c7d
7 files changed
+123
-135
vllm/distributed/kv_transfer/kv_connector/v1/metrics.py
+2
-13
@@ -126,28 +126,17 @@ class KVConnectorPromMetrics:
126
self._labelnames = labelnames
127
self.per_engine_labelvalues = per_engine_labelvalues
128
129
- def make_per_engine(self, metric: PromMetric) -> dict[int, PromMetric]:
130
- """
131
- Create a per-engine child of a prometheus_client.Metric with
132
- the appropriate labels set. The parent metric must be created
133
- using the labelnames list.
134
- """
135
- return {
136
- idx: metric.labels(*labelvalues)
137
- for idx, labelvalues in self.per_engine_labelvalues.items()
138
- }
139
-
129
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
130
"""
131
Record the supplied transfer statistics to Prometheus metrics. These
132
statistics are engine-specific, and should be recorded to a metric
133
with the appropriate 'engine' label. These metric instances can be
145
- created using the make_per_engine() helper method.
134
+ created using the create_metric_per_engine() helper method.
135
"""
136
raise NotImplementedError
137
138
150
-class KVConnectorPrometheus:
139
+class KVConnectorProm:
140
"""
141
Support for registering per-connector Prometheus metrics, and
142
recording transfer statistics to those metrics. Uses
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+17
-12
@@ -65,6 +65,7 @@ from vllm.v1.kv_cache_interface import (
65
SlidingWindowSpec,
66
UniformTypeKVCacheSpecs,
67
)
68
+from vllm.v1.metrics.utils import create_metric_per_engine
69
from vllm.v1.worker.block_table import BlockTable
70
from vllm.v1.worker.utils import select_common_block_size
71
@@ -3057,7 +3058,9 @@ class NixlPromMetrics(KVConnectorPromMetrics):
3058
buckets=buckets[1:],
3059
labelnames=labelnames,
3060
)
3060
- self.nixl_histogram_xfer_time = self.make_per_engine(nixl_histogram_xfer_time)
3061
+ self.nixl_histogram_xfer_time = create_metric_per_engine(
3062
+ nixl_histogram_xfer_time, self.per_engine_labelvalues
3063
+ )
3064
nixl_histogram_post_time = self._histogram_cls(
3065
name="vllm:nixl_post_time_seconds",
3066
documentation="Histogram of transfer post time for NIXL KV"
@@ -3065,7 +3068,9 @@ class NixlPromMetrics(KVConnectorPromMetrics):
3068
buckets=buckets,
3069
labelnames=labelnames,
3070
)
3068
- self.nixl_histogram_post_time = self.make_per_engine(nixl_histogram_post_time)
3071
+ self.nixl_histogram_post_time = create_metric_per_engine(
3072
+ nixl_histogram_post_time, self.per_engine_labelvalues
3073
+ )
3074
# uniform 2kb to 16gb range
3075
buckets = [2 ** (10 + i) for i in range(1, 25, 2)]
3076
nixl_histogram_bytes_transferred = self._histogram_cls(
@@ -3074,8 +3079,8 @@ class NixlPromMetrics(KVConnectorPromMetrics):
3079
buckets=buckets,
3080
labelnames=labelnames,
3081
)
3077
- self.nixl_histogram_bytes_transferred = self.make_per_engine(
3078
- nixl_histogram_bytes_transferred
3082
+ self.nixl_histogram_bytes_transferred = create_metric_per_engine(
3083
+ nixl_histogram_bytes_transferred, self.per_engine_labelvalues
3084
)
3085
buckets = [
3086
10,
@@ -3100,24 +3105,24 @@ class NixlPromMetrics(KVConnectorPromMetrics):
3105
buckets=buckets,
3106
labelnames=labelnames,
3107
)
3103
- self.nixl_histogram_num_descriptors = self.make_per_engine(
3104
- nixl_histogram_num_descriptors
3108
+ self.nixl_histogram_num_descriptors = create_metric_per_engine(
3109
+ nixl_histogram_num_descriptors, self.per_engine_labelvalues
3110
)
3111
counter_nixl_num_failed_transfers = self._counter_cls(
3112
name="vllm:nixl_num_failed_transfers",
3113
documentation="Number of failed NIXL KV Cache transfers.",
3114
labelnames=labelnames,
3115
)
3111
- self.counter_nixl_num_failed_transfers = self.make_per_engine(
3112
- counter_nixl_num_failed_transfers
3116
+ self.counter_nixl_num_failed_transfers = create_metric_per_engine(
3117
+ counter_nixl_num_failed_transfers, self.per_engine_labelvalues
3118
)
3119
counter_nixl_num_failed_notifications = self._counter_cls(
3120
name="vllm:nixl_num_failed_notifications",
3121
documentation="Number of failed NIXL KV Cache notifications.",
3122
labelnames=labelnames,
3123
)
3119
- self.counter_nixl_num_failed_notifications = self.make_per_engine(
3120
- counter_nixl_num_failed_notifications
3124
+ self.counter_nixl_num_failed_notifications = create_metric_per_engine(
3125
+ counter_nixl_num_failed_notifications, self.per_engine_labelvalues
3126
)
3127
3128
counter_nixl_num_kv_expired_reqs = self._counter_cls(
@@ -3126,8 +3131,8 @@ class NixlPromMetrics(KVConnectorPromMetrics):
3131
"NOTE: This metric is tracked on the P instance.",
3132
labelnames=labelnames,
3133
)
3129
- self.counter_nixl_num_kv_expired_reqs = self.make_per_engine(
3130
- counter_nixl_num_kv_expired_reqs
3134
+ self.counter_nixl_num_kv_expired_reqs = create_metric_per_engine(
3135
+ counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
3136
)
3137
3138
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
vllm/v1/metrics/loggers.py
+71
-79
@@ -5,7 +5,6 @@ import logging
5
import time
6
from abc import ABC, abstractmethod
7
from collections.abc import Callable
8
-from typing import TypeAlias
8
9
from prometheus_client import Counter, Gauge, Histogram
10
@@ -14,7 +13,7 @@ from vllm.compilation.cuda_graph import CUDAGraphLogging
13
from vllm.config import SupportsMetricsInfo, VllmConfig
14
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
15
KVConnectorLogging,
17
- KVConnectorPrometheus,
16
+ KVConnectorProm,
17
)
18
from vllm.logger import init_logger
19
from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group
@@ -28,6 +27,7 @@ from vllm.v1.metrics.stats import (
27
PromptTokenStats,
28
SchedulerStats,
29
)
30
+from vllm.v1.metrics.utils import create_metric_per_engine
31
from vllm.v1.spec_decode.metrics import SpecDecodingLogging, SpecDecodingProm
32
33
logger = init_logger(__name__)
@@ -391,7 +391,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
391
_counter_cls = Counter
392
_histogram_cls = Histogram
393
_spec_decoding_cls = SpecDecodingProm
394
- _kv_connector_cls = KVConnectorPrometheus
394
+ _kv_connector_cls = KVConnectorProm
395
_perf_metrics_cls = PerfMetricsProm
396
397
def __init__(
@@ -415,9 +415,10 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
415
model_name = vllm_config.model_config.served_model_name
416
max_model_len = vllm_config.model_config.max_model_len
417
418
- per_engine_labelvalues: dict[int, list[object]] = {
418
+ self.per_engine_labelvalues: dict[int, list[object]] = {
419
idx: [model_name, str(idx)] for idx in engine_indexes
420
}
421
+ per_engine_labelvalues = self.per_engine_labelvalues
422
423
self.spec_decoding_prom = self._spec_decoding_cls(
424
vllm_config.speculative_config, labelnames, per_engine_labelvalues
@@ -438,8 +439,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
439
multiprocess_mode="mostrecent",
440
labelnames=labelnames,
441
)
441
- self.gauge_scheduler_running = make_per_engine(
442
- gauge_scheduler_running, engine_indexes, model_name
442
+ self.gauge_scheduler_running = create_metric_per_engine(
443
+ gauge_scheduler_running, per_engine_labelvalues
444
)
445
446
gauge_scheduler_waiting = self._gauge_cls(
@@ -448,8 +449,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
449
multiprocess_mode="mostrecent",
450
labelnames=labelnames,
451
)
451
- self.gauge_scheduler_waiting = make_per_engine(
452
- gauge_scheduler_waiting, engine_indexes, model_name
452
+ self.gauge_scheduler_waiting = create_metric_per_engine(
453
+ gauge_scheduler_waiting, per_engine_labelvalues
454
)
455
456
gauge_engine_sleep_state = self._gauge_cls(
@@ -484,8 +485,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
485
multiprocess_mode="mostrecent",
486
labelnames=labelnames,
487
)
487
- self.gauge_kv_cache_usage = make_per_engine(
488
- gauge_kv_cache_usage, engine_indexes, model_name
488
+ self.gauge_kv_cache_usage = create_metric_per_engine(
489
+ gauge_kv_cache_usage, per_engine_labelvalues
490
)
491
492
if envs.VLLM_COMPUTE_NANS_IN_LOGITS:
@@ -497,8 +498,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
498
),
499
labelnames=labelnames,
500
)
500
- self.counter_corrupted_requests = make_per_engine(
501
- counter_corrupted_requests, engine_indexes, model_name
501
+ self.counter_corrupted_requests = create_metric_per_engine(
502
+ counter_corrupted_requests, per_engine_labelvalues
503
)
504
505
counter_prefix_cache_queries = self._counter_cls(
@@ -508,8 +509,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
509
),
510
labelnames=labelnames,
511
)
511
- self.counter_prefix_cache_queries = make_per_engine(
512
- counter_prefix_cache_queries, engine_indexes, model_name
512
+ self.counter_prefix_cache_queries = create_metric_per_engine(
513
+ counter_prefix_cache_queries, per_engine_labelvalues
514
)
515
516
counter_prefix_cache_hits = self._counter_cls(
@@ -517,8 +518,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
518
documentation=("Prefix cache hits, in terms of number of cached tokens."),
519
labelnames=labelnames,
520
)
520
- self.counter_prefix_cache_hits = make_per_engine(
521
- counter_prefix_cache_hits, engine_indexes, model_name
521
+ self.counter_prefix_cache_hits = create_metric_per_engine(
522
+ counter_prefix_cache_hits, per_engine_labelvalues
523
)
524
525
#
@@ -533,8 +534,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
534
),
535
labelnames=labelnames,
536
)
536
- self.counter_connector_prefix_cache_queries = make_per_engine(
537
- counter_connector_prefix_cache_queries, engine_indexes, model_name
537
+ self.counter_connector_prefix_cache_queries = create_metric_per_engine(
538
+ counter_connector_prefix_cache_queries, per_engine_labelvalues
539
)
540
541
counter_connector_prefix_cache_hits = self._counter_cls(
@@ -545,8 +546,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
546
),
547
labelnames=labelnames,
548
)
548
- self.counter_connector_prefix_cache_hits = make_per_engine(
549
- counter_connector_prefix_cache_hits, engine_indexes, model_name
549
+ self.counter_connector_prefix_cache_hits = create_metric_per_engine(
550
+ counter_connector_prefix_cache_hits, per_engine_labelvalues
551
)
552
553
#
@@ -560,8 +561,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
561
),
562
labelnames=labelnames,
563
)
563
- self.counter_mm_cache_queries = make_per_engine(
564
- counter_mm_cache_queries, engine_indexes, model_name
564
+ self.counter_mm_cache_queries = create_metric_per_engine(
565
+ counter_mm_cache_queries, per_engine_labelvalues
566
)
567
568
counter_mm_cache_hits = self._counter_cls(
@@ -571,8 +572,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
572
),
573
labelnames=labelnames,
574
)
574
- self.counter_mm_cache_hits = make_per_engine(
575
- counter_mm_cache_hits, engine_indexes, model_name
575
+ self.counter_mm_cache_hits = create_metric_per_engine(
576
+ counter_mm_cache_hits, per_engine_labelvalues
577
)
578
579
#
@@ -583,8 +584,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
584
documentation="Cumulative number of preemption from the engine.",
585
labelnames=labelnames,
586
)
586
- self.counter_num_preempted_reqs = make_per_engine(
587
- counter_num_preempted_reqs, engine_indexes, model_name
587
+ self.counter_num_preempted_reqs = create_metric_per_engine(
588
+ counter_num_preempted_reqs, per_engine_labelvalues
589
)
590
591
counter_prompt_tokens = self._counter_cls(
@@ -592,8 +593,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
593
documentation="Number of prefill tokens processed.",
594
labelnames=labelnames,
595
)
595
- self.counter_prompt_tokens = make_per_engine(
596
- counter_prompt_tokens, engine_indexes, model_name
596
+ self.counter_prompt_tokens = create_metric_per_engine(
597
+ counter_prompt_tokens, per_engine_labelvalues
598
)
599
600
# Labeled prompt token counters by source
@@ -617,8 +618,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
618
documentation="Number of cached prompt tokens (local + external).",
619
labelnames=labelnames,
620
)
620
- self.counter_prompt_tokens_cached = make_per_engine(
621
- counter_prompt_tokens_cached, engine_indexes, model_name
621
+ self.counter_prompt_tokens_cached = create_metric_per_engine(
622
+ counter_prompt_tokens_cached, per_engine_labelvalues
623
)
624
625
# Recomputed tokens (last token recomputed when entire prompt is cached)
@@ -627,8 +628,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
628
documentation="Number of cached tokens recomputed for forward pass.",
629
labelnames=labelnames,
630
)
630
- self.counter_prompt_tokens_recomputed = make_per_engine(
631
- counter_prompt_tokens_recomputed, engine_indexes, model_name
631
+ self.counter_prompt_tokens_recomputed = create_metric_per_engine(
632
+ counter_prompt_tokens_recomputed, per_engine_labelvalues
633
)
634
635
counter_generation_tokens = self._counter_cls(
@@ -636,8 +637,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
637
documentation="Number of generation tokens processed.",
638
labelnames=labelnames,
639
)
639
- self.counter_generation_tokens = make_per_engine(
640
- counter_generation_tokens, engine_indexes, model_name
640
+ self.counter_generation_tokens = create_metric_per_engine(
641
+ counter_generation_tokens, per_engine_labelvalues
642
)
643
644
self.counter_request_success: dict[FinishReason, dict[int, Counter]] = {}
@@ -663,8 +664,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
664
buckets=build_1_2_5_buckets(max_model_len),
665
labelnames=labelnames,
666
)
666
- self.histogram_num_prompt_tokens_request = make_per_engine(
667
- histogram_num_prompt_tokens_request, engine_indexes, model_name
667
+ self.histogram_num_prompt_tokens_request = create_metric_per_engine(
668
+ histogram_num_prompt_tokens_request, per_engine_labelvalues
669
)
670
671
histogram_num_generation_tokens_request = self._histogram_cls(
@@ -673,8 +674,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
674
buckets=build_1_2_5_buckets(max_model_len),
675
labelnames=labelnames,
676
)
676
- self.histogram_num_generation_tokens_request = make_per_engine(
677
- histogram_num_generation_tokens_request, engine_indexes, model_name
677
+ self.histogram_num_generation_tokens_request = create_metric_per_engine(
678
+ histogram_num_generation_tokens_request, per_engine_labelvalues
679
)
680
681
# TODO: This metric might be incorrect in case of using multiple
@@ -686,8 +687,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
687
buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
688
labelnames=labelnames,
689
)
689
- self.histogram_iteration_tokens = make_per_engine(
690
- histogram_iteration_tokens, engine_indexes, model_name
690
+ self.histogram_iteration_tokens = create_metric_per_engine(
691
+ histogram_iteration_tokens, per_engine_labelvalues
692
)
693
694
histogram_max_num_generation_tokens_request = self._histogram_cls(
@@ -696,8 +697,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
697
buckets=build_1_2_5_buckets(max_model_len),
698
labelnames=labelnames,
699
)
699
- self.histogram_max_num_generation_tokens_request = make_per_engine(
700
- histogram_max_num_generation_tokens_request, engine_indexes, model_name
700
+ self.histogram_max_num_generation_tokens_request = create_metric_per_engine(
701
+ histogram_max_num_generation_tokens_request, per_engine_labelvalues
702
)
703
704
histogram_n_request = self._histogram_cls(
@@ -706,8 +707,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
707
buckets=[1, 2, 5, 10, 20],
708
labelnames=labelnames,
709
)
709
- self.histogram_n_request = make_per_engine(
710
- histogram_n_request, engine_indexes, model_name
710
+ self.histogram_n_request = create_metric_per_engine(
711
+ histogram_n_request, per_engine_labelvalues
712
)
713
714
histogram_max_tokens_request = self._histogram_cls(
@@ -716,8 +717,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
717
buckets=build_1_2_5_buckets(max_model_len),
718
labelnames=labelnames,
719
)
719
- self.histogram_max_tokens_request = make_per_engine(
720
- histogram_max_tokens_request, engine_indexes, model_name
720
+ self.histogram_max_tokens_request = create_metric_per_engine(
721
+ histogram_max_tokens_request, per_engine_labelvalues
722
)
723
724
#
@@ -752,8 +753,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
753
],
754
labelnames=labelnames,
755
)
755
- self.histogram_time_to_first_token = make_per_engine(
756
- histogram_time_to_first_token, engine_indexes, model_name
756
+ self.histogram_time_to_first_token = create_metric_per_engine(
757
+ histogram_time_to_first_token, per_engine_labelvalues
758
)
759
760
histogram_inter_token_latency = self._histogram_cls(
@@ -782,8 +783,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
783
],
784
labelnames=labelnames,
785
)
785
- self.histogram_inter_token_latency = make_per_engine(
786
- histogram_inter_token_latency, engine_indexes, model_name
786
+ self.histogram_inter_token_latency = create_metric_per_engine(
787
+ histogram_inter_token_latency, per_engine_labelvalues
788
)
789
790
histogram_request_time_per_output_token = self._histogram_cls(
@@ -812,8 +813,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
813
],
814
labelnames=labelnames,
815
)
815
- self.histogram_request_time_per_output_token = make_per_engine(
816
- histogram_request_time_per_output_token, engine_indexes, model_name
816
+ self.histogram_request_time_per_output_token = create_metric_per_engine(
817
+ histogram_request_time_per_output_token, per_engine_labelvalues
818
)
819
820
request_latency_buckets = [
@@ -845,8 +846,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
846
buckets=request_latency_buckets,
847
labelnames=labelnames,
848
)
848
- self.histogram_e2e_time_request = make_per_engine(
849
- histogram_e2e_time_request, engine_indexes, model_name
849
+ self.histogram_e2e_time_request = create_metric_per_engine(
850
+ histogram_e2e_time_request, per_engine_labelvalues
851
)
852
853
histogram_queue_time_request = self._histogram_cls(
@@ -855,8 +856,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
856
buckets=request_latency_buckets,
857
labelnames=labelnames,
858
)
858
- self.histogram_queue_time_request = make_per_engine(
859
- histogram_queue_time_request, engine_indexes, model_name
859
+ self.histogram_queue_time_request = create_metric_per_engine(
860
+ histogram_queue_time_request, per_engine_labelvalues
861
)
862
863
histogram_inference_time_request = self._histogram_cls(
@@ -865,8 +866,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
866
buckets=request_latency_buckets,
867
labelnames=labelnames,
868
)
868
- self.histogram_inference_time_request = make_per_engine(
869
- histogram_inference_time_request, engine_indexes, model_name
869
+ self.histogram_inference_time_request = create_metric_per_engine(
870
+ histogram_inference_time_request, per_engine_labelvalues
871
)
872
873
histogram_prefill_time_request = self._histogram_cls(
@@ -875,8 +876,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
876
buckets=request_latency_buckets,
877
labelnames=labelnames,
878
)
878
- self.histogram_prefill_time_request = make_per_engine(
879
- histogram_prefill_time_request, engine_indexes, model_name
879
+ self.histogram_prefill_time_request = create_metric_per_engine(
880
+ histogram_prefill_time_request, per_engine_labelvalues
881
)
882
883
histogram_decode_time_request = self._histogram_cls(
@@ -885,8 +886,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
886
buckets=request_latency_buckets,
887
labelnames=labelnames,
888
)
888
- self.histogram_decode_time_request = make_per_engine(
889
- histogram_decode_time_request, engine_indexes, model_name
889
+ self.histogram_decode_time_request = create_metric_per_engine(
890
+ histogram_decode_time_request, per_engine_labelvalues
891
)
892
893
histogram_prefill_kv_computed_request = self._histogram_cls(
@@ -898,8 +899,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
899
buckets=build_1_2_5_buckets(max_model_len),
900
labelnames=labelnames,
901
)
901
- self.histogram_prefill_kv_computed_request = make_per_engine(
902
- histogram_prefill_kv_computed_request, engine_indexes, model_name
902
+ self.histogram_prefill_kv_computed_request = create_metric_per_engine(
903
+ histogram_prefill_kv_computed_request, per_engine_labelvalues
904
)
905
906
#
@@ -939,8 +940,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
940
buckets=kv_cache_residency_buckets,
941
labelnames=labelnames,
942
)
942
- self.histogram_kv_block_lifetime = make_per_engine(
943
- histogram_kv_block_lifetime, engine_indexes, model_name
943
+ self.histogram_kv_block_lifetime = create_metric_per_engine(
944
+ histogram_kv_block_lifetime, per_engine_labelvalues
945
)
946
947
histogram_kv_block_idle_before_evict = self._histogram_cls(
@@ -952,8 +953,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
953
buckets=kv_cache_residency_buckets,
954
labelnames=labelnames,
955
)
955
- self.histogram_kv_block_idle_before_evict = make_per_engine(
956
- histogram_kv_block_idle_before_evict, engine_indexes, model_name
956
+ self.histogram_kv_block_idle_before_evict = create_metric_per_engine(
957
+ histogram_kv_block_idle_before_evict, per_engine_labelvalues
958
)
959
960
histogram_kv_block_reuse_gap = self._histogram_cls(
@@ -967,8 +968,8 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
968
buckets=kv_cache_residency_buckets,
969
labelnames=labelnames,
970
)
970
- self.histogram_kv_block_reuse_gap = make_per_engine(
971
- histogram_kv_block_reuse_gap, engine_indexes, model_name
971
+ self.histogram_kv_block_reuse_gap = create_metric_per_engine(
972
+ histogram_kv_block_reuse_gap, per_engine_labelvalues
973
)
974
else:
975
self.histogram_kv_block_lifetime = {}
@@ -1203,15 +1204,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1204
self.log_metrics_info("cache_config", self.vllm_config.cache_config)
1205
1206
1206
-PromMetric: TypeAlias = Gauge | Counter | Histogram
1207
-
1208
-
1209
-def make_per_engine(
1210
- metric: PromMetric, engine_idxs: list[int], model_name: object
1211
-) -> dict[int, PromMetric]:
1212
- return {idx: metric.labels(model_name, str(idx)) for idx in engine_idxs}
1213
-
1214
-
1207
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
1208
"""
1209
Builds a list of buckets with increasing powers of 10 multiplied by
vllm/v1/metrics/perf.py
+6
-13
@@ -27,6 +27,7 @@ from vllm.utils.torch_utils import (
27
get_kv_cache_torch_dtype,
28
)
29
from vllm.v1.core.sched.output import SchedulerOutput
30
+from vllm.v1.metrics.utils import create_metric_per_engine
31
32
logger = init_logger(__name__)
33
@@ -1291,7 +1292,9 @@ class PerfMetricsProm:
1292
),
1293
labelnames=labelnames,
1294
)
1294
- self.counter_flops = make_per_engine(counter_flops, per_engine_labelvalues)
1295
+ self.counter_flops = create_metric_per_engine(
1296
+ counter_flops, per_engine_labelvalues
1297
+ )
1298
1299
counter_read_bytes = self._counter_cls(
1300
name="vllm:estimated_read_bytes_per_gpu_total",
@@ -1301,7 +1304,7 @@ class PerfMetricsProm:
1304
),
1305
labelnames=labelnames,
1306
)
1304
- self.counter_read_bytes = make_per_engine(
1307
+ self.counter_read_bytes = create_metric_per_engine(
1308
counter_read_bytes, per_engine_labelvalues
1309
)
1310
@@ -1313,7 +1316,7 @@ class PerfMetricsProm:
1316
),
1317
labelnames=labelnames,
1318
)
1316
- self.counter_write_bytes = make_per_engine(
1319
+ self.counter_write_bytes = create_metric_per_engine(
1320
counter_write_bytes, per_engine_labelvalues
1321
)
1322
@@ -1329,16 +1332,6 @@ class PerfMetricsProm:
1332
self.counter_write_bytes[engine_idx].inc(perf_stats.num_write_bytes_per_gpu)
1333
1334
1332
-def make_per_engine(
1333
- counter: prometheus_client.Counter, per_engine_labelvalues: dict[int, list[object]]
1334
-):
1335
- """Create a counter for each label value."""
1336
- return {
1337
- idx: counter.labels(*labelvalues)
1338
- for idx, labelvalues in per_engine_labelvalues.items()
1339
- }
1340
-
1341
-
1335
## util functions
1336
1337
vllm/v1/metrics/ray_wrappers.py
+4
-4
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
5
-from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorPrometheus
5
+from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorProm
6
from vllm.v1.metrics.loggers import PrometheusStatLogger
7
from vllm.v1.metrics.perf import PerfMetricsProm
8
from vllm.v1.spec_decode.metrics import SpecDecodingProm
@@ -168,9 +168,9 @@ class RaySpecDecodingProm(SpecDecodingProm):
168
_counter_cls = RayCounterWrapper
169
170
171
-class RayKVConnectorPrometheus(KVConnectorPrometheus):
171
+class RayKVConnectorProm(KVConnectorProm):
172
"""
173
- RayKVConnectorPrometheus is used by RayMetrics to log Ray
173
+ RayKVConnectorProm is used by RayMetrics to log Ray
174
metrics. Provides the same metrics as KV connectors but
175
uses Ray's util.metrics library.
176
"""
@@ -197,7 +197,7 @@ class RayPrometheusStatLogger(PrometheusStatLogger):
197
_counter_cls = RayCounterWrapper
198
_histogram_cls = RayHistogramWrapper
199
_spec_decoding_cls = RaySpecDecodingProm
200
- _kv_connector_cls = RayKVConnectorPrometheus
200
+ _kv_connector_cls = RayKVConnectorProm
201
_perf_metrics_cls = RayPerfMetricsProm
202
203
@staticmethod
vllm/v1/metrics/utils.py
new
+19
@@ -0,0 +1,19 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import TypeAlias
5
+
6
+from prometheus_client import Counter, Gauge, Histogram
7
+
8
+PromMetric: TypeAlias = Gauge | Counter | Histogram
9
+
10
+
11
+def create_metric_per_engine(
12
+ metric: PromMetric,
13
+ per_engine_labelvalues: dict[int, list[object]],
14
+) -> dict[int, PromMetric]:
15
+ """Create a labeled metric child for each engine index."""
16
+ return {
17
+ idx: metric.labels(*labelvalues)
18
+ for idx, labelvalues in per_engine_labelvalues.items()
19
+ }
vllm/v1/spec_decode/metrics.py
+4
-14
@@ -9,6 +9,7 @@ import prometheus_client
9
10
from vllm.config import SpeculativeConfig
11
from vllm.logger import init_logger
12
+from vllm.v1.metrics.utils import create_metric_per_engine
13
14
logger = init_logger(__name__)
15
@@ -155,7 +156,7 @@ class SpecDecodingProm:
156
documentation="Number of spec decoding drafts.",
157
labelnames=labelnames,
158
)
158
- self.counter_spec_decode_num_drafts = make_per_engine(
159
+ self.counter_spec_decode_num_drafts = create_metric_per_engine(
160
counter_drafts, per_engine_labelvalues
161
)
162
@@ -164,7 +165,7 @@ class SpecDecodingProm:
165
documentation="Number of draft tokens.",
166
labelnames=labelnames,
167
)
167
- self.counter_spec_decode_num_draft_tokens = make_per_engine(
168
+ self.counter_spec_decode_num_draft_tokens = create_metric_per_engine(
169
counter_draft_tokens, per_engine_labelvalues
170
)
171
@@ -173,7 +174,7 @@ class SpecDecodingProm:
174
documentation="Number of accepted tokens.",
175
labelnames=labelnames,
176
)
176
- self.counter_spec_decode_num_accepted_tokens = make_per_engine(
177
+ self.counter_spec_decode_num_accepted_tokens = create_metric_per_engine(
178
counter_accepted_tokens, per_engine_labelvalues
179
)
180
@@ -212,14 +213,3 @@ class SpecDecodingProm:
213
self.counter_spec_decode_num_accepted_tokens_per_pos[engine_idx]
214
):
215
counter.inc(spec_decoding_stats.num_accepted_tokens_per_pos[pos])
215
-
216
-
217
-def make_per_engine(
218
- counter: prometheus_client.Counter,
219
- per_engine_labelvalues: dict[int, list[object]],
220
-):
221
- """Create a counter for each label value."""
222
- return {
223
- idx: counter.labels(*labelvalues)
224
- for idx, labelvalues in per_engine_labelvalues.items()
225
- }