A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
[Misc] Add ReplicaId to Ray metrics (#24267)
Signed-off-by: Seiji Eicher <seiji@anyscale.com> Co-authored-by: rongfu.leng <1275177125@qq.com>
Seiji Eicher committed
Dec 1, 2025 at 19:21 UTC
22274b2184d7b1adadd7b244bf3e3ee0cfd71280
1 file changed
+42
-17
vllm/v1/metrics/ray_wrappers.py
+42
-17
@@ -7,37 +7,55 @@ from vllm.v1.metrics.loggers import PrometheusStatLogger
7
from vllm.v1.spec_decode.metrics import SpecDecodingProm
8
9
try:
10
+ from ray import serve as ray_serve
11
from ray.util import metrics as ray_metrics
12
from ray.util.metrics import Metric
13
except ImportError:
14
ray_metrics = None
15
+ ray_serve = None
16
import regex as re
17
18
19
+def _get_replica_id() -> str | None:
20
+ """Get the current Ray Serve replica ID, or None if not in a Serve context."""
21
+ if ray_serve is None:
22
+ return None
23
+ try:
24
+ return ray_serve.get_replica_context().replica_id.unique_id
25
+ except ray_serve.exceptions.RayServeException:
26
+ return None
27
+
28
+
29
class RayPrometheusMetric:
30
def __init__(self):
31
if ray_metrics is None:
32
raise ImportError("RayPrometheusMetric requires Ray to be installed.")
21
-
33
self.metric: Metric = None
34
24
- def labels(self, *labels, **labelskwargs):
25
- if labelskwargs:
26
- for k, v in labelskwargs.items():
27
- if not isinstance(v, str):
28
- labelskwargs[k] = str(v)
29
-
30
- self.metric.set_default_tags(labelskwargs)
35
+ @staticmethod
36
+ def _get_tag_keys(labelnames: list[str] | None) -> tuple[str, ...]:
37
+ labels = list(labelnames) if labelnames else []
38
+ labels.append("ReplicaId")
39
+ return tuple(labels)
40
41
+ def labels(self, *labels, **labelskwargs):
42
if labels:
33
- if len(labels) != len(self.metric._tag_keys):
43
+ # -1 because ReplicaId was added automatically
44
+ expected = len(self.metric._tag_keys) - 1
45
+ if len(labels) != expected:
46
raise ValueError(
47
"Number of labels must match the number of tag keys. "
36
- f"Expected {len(self.metric._tag_keys)}, got {len(labels)}"
48
+ f"Expected {expected}, got {len(labels)}"
49
)
50
+ labelskwargs.update(zip(self.metric._tag_keys, labels))
51
39
- self.metric.set_default_tags(dict(zip(self.metric._tag_keys, labels)))
52
+ labelskwargs["ReplicaId"] = _get_replica_id() or ""
53
54
+ if labelskwargs:
55
+ for k, v in labelskwargs.items():
56
+ if not isinstance(v, str):
57
+ labelskwargs[k] = str(v)
58
+ self.metric.set_default_tags(labelskwargs)
59
return self
60
61
@staticmethod
@@ -71,10 +89,14 @@ class RayGaugeWrapper(RayPrometheusMetric):
89
# "mostrecent", "all", "sum" do not apply. This logic can be manually
90
# implemented at the observability layer (Prometheus/Grafana).
91
del multiprocess_mode
74
- labelnames_tuple = tuple(labelnames) if labelnames else None
92
+
93
+ tag_keys = self._get_tag_keys(labelnames)
94
name = self._get_sanitized_opentelemetry_name(name)
95
+
96
self.metric = ray_metrics.Gauge(
77
- name=name, description=documentation, tag_keys=labelnames_tuple
97
+ name=name,
98
+ description=documentation,
99
+ tag_keys=tag_keys,
100
)
101
102
def set(self, value: int | float):
@@ -95,10 +117,12 @@ class RayCounterWrapper(RayPrometheusMetric):
117
documentation: str | None = "",
118
labelnames: list[str] | None = None,
119
):
98
- labelnames_tuple = tuple(labelnames) if labelnames else None
120
+ tag_keys = self._get_tag_keys(labelnames)
121
name = self._get_sanitized_opentelemetry_name(name)
122
self.metric = ray_metrics.Counter(
101
- name=name, description=documentation, tag_keys=labelnames_tuple
123
+ name=name,
124
+ description=documentation,
125
+ tag_keys=tag_keys,
126
)
127
128
def inc(self, value: int | float = 1.0):
@@ -118,13 +142,14 @@ class RayHistogramWrapper(RayPrometheusMetric):
142
labelnames: list[str] | None = None,
143
buckets: list[float] | None = None,
144
):
121
- labelnames_tuple = tuple(labelnames) if labelnames else None
145
+ tag_keys = self._get_tag_keys(labelnames)
146
name = self._get_sanitized_opentelemetry_name(name)
147
+
148
boundaries = buckets if buckets else []
149
self.metric = ray_metrics.Histogram(
150
name=name,
151
description=documentation,
127
- tag_keys=labelnames_tuple,
152
+ tag_keys=tag_keys,
153
boundaries=boundaries,
154
)
155