A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix Ray Metrics API usage (#6354)
Antoni Baum committed
Jul 17, 2024 at 12:40 UTC
5f0b9933e63839e816b9736a65a3c55005df2cfe
4 files changed
+195
-40
tests/metrics/test_metrics.py
+54
@@ -1,11 +1,13 @@
1
from typing import List
2
3
import pytest
4
+import ray
5
from prometheus_client import REGISTRY
6
7
from vllm import EngineArgs, LLMEngine
8
from vllm.engine.arg_utils import AsyncEngineArgs
9
from vllm.engine.async_llm_engine import AsyncLLMEngine
10
+from vllm.engine.metrics import RayPrometheusStatLogger
11
from vllm.sampling_params import SamplingParams
12
13
MODELS = [
@@ -241,3 +243,55 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
243
labels)
244
assert (
245
metric_value == num_requests), "Metrics should be collected"
246
+
247
+
248
+@pytest.mark.parametrize("model", MODELS)
249
+@pytest.mark.parametrize("dtype", ["half"])
250
+@pytest.mark.parametrize("max_tokens", [16])
251
+def test_engine_log_metrics_ray(
252
+ example_prompts,
253
+ model: str,
254
+ dtype: str,
255
+ max_tokens: int,
256
+) -> None:
257
+ # This test is quite weak - it only checks that we can use
258
+ # RayPrometheusStatLogger without exceptions.
259
+ # Checking whether the metrics are actually emitted is unfortunately
260
+ # non-trivial.
261
+
262
+ # We have to run in a Ray task for Ray metrics to be emitted correctly
263
+ @ray.remote(num_gpus=1)
264
+ def _inner():
265
+
266
+ class _RayPrometheusStatLogger(RayPrometheusStatLogger):
267
+
268
+ def __init__(self, *args, **kwargs):
269
+ self._i = 0
270
+ super().__init__(*args, **kwargs)
271
+
272
+ def log(self, *args, **kwargs):
273
+ self._i += 1
274
+ return super().log(*args, **kwargs)
275
+
276
+ engine_args = EngineArgs(
277
+ model=model,
278
+ dtype=dtype,
279
+ disable_log_stats=False,
280
+ )
281
+ engine = LLMEngine.from_engine_args(engine_args)
282
+ logger = _RayPrometheusStatLogger(
283
+ local_interval=0.5,
284
+ labels=dict(model_name=engine.model_config.served_model_name),
285
+ max_model_len=engine.model_config.max_model_len)
286
+ engine.add_logger("ray", logger)
287
+ for i, prompt in enumerate(example_prompts):
288
+ engine.add_request(
289
+ f"request-id-{i}",
290
+ prompt,
291
+ SamplingParams(max_tokens=max_tokens),
292
+ )
293
+ while engine.has_unfinished_requests():
294
+ engine.step()
295
+ assert logger._i > 0, ".log must be called at least once"
296
+
297
+ ray.get(_inner.remote())
vllm/engine/async_llm_engine.py
+19
@@ -12,6 +12,7 @@ from vllm.core.scheduler import SchedulerOutputs
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
from vllm.engine.async_timeout import asyncio_timeout
14
from vllm.engine.llm_engine import LLMEngine
15
+from vllm.engine.metrics import StatLoggerBase
16
from vllm.executor.ray_utils import initialize_ray_cluster, ray
17
from vllm.inputs import LLMInputs, PromptInputs
18
from vllm.logger import init_logger
@@ -389,6 +390,7 @@ class AsyncLLMEngine:
390
engine_args: AsyncEngineArgs,
391
start_engine_loop: bool = True,
392
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
393
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
394
) -> "AsyncLLMEngine":
395
"""Creates an async LLM engine from the engine arguments."""
396
# Create the engine configs.
@@ -451,6 +453,7 @@ class AsyncLLMEngine:
453
max_log_len=engine_args.max_log_len,
454
start_engine_loop=start_engine_loop,
455
usage_context=usage_context,
456
+ stat_loggers=stat_loggers,
457
)
458
return engine
459
@@ -957,3 +960,19 @@ class AsyncLLMEngine:
960
)
961
else:
962
return self.engine.is_tracing_enabled()
963
+
964
+ def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
965
+ if self.engine_use_ray:
966
+ ray.get(
967
+ self.engine.add_logger.remote( # type: ignore
968
+ logger_name=logger_name, logger=logger))
969
+ else:
970
+ self.engine.add_logger(logger_name=logger_name, logger=logger)
971
+
972
+ def remove_logger(self, logger_name: str) -> None:
973
+ if self.engine_use_ray:
974
+ ray.get(
975
+ self.engine.remove_logger.remote( # type: ignore
976
+ logger_name=logger_name))
977
+ else:
978
+ self.engine.remove_logger(logger_name=logger_name)
vllm/engine/llm_engine.py
+2
@@ -379,6 +379,7 @@ class LLMEngine:
379
cls,
380
engine_args: EngineArgs,
381
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
382
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
383
) -> "LLMEngine":
384
"""Creates an LLM engine from the engine arguments."""
385
# Create the engine configs.
@@ -423,6 +424,7 @@ class LLMEngine:
424
executor_class=executor_class,
425
log_stats=not engine_args.disable_log_stats,
426
usage_context=usage_context,
427
+ stat_loggers=stat_loggers,
428
)
429
return engine
430
vllm/engine/metrics.py
+120
-40
@@ -30,55 +30,55 @@ prometheus_client.disable_created_metrics()
30
# begin-metrics-definitions
31
class Metrics:
32
labelname_finish_reason = "finished_reason"
33
- _base_library = prometheus_client
33
+ _gauge_cls = prometheus_client.Gauge
34
+ _counter_cls = prometheus_client.Counter
35
+ _histogram_cls = prometheus_client.Histogram
36
37
def __init__(self, labelnames: List[str], max_model_len: int):
38
# Unregister any existing vLLM collectors
39
self._unregister_vllm_metrics()
40
41
# Config Information
40
- self.info_cache_config = prometheus_client.Info(
41
- name='vllm:cache_config',
42
- documentation='information of cache_config')
42
+ self._create_info_cache_config()
43
44
# System stats
45
# Scheduler State
46
- self.gauge_scheduler_running = self._base_library.Gauge(
46
+ self.gauge_scheduler_running = self._gauge_cls(
47
name="vllm:num_requests_running",
48
documentation="Number of requests currently running on GPU.",
49
labelnames=labelnames)
50
- self.gauge_scheduler_waiting = self._base_library.Gauge(
50
+ self.gauge_scheduler_waiting = self._gauge_cls(
51
name="vllm:num_requests_waiting",
52
documentation="Number of requests waiting to be processed.",
53
labelnames=labelnames)
54
- self.gauge_scheduler_swapped = self._base_library.Gauge(
54
+ self.gauge_scheduler_swapped = self._gauge_cls(
55
name="vllm:num_requests_swapped",
56
documentation="Number of requests swapped to CPU.",
57
labelnames=labelnames)
58
# KV Cache Usage in %
59
- self.gauge_gpu_cache_usage = self._base_library.Gauge(
59
+ self.gauge_gpu_cache_usage = self._gauge_cls(
60
name="vllm:gpu_cache_usage_perc",
61
documentation="GPU KV-cache usage. 1 means 100 percent usage.",
62
labelnames=labelnames)
63
- self.gauge_cpu_cache_usage = self._base_library.Gauge(
63
+ self.gauge_cpu_cache_usage = self._gauge_cls(
64
name="vllm:cpu_cache_usage_perc",
65
documentation="CPU KV-cache usage. 1 means 100 percent usage.",
66
labelnames=labelnames)
67
68
# Iteration stats
69
- self.counter_num_preemption = self._base_library.Counter(
69
+ self.counter_num_preemption = self._counter_cls(
70
name="vllm:num_preemptions_total",
71
documentation="Cumulative number of preemption from the engine.",
72
labelnames=labelnames)
73
- self.counter_prompt_tokens = self._base_library.Counter(
73
+ self.counter_prompt_tokens = self._counter_cls(
74
name="vllm:prompt_tokens_total",
75
documentation="Number of prefill tokens processed.",
76
labelnames=labelnames)
77
- self.counter_generation_tokens = self._base_library.Counter(
77
+ self.counter_generation_tokens = self._counter_cls(
78
name="vllm:generation_tokens_total",
79
documentation="Number of generation tokens processed.",
80
labelnames=labelnames)
81
- self.histogram_time_to_first_token = self._base_library.Histogram(
81
+ self.histogram_time_to_first_token = self._histogram_cls(
82
name="vllm:time_to_first_token_seconds",
83
documentation="Histogram of time to first token in seconds.",
84
labelnames=labelnames,
@@ -86,7 +86,7 @@ class Metrics:
86
0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
87
0.75, 1.0, 2.5, 5.0, 7.5, 10.0
88
])
89
- self.histogram_time_per_output_token = self._base_library.Histogram(
89
+ self.histogram_time_per_output_token = self._histogram_cls(
90
name="vllm:time_per_output_token_seconds",
91
documentation="Histogram of time per output token in seconds.",
92
labelnames=labelnames,
@@ -97,83 +97,157 @@ class Metrics:
97
98
# Request stats
99
# Latency
100
- self.histogram_e2e_time_request = self._base_library.Histogram(
100
+ self.histogram_e2e_time_request = self._histogram_cls(
101
name="vllm:e2e_request_latency_seconds",
102
documentation="Histogram of end to end request latency in seconds.",
103
labelnames=labelnames,
104
buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
105
# Metadata
106
- self.histogram_num_prompt_tokens_request = self._base_library.Histogram(
106
+ self.histogram_num_prompt_tokens_request = self._histogram_cls(
107
name="vllm:request_prompt_tokens",
108
documentation="Number of prefill tokens processed.",
109
labelnames=labelnames,
110
buckets=build_1_2_5_buckets(max_model_len),
111
)
112
self.histogram_num_generation_tokens_request = \
113
- self._base_library.Histogram(
113
+ self._histogram_cls(
114
name="vllm:request_generation_tokens",
115
documentation="Number of generation tokens processed.",
116
labelnames=labelnames,
117
buckets=build_1_2_5_buckets(max_model_len),
118
)
119
- self.histogram_best_of_request = self._base_library.Histogram(
119
+ self.histogram_best_of_request = self._histogram_cls(
120
name="vllm:request_params_best_of",
121
documentation="Histogram of the best_of request parameter.",
122
labelnames=labelnames,
123
buckets=[1, 2, 5, 10, 20],
124
)
125
- self.histogram_n_request = self._base_library.Histogram(
125
+ self.histogram_n_request = self._histogram_cls(
126
name="vllm:request_params_n",
127
documentation="Histogram of the n request parameter.",
128
labelnames=labelnames,
129
buckets=[1, 2, 5, 10, 20],
130
)
131
- self.counter_request_success = self._base_library.Counter(
131
+ self.counter_request_success = self._counter_cls(
132
name="vllm:request_success_total",
133
documentation="Count of successfully processed requests.",
134
labelnames=labelnames + [Metrics.labelname_finish_reason])
135
136
# Speculatie decoding stats
137
- self.gauge_spec_decode_draft_acceptance_rate = self._base_library.Gauge(
137
+ self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
138
name="vllm:spec_decode_draft_acceptance_rate",
139
documentation="Speulative token acceptance rate.",
140
labelnames=labelnames)
141
- self.gauge_spec_decode_efficiency = self._base_library.Gauge(
141
+ self.gauge_spec_decode_efficiency = self._gauge_cls(
142
name="vllm:spec_decode_efficiency",
143
documentation="Speculative decoding system efficiency.",
144
labelnames=labelnames)
145
- self.counter_spec_decode_num_accepted_tokens = (
146
- self._base_library.Counter(
147
- name="vllm:spec_decode_num_accepted_tokens_total",
148
- documentation="Number of accepted tokens.",
149
- labelnames=labelnames))
150
- self.counter_spec_decode_num_draft_tokens = self._base_library.Counter(
145
+ self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
146
+ name="vllm:spec_decode_num_accepted_tokens_total",
147
+ documentation="Number of accepted tokens.",
148
+ labelnames=labelnames))
149
+ self.counter_spec_decode_num_draft_tokens = self._counter_cls(
150
name="vllm:spec_decode_num_draft_tokens_total",
151
documentation="Number of draft tokens.",
152
labelnames=labelnames)
154
- self.counter_spec_decode_num_emitted_tokens = (
155
- self._base_library.Counter(
156
- name="vllm:spec_decode_num_emitted_tokens_total",
157
- documentation="Number of emitted tokens.",
158
- labelnames=labelnames))
153
+ self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
154
+ name="vllm:spec_decode_num_emitted_tokens_total",
155
+ documentation="Number of emitted tokens.",
156
+ labelnames=labelnames))
157
158
# Deprecated in favor of vllm:prompt_tokens_total
161
- self.gauge_avg_prompt_throughput = self._base_library.Gauge(
159
+ self.gauge_avg_prompt_throughput = self._gauge_cls(
160
name="vllm:avg_prompt_throughput_toks_per_s",
161
documentation="Average prefill throughput in tokens/s.",
162
labelnames=labelnames,
163
)
164
# Deprecated in favor of vllm:generation_tokens_total
167
- self.gauge_avg_generation_throughput = self._base_library.Gauge(
165
+ self.gauge_avg_generation_throughput = self._gauge_cls(
166
name="vllm:avg_generation_throughput_toks_per_s",
167
documentation="Average generation throughput in tokens/s.",
168
labelnames=labelnames,
169
)
170
171
+ def _create_info_cache_config(self) -> None:
172
+ # Config Information
173
+ self.info_cache_config = prometheus_client.Info(
174
+ name='vllm:cache_config',
175
+ documentation='information of cache_config')
176
+
177
def _unregister_vllm_metrics(self) -> None:
174
- for collector in list(self._base_library.REGISTRY._collector_to_names):
178
+ for collector in list(prometheus_client.REGISTRY._collector_to_names):
179
if hasattr(collector, "_name") and "vllm" in collector._name:
176
- self._base_library.REGISTRY.unregister(collector)
180
+ prometheus_client.REGISTRY.unregister(collector)
181
+
182
+
183
+# end-metrics-definitions
184
+
185
+
186
+class _RayGaugeWrapper:
187
+ """Wraps around ray.util.metrics.Gauge to provide same API as
188
+ prometheus_client.Gauge"""
189
+
190
+ def __init__(self,
191
+ name: str,
192
+ documentation: str = "",
193
+ labelnames: Optional[List[str]] = None):
194
+ labelnames_tuple = tuple(labelnames) if labelnames else None
195
+ self._gauge = ray_metrics.Gauge(name=name,
196
+ description=documentation,
197
+ tag_keys=labelnames_tuple)
198
+
199
+ def labels(self, **labels):
200
+ self._gauge.set_default_tags(labels)
201
+ return self
202
+
203
+ def set(self, value: Union[int, float]):
204
+ return self._gauge.set(value)
205
+
206
+
207
+class _RayCounterWrapper:
208
+ """Wraps around ray.util.metrics.Counter to provide same API as
209
+ prometheus_client.Counter"""
210
+
211
+ def __init__(self,
212
+ name: str,
213
+ documentation: str = "",
214
+ labelnames: Optional[List[str]] = None):
215
+ labelnames_tuple = tuple(labelnames) if labelnames else None
216
+ self._counter = ray_metrics.Counter(name=name,
217
+ description=documentation,
218
+ tag_keys=labelnames_tuple)
219
+
220
+ def labels(self, **labels):
221
+ self._counter.set_default_tags(labels)
222
+ return self
223
+
224
+ def inc(self, value: Union[int, float] = 1.0):
225
+ if value == 0:
226
+ return
227
+ return self._counter.inc(value)
228
+
229
+
230
+class _RayHistogramWrapper:
231
+ """Wraps around ray.util.metrics.Histogram to provide same API as
232
+ prometheus_client.Histogram"""
233
+
234
+ def __init__(self,
235
+ name: str,
236
+ documentation: str = "",
237
+ labelnames: Optional[List[str]] = None,
238
+ buckets: Optional[List[float]] = None):
239
+ labelnames_tuple = tuple(labelnames) if labelnames else None
240
+ self._histogram = ray_metrics.Histogram(name=name,
241
+ description=documentation,
242
+ tag_keys=labelnames_tuple,
243
+ boundaries=buckets)
244
+
245
+ def labels(self, **labels):
246
+ self._histogram.set_default_tags(labels)
247
+ return self
248
+
249
+ def observe(self, value: Union[int, float]):
250
+ return self._histogram.observe(value)
251
252
253
class RayMetrics(Metrics):
@@ -181,7 +255,9 @@ class RayMetrics(Metrics):
255
RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
256
Provides the same metrics as Metrics but uses Ray's util.metrics library.
257
"""
184
- _base_library = ray_metrics
258
+ _gauge_cls = _RayGaugeWrapper
259
+ _counter_cls = _RayCounterWrapper
260
+ _histogram_cls = _RayHistogramWrapper
261
262
def __init__(self, labelnames: List[str], max_model_len: int):
263
if ray_metrics is None:
@@ -192,8 +268,9 @@ class RayMetrics(Metrics):
268
# No-op on purpose
269
pass
270
195
-
196
-# end-metrics-definitions
271
+ def _create_info_cache_config(self) -> None:
272
+ # No-op on purpose
273
+ pass
274
275
276
def build_1_2_5_buckets(max_value: int) -> List[int]:
@@ -498,3 +575,6 @@ class PrometheusStatLogger(StatLoggerBase):
575
class RayPrometheusStatLogger(PrometheusStatLogger):
576
"""RayPrometheusStatLogger uses Ray metrics instead."""
577
_metrics_cls = RayMetrics
578
+
579
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
580
+ return None