A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Misc] Extend vLLM Metrics logging API (#5925)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
William Lin committed
Jun 28, 2024 at 19:36 UTC
906a19cdb06b390f3dde287b06a3fe26c03a45e5
3 files changed
+225
-118
tests/metrics/test_metrics.py
+6
-6
@@ -39,7 +39,7 @@ def test_metric_counter_prompt_tokens(
39
vllm_prompt_token_count = sum(prompt_token_counts)
40
41
_ = vllm_model.generate_greedy(example_prompts, max_tokens)
42
- stat_logger = vllm_model.model.llm_engine.stat_logger
42
+ stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
43
metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
44
**stat_logger.labels)._value.get()
45
@@ -64,7 +64,7 @@ def test_metric_counter_generation_tokens(
64
gpu_memory_utilization=0.4) as vllm_model:
65
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66
tokenizer = vllm_model.model.get_tokenizer()
67
- stat_logger = vllm_model.model.llm_engine.stat_logger
67
+ stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
68
metric_count = stat_logger.metrics.counter_generation_tokens.labels(
69
**stat_logger.labels)._value.get()
70
vllm_generation_count = 0
@@ -92,7 +92,7 @@ def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
92
disable_log_stats=False,
93
gpu_memory_utilization=0.3,
94
served_model_name=served_model_name) as vllm_model:
95
- stat_logger = vllm_model.model.llm_engine.stat_logger
95
+ stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
96
metrics_tag_content = stat_logger.labels["model_name"]
97
98
if served_model_name is None or served_model_name == []:
@@ -172,10 +172,10 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
172
num_requests: int) -> None:
173
if disable_log_stats:
174
with pytest.raises(AttributeError):
175
- _ = engine.stat_logger
175
+ _ = engine.stat_loggers
176
else:
177
- assert (engine.stat_logger
178
- is not None), "engine.stat_logger should be set"
177
+ assert (engine.stat_loggers
178
+ is not None), "engine.stat_loggers should be set"
179
# Ensure the count bucket of request-level histogram metrics matches
180
# the number of requests as a simple sanity check to ensure metrics are
181
# generated
vllm/engine/llm_engine.py
+30
-8
@@ -13,7 +13,8 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
13
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
14
SchedulerOutputs)
15
from vllm.engine.arg_utils import EngineArgs
16
-from vllm.engine.metrics import StatLogger, Stats
16
+from vllm.engine.metrics import (LoggingStatLogger, PrometheusStatLogger,
17
+ StatLoggerBase, Stats)
18
from vllm.engine.output_processor.interfaces import (
19
SequenceGroupOutputProcessor)
20
from vllm.engine.output_processor.stop_checker import StopChecker
@@ -160,6 +161,7 @@ class LLMEngine:
161
executor_class: Type[ExecutorBase],
162
log_stats: bool,
163
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
164
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
165
) -> None:
166
logger.info(
167
"Initializing an LLM engine (v%s) with config: "
@@ -292,11 +294,21 @@ class LLMEngine:
294
295
# Metric Logging.
296
if self.log_stats:
295
- self.stat_logger = StatLogger(
296
- local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
297
- labels=dict(model_name=model_config.served_model_name),
298
- max_model_len=self.model_config.max_model_len)
299
- self.stat_logger.info("cache_config", self.cache_config)
297
+ if stat_loggers is not None:
298
+ self.stat_loggers = stat_loggers
299
+ else:
300
+ self.stat_loggers = {
301
+ "logging":
302
+ LoggingStatLogger(
303
+ local_interval=_LOCAL_LOGGING_INTERVAL_SEC),
304
+ "prometheus":
305
+ PrometheusStatLogger(
306
+ local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
307
+ labels=dict(model_name=model_config.served_model_name),
308
+ max_model_len=self.model_config.max_model_len),
309
+ }
310
+ self.stat_loggers["prometheus"].info("cache_config",
311
+ self.cache_config)
312
313
self.tracer = None
314
if self.observability_config.otlp_traces_endpoint:
@@ -833,14 +845,24 @@ class LLMEngine:
845
846
return request_outputs
847
848
+ def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
849
+ if logger_name in self.stat_loggers:
850
+ raise KeyError(f"Logger with name {logger_name} already exists.")
851
+ self.stat_loggers[logger_name] = logger
852
+
853
+ def remove_logger(self, logger_name: str) -> None:
854
+ if logger_name not in self.stat_loggers:
855
+ raise KeyError(f"Logger with name {logger_name} does not exist.")
856
+ del self.stat_loggers[logger_name]
857
+
858
def do_log_stats(
859
self,
860
scheduler_outputs: Optional[SchedulerOutputs] = None,
861
model_output: Optional[List[SamplerOutput]] = None) -> None:
862
"""Forced log when no requests active."""
863
if self.log_stats:
842
- self.stat_logger.log(
843
- self._get_stats(scheduler_outputs, model_output))
864
+ for logger in self.stat_loggers.values():
865
+ logger.log(self._get_stats(scheduler_outputs, model_output))
866
867
def _get_stats(
868
self,
vllm/engine/metrics.py
+189
-104
@@ -1,21 +1,27 @@
1
import time
2
+from abc import ABC, abstractmethod
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING
5
from typing import Counter as CollectionsCounter
6
from typing import Dict, List, Optional, Protocol, Union
7
8
import numpy as np
8
-from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
9
- disable_created_metrics)
9
+import prometheus_client
10
11
+from vllm.executor.ray_utils import ray
12
from vllm.logger import init_logger
13
14
+if ray is not None:
15
+ from ray.util import metrics as ray_metrics
16
+else:
17
+ ray_metrics = None
18
+
19
if TYPE_CHECKING:
20
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22
logger = init_logger(__name__)
23
18
-disable_created_metrics()
24
+prometheus_client.disable_created_metrics()
25
26
# The begin-* and end* here are used by the documentation generator
27
# to extract the metrics definitions.
@@ -24,56 +30,55 @@ disable_created_metrics()
30
# begin-metrics-definitions
31
class Metrics:
32
labelname_finish_reason = "finished_reason"
33
+ _base_library = prometheus_client
34
35
def __init__(self, labelnames: List[str], max_model_len: int):
36
# Unregister any existing vLLM collectors
30
- for collector in list(REGISTRY._collector_to_names):
31
- if hasattr(collector, "_name") and "vllm" in collector._name:
32
- REGISTRY.unregister(collector)
37
+ self._unregister_vllm_metrics()
38
39
# Config Information
35
- self.info_cache_config = Info(
40
+ self.info_cache_config = prometheus_client.Info(
41
name='vllm:cache_config',
42
documentation='information of cache_config')
43
44
# System stats
45
# Scheduler State
41
- self.gauge_scheduler_running = Gauge(
46
+ self.gauge_scheduler_running = self._base_library.Gauge(
47
name="vllm:num_requests_running",
48
documentation="Number of requests currently running on GPU.",
49
labelnames=labelnames)
45
- self.gauge_scheduler_waiting = Gauge(
50
+ self.gauge_scheduler_waiting = self._base_library.Gauge(
51
name="vllm:num_requests_waiting",
52
documentation="Number of requests waiting to be processed.",
53
labelnames=labelnames)
49
- self.gauge_scheduler_swapped = Gauge(
54
+ self.gauge_scheduler_swapped = self._base_library.Gauge(
55
name="vllm:num_requests_swapped",
56
documentation="Number of requests swapped to CPU.",
57
labelnames=labelnames)
58
# KV Cache Usage in %
54
- self.gauge_gpu_cache_usage = Gauge(
59
+ self.gauge_gpu_cache_usage = self._base_library.Gauge(
60
name="vllm:gpu_cache_usage_perc",
61
documentation="GPU KV-cache usage. 1 means 100 percent usage.",
62
labelnames=labelnames)
58
- self.gauge_cpu_cache_usage = Gauge(
63
+ self.gauge_cpu_cache_usage = self._base_library.Gauge(
64
name="vllm:cpu_cache_usage_perc",
65
documentation="CPU KV-cache usage. 1 means 100 percent usage.",
66
labelnames=labelnames)
67
68
# Iteration stats
64
- self.counter_num_preemption = Counter(
69
+ self.counter_num_preemption = self._base_library.Counter(
70
name="vllm:num_preemptions_total",
71
documentation="Cumulative number of preemption from the engine.",
72
labelnames=labelnames)
68
- self.counter_prompt_tokens = Counter(
73
+ self.counter_prompt_tokens = self._base_library.Counter(
74
name="vllm:prompt_tokens_total",
75
documentation="Number of prefill tokens processed.",
76
labelnames=labelnames)
72
- self.counter_generation_tokens = Counter(
77
+ self.counter_generation_tokens = self._base_library.Counter(
78
name="vllm:generation_tokens_total",
79
documentation="Number of generation tokens processed.",
80
labelnames=labelnames)
76
- self.histogram_time_to_first_token = Histogram(
81
+ self.histogram_time_to_first_token = self._base_library.Histogram(
82
name="vllm:time_to_first_token_seconds",
83
documentation="Histogram of time to first token in seconds.",
84
labelnames=labelnames,
@@ -81,7 +86,7 @@ class Metrics:
86
0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
87
0.75, 1.0, 2.5, 5.0, 7.5, 10.0
88
])
84
- self.histogram_time_per_output_token = Histogram(
89
+ self.histogram_time_per_output_token = self._base_library.Histogram(
90
name="vllm:time_per_output_token_seconds",
91
documentation="Histogram of time per output token in seconds.",
92
labelnames=labelnames,
@@ -92,54 +97,77 @@ class Metrics:
97
98
# Request stats
99
# Latency
95
- self.histogram_e2e_time_request = Histogram(
100
+ self.histogram_e2e_time_request = self._base_library.Histogram(
101
name="vllm:e2e_request_latency_seconds",
102
documentation="Histogram of end to end request latency in seconds.",
103
labelnames=labelnames,
104
buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
105
# Metadata
101
- self.histogram_num_prompt_tokens_request = Histogram(
106
+ self.histogram_num_prompt_tokens_request = self._base_library.Histogram(
107
name="vllm:request_prompt_tokens",
108
documentation="Number of prefill tokens processed.",
109
labelnames=labelnames,
110
buckets=build_1_2_5_buckets(max_model_len),
111
)
107
- self.histogram_num_generation_tokens_request = Histogram(
108
- name="vllm:request_generation_tokens",
109
- documentation="Number of generation tokens processed.",
110
- labelnames=labelnames,
111
- buckets=build_1_2_5_buckets(max_model_len),
112
- )
113
- self.histogram_best_of_request = Histogram(
112
+ self.histogram_num_generation_tokens_request = \
113
+ self._base_library.Histogram(
114
+ name="vllm:request_generation_tokens",
115
+ documentation="Number of generation tokens processed.",
116
+ labelnames=labelnames,
117
+ buckets=build_1_2_5_buckets(max_model_len),
118
+ )
119
+ self.histogram_best_of_request = self._base_library.Histogram(
120
name="vllm:request_params_best_of",
121
documentation="Histogram of the best_of request parameter.",
122
labelnames=labelnames,
123
buckets=[1, 2, 5, 10, 20],
124
)
119
- self.histogram_n_request = Histogram(
125
+ self.histogram_n_request = self._base_library.Histogram(
126
name="vllm:request_params_n",
127
documentation="Histogram of the n request parameter.",
128
labelnames=labelnames,
129
buckets=[1, 2, 5, 10, 20],
130
)
125
- self.counter_request_success = Counter(
131
+ self.counter_request_success = self._base_library.Counter(
132
name="vllm:request_success_total",
133
documentation="Count of successfully processed requests.",
134
labelnames=labelnames + [Metrics.labelname_finish_reason])
135
136
# Deprecated in favor of vllm:prompt_tokens_total
131
- self.gauge_avg_prompt_throughput = Gauge(
137
+ self.gauge_avg_prompt_throughput = self._base_library.Gauge(
138
name="vllm:avg_prompt_throughput_toks_per_s",
139
documentation="Average prefill throughput in tokens/s.",
140
labelnames=labelnames,
141
)
142
# Deprecated in favor of vllm:generation_tokens_total
137
- self.gauge_avg_generation_throughput = Gauge(
143
+ self.gauge_avg_generation_throughput = self._base_library.Gauge(
144
name="vllm:avg_generation_throughput_toks_per_s",
145
documentation="Average generation throughput in tokens/s.",
146
labelnames=labelnames,
147
)
148
149
+ def _unregister_vllm_metrics(self) -> None:
150
+ for collector in list(self._base_library.REGISTRY._collector_to_names):
151
+ if hasattr(collector, "_name") and "vllm" in collector._name:
152
+ self._base_library.REGISTRY.unregister(collector)
153
+
154
+
155
+class RayMetrics(Metrics):
156
+ """
157
+ RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
158
+ Provides the same metrics as Metrics but uses Ray's util.metrics library.
159
+ """
160
+ _base_library = ray_metrics
161
+
162
+ def __init__(self, labelnames: List[str], max_model_len: int):
163
+ if ray_metrics is None:
164
+ raise ImportError("RayMetrics requires Ray to be installed.")
165
+ super().__init__(labelnames, max_model_len)
166
+
167
+ def _unregister_vllm_metrics(self) -> None:
168
+ # No-op on purpose
169
+ pass
170
+
171
172
# end-metrics-definitions
173
@@ -206,34 +234,136 @@ class SupportsMetricsInfo(Protocol):
234
...
235
236
209
-class StatLogger:
210
- """StatLogger is used LLMEngine to log to Promethus and Stdout."""
237
+def local_interval_elapsed(now: float, last_log: float,
238
+ local_interval: float) -> bool:
239
+ elapsed_time = now - last_log
240
+ return elapsed_time > local_interval
241
+
242
+
243
+def get_throughput(tracked_stats: List[int], now: float,
244
+ last_log: float) -> float:
245
+ return float(np.sum(tracked_stats) / (now - last_log))
246
212
- def __init__(self, local_interval: float, labels: Dict[str, str],
213
- max_model_len: int) -> None:
214
- # Metadata for logging locally.
215
- self.last_local_log = time.time()
216
- self.local_interval = local_interval
247
248
+class StatLoggerBase(ABC):
249
+ """Base class for StatLogger."""
250
+
251
+ def __init__(self, local_interval: float) -> None:
252
# Tracked stats over current local logging interval.
253
self.num_prompt_tokens: List[int] = []
254
self.num_generation_tokens: List[int] = []
255
+ self.last_local_log = time.time()
256
+ self.local_interval = local_interval
257
+
258
+ @abstractmethod
259
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
260
+ raise NotImplementedError
261
+
262
+ @abstractmethod
263
+ def log(self, stats: Stats) -> None:
264
+ raise NotImplementedError
265
266
+
267
+class LoggingStatLogger(StatLoggerBase):
268
+ """LoggingStatLogger is used in LLMEngine to log to Stdout."""
269
+
270
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
271
+ raise NotImplementedError
272
+
273
+ def log(self, stats: Stats) -> None:
274
+ """Called by LLMEngine.
275
+ Logs to Stdout every self.local_interval seconds."""
276
+
277
+ # Save tracked stats for token counters.
278
+ self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
279
+ self.num_generation_tokens.append(stats.num_generation_tokens_iter)
280
+
281
+ # Log locally every local_interval seconds.
282
+ if local_interval_elapsed(stats.now, self.last_local_log,
283
+ self.local_interval):
284
+ # Compute summary metrics for tracked stats (and log them
285
+ # to promethus if applicable).
286
+ prompt_throughput = get_throughput(self.num_prompt_tokens,
287
+ now=stats.now,
288
+ last_log=self.last_local_log)
289
+ generation_throughput = get_throughput(
290
+ self.num_generation_tokens,
291
+ now=stats.now,
292
+ last_log=self.last_local_log)
293
+
294
+ # Log to stdout.
295
+ logger.info(
296
+ "Avg prompt throughput: %.1f tokens/s, "
297
+ "Avg generation throughput: %.1f tokens/s, "
298
+ "Running: %d reqs, Swapped: %d reqs, "
299
+ "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
300
+ "CPU KV cache usage: %.1f%%.",
301
+ prompt_throughput,
302
+ generation_throughput,
303
+ stats.num_running_sys,
304
+ stats.num_swapped_sys,
305
+ stats.num_waiting_sys,
306
+ stats.gpu_cache_usage_sys * 100,
307
+ stats.cpu_cache_usage_sys * 100,
308
+ )
309
+
310
+ # Reset tracked stats for next interval.
311
+ self.num_prompt_tokens = []
312
+ self.num_generation_tokens = []
313
+ self.last_local_log = stats.now
314
+
315
+ if stats.spec_decode_metrics is not None:
316
+ logger.info(
317
+ self._format_spec_decode_metrics_str(
318
+ stats.spec_decode_metrics))
319
+
320
+ def _format_spec_decode_metrics_str(
321
+ self, metrics: "SpecDecodeWorkerMetrics") -> str:
322
+
323
+ return ("Speculative metrics: "
324
+ f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
325
+ f"System efficiency: {metrics.system_efficiency:.3f}, "
326
+ f"Number of speculative tokens: {metrics.num_spec_tokens}, "
327
+ f"Number of accepted tokens: {metrics.accepted_tokens}, "
328
+ f"Number of draft tokens tokens: {metrics.draft_tokens}, "
329
+ f"Number of emitted tokens tokens: {metrics.emitted_tokens}.")
330
+
331
+
332
+class PrometheusStatLogger(StatLoggerBase):
333
+ """PrometheusStatLogger is used LLMEngine to log to Promethus."""
334
+ _metrics_cls = Metrics
335
+
336
+ def __init__(self, local_interval: float, labels: Dict[str, str],
337
+ max_model_len: int) -> None:
338
+ super().__init__(local_interval)
339
# Prometheus metrics
340
self.labels = labels
224
- self.metrics = Metrics(labelnames=list(labels.keys()),
225
- max_model_len=max_model_len)
341
+ self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
342
+ max_model_len=max_model_len)
343
344
def info(self, type: str, obj: SupportsMetricsInfo) -> None:
345
if type == "cache_config":
346
self.metrics.info_cache_config.info(obj.metrics_info())
347
231
- def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
232
- return float(np.sum(tracked_stats) / (now - self.last_local_log))
348
+ def _log_gauge(self, gauge, data: Union[int, float]) -> None:
349
+ # Convenience function for logging to gauge.
350
+ gauge.labels(**self.labels).set(data)
351
234
- def _local_interval_elapsed(self, now: float) -> bool:
235
- elapsed_time = now - self.last_local_log
236
- return elapsed_time > self.local_interval
352
+ def _log_counter(self, counter, data: Union[int, float]) -> None:
353
+ # Convenience function for logging to counter.
354
+ counter.labels(**self.labels).inc(data)
355
+
356
+ def _log_counter_labels(self, counter, data: CollectionsCounter,
357
+ label_key: str) -> None:
358
+ # Convenience function for collection counter of labels.
359
+ for label, count in data.items():
360
+ counter.labels(**{**self.labels, label_key: label}).inc(count)
361
+
362
+ def _log_histogram(self, histogram, data: Union[List[int],
363
+ List[float]]) -> None:
364
+ # Convenience function for logging list to histogram.
365
+ for datum in data:
366
+ histogram.labels(**self.labels).observe(datum)
367
368
def _log_prometheus(self, stats: Stats) -> None:
369
# System state data
@@ -279,26 +409,6 @@ class StatLogger:
409
self._log_histogram(self.metrics.histogram_best_of_request,
410
stats.best_of_requests)
411
282
- def _log_gauge(self, gauge: Gauge, data: Union[int, float]) -> None:
283
- # Convenience function for logging to gauge.
284
- gauge.labels(**self.labels).set(data)
285
-
286
- def _log_counter(self, counter: Counter, data: Union[int, float]) -> None:
287
- # Convenience function for logging to counter.
288
- counter.labels(**self.labels).inc(data)
289
-
290
- def _log_counter_labels(self, counter: Counter, data: CollectionsCounter,
291
- label_key: str) -> None:
292
- # Convenience function for collection counter of labels.
293
- for label, count in data.items():
294
- counter.labels(**{**self.labels, label_key: label}).inc(count)
295
-
296
- def _log_histogram(self, histogram: Histogram,
297
- data: Union[List[int], List[float]]) -> None:
298
- # Convenience function for logging list to histogram.
299
- for datum in data:
300
- histogram.labels(**self.labels).observe(datum)
301
-
412
def _log_prometheus_interval(self, prompt_throughput: float,
413
generation_throughput: float) -> None:
414
# Logs metrics to prometheus that are computed every logging_interval.
@@ -313,11 +423,8 @@ class StatLogger:
423
self.metrics.gauge_avg_generation_throughput.labels(
424
**self.labels).set(generation_throughput)
425
316
- def log(self, stats: Stats) -> None:
317
- """Called by LLMEngine.
318
- Logs to prometheus and tracked stats every iteration.
319
- Logs to Stdout every self.local_interval seconds."""
320
-
426
+ def log(self, stats: Stats):
427
+ """Logs to prometheus and tracked stats every iteration."""
428
# Log to prometheus.
429
self._log_prometheus(stats)
430
@@ -326,50 +433,28 @@ class StatLogger:
433
self.num_generation_tokens.append(stats.num_generation_tokens_iter)
434
435
# Log locally every local_interval seconds.
329
- if self._local_interval_elapsed(stats.now):
436
+ if local_interval_elapsed(stats.now, self.last_local_log,
437
+ self.local_interval):
438
# Compute summary metrics for tracked stats (and log them
439
# to promethus if applicable).
332
- prompt_throughput = self._get_throughput(self.num_prompt_tokens,
333
- now=stats.now)
334
- generation_throughput = self._get_throughput(
335
- self.num_generation_tokens, now=stats.now)
440
+ prompt_throughput = get_throughput(self.num_prompt_tokens,
441
+ now=stats.now,
442
+ last_log=self.last_local_log)
443
+ generation_throughput = get_throughput(
444
+ self.num_generation_tokens,
445
+ now=stats.now,
446
+ last_log=self.last_local_log)
447
+
448
self._log_prometheus_interval(
449
prompt_throughput=prompt_throughput,
450
generation_throughput=generation_throughput)
451
340
- # Log to stdout.
341
- logger.info(
342
- "Avg prompt throughput: %.1f tokens/s, "
343
- "Avg generation throughput: %.1f tokens/s, "
344
- "Running: %d reqs, Swapped: %d reqs, "
345
- "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
346
- "CPU KV cache usage: %.1f%%.",
347
- prompt_throughput,
348
- generation_throughput,
349
- stats.num_running_sys,
350
- stats.num_swapped_sys,
351
- stats.num_waiting_sys,
352
- stats.gpu_cache_usage_sys * 100,
353
- stats.cpu_cache_usage_sys * 100,
354
- )
355
-
452
# Reset tracked stats for next interval.
453
self.num_prompt_tokens = []
454
self.num_generation_tokens = []
455
self.last_local_log = stats.now
456
361
- if stats.spec_decode_metrics is not None:
362
- logger.info(
363
- self._format_spec_decode_metrics_str(
364
- stats.spec_decode_metrics))
365
-
366
- def _format_spec_decode_metrics_str(
367
- self, metrics: "SpecDecodeWorkerMetrics") -> str:
457
369
- return ("Speculative metrics: "
370
- f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
371
- f"System efficiency: {metrics.system_efficiency:.3f}, "
372
- f"Number of speculative tokens: {metrics.num_spec_tokens}, "
373
- f"Number of accepted tokens: {metrics.accepted_tokens}, "
374
- f"Number of draft tokens tokens: {metrics.draft_tokens}, "
375
- f"Number of emitted tokens tokens: {metrics.emitted_tokens}.")
458
+class RayPrometheusStatLogger(PrometheusStatLogger):
459
+ """RayPrometheusStatLogger uses Ray metrics instead."""
460
+ _metrics_cls = RayMetrics