@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 906a19cdb0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Misc] Extend vLLM Metrics logging API (#5925)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

William Lin committed Jun 28, 2024 at 19:36 UTC 906a19cdb06b390f3dde287b06a3fe26c03a45e5
3 files changed +225 -118
tests/metrics/test_metrics.py
+6 -6
@@ -39,7 +39,7 @@ def test_metric_counter_prompt_tokens(
39 vllm_prompt_token_count = sum(prompt_token_counts)
40
41 _ = vllm_model.generate_greedy(example_prompts, max_tokens)
42 - stat_logger = vllm_model.model.llm_engine.stat_logger
42 + stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
43 metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
44 **stat_logger.labels)._value.get()
45
@@ -64,7 +64,7 @@ def test_metric_counter_generation_tokens(
64 gpu_memory_utilization=0.4) as vllm_model:
65 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66 tokenizer = vllm_model.model.get_tokenizer()
67 - stat_logger = vllm_model.model.llm_engine.stat_logger
67 + stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
68 metric_count = stat_logger.metrics.counter_generation_tokens.labels(
69 **stat_logger.labels)._value.get()
70 vllm_generation_count = 0
@@ -92,7 +92,7 @@ def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
92 disable_log_stats=False,
93 gpu_memory_utilization=0.3,
94 served_model_name=served_model_name) as vllm_model:
95 - stat_logger = vllm_model.model.llm_engine.stat_logger
95 + stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
96 metrics_tag_content = stat_logger.labels["model_name"]
97
98 if served_model_name is None or served_model_name == []:
@@ -172,10 +172,10 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
172 num_requests: int) -> None:
173 if disable_log_stats:
174 with pytest.raises(AttributeError):
175 - _ = engine.stat_logger
175 + _ = engine.stat_loggers
176 else:
177 - assert (engine.stat_logger
178 - is not None), "engine.stat_logger should be set"
177 + assert (engine.stat_loggers
178 + is not None), "engine.stat_loggers should be set"
179 # Ensure the count bucket of request-level histogram metrics matches
180 # the number of requests as a simple sanity check to ensure metrics are
181 # generated
vllm/engine/llm_engine.py
+30 -8
@@ -13,7 +13,8 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
13 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
14 SchedulerOutputs)
15 from vllm.engine.arg_utils import EngineArgs
16 -from vllm.engine.metrics import StatLogger, Stats
16 +from vllm.engine.metrics import (LoggingStatLogger, PrometheusStatLogger,
17 + StatLoggerBase, Stats)
18 from vllm.engine.output_processor.interfaces import (
19 SequenceGroupOutputProcessor)
20 from vllm.engine.output_processor.stop_checker import StopChecker
@@ -160,6 +161,7 @@ class LLMEngine:
161 executor_class: Type[ExecutorBase],
162 log_stats: bool,
163 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
164 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
165 ) -> None:
166 logger.info(
167 "Initializing an LLM engine (v%s) with config: "
@@ -292,11 +294,21 @@ class LLMEngine:
294
295 # Metric Logging.
296 if self.log_stats:
295 - self.stat_logger = StatLogger(
296 - local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
297 - labels=dict(model_name=model_config.served_model_name),
298 - max_model_len=self.model_config.max_model_len)
299 - self.stat_logger.info("cache_config", self.cache_config)
297 + if stat_loggers is not None:
298 + self.stat_loggers = stat_loggers
299 + else:
300 + self.stat_loggers = {
301 + "logging":
302 + LoggingStatLogger(
303 + local_interval=_LOCAL_LOGGING_INTERVAL_SEC),
304 + "prometheus":
305 + PrometheusStatLogger(
306 + local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
307 + labels=dict(model_name=model_config.served_model_name),
308 + max_model_len=self.model_config.max_model_len),
309 + }
310 + self.stat_loggers["prometheus"].info("cache_config",
311 + self.cache_config)
312
313 self.tracer = None
314 if self.observability_config.otlp_traces_endpoint:
@@ -833,14 +845,24 @@ class LLMEngine:
845
846 return request_outputs
847
848 + def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
849 + if logger_name in self.stat_loggers:
850 + raise KeyError(f"Logger with name {logger_name} already exists.")
851 + self.stat_loggers[logger_name] = logger
852 +
853 + def remove_logger(self, logger_name: str) -> None:
854 + if logger_name not in self.stat_loggers:
855 + raise KeyError(f"Logger with name {logger_name} does not exist.")
856 + del self.stat_loggers[logger_name]
857 +
858 def do_log_stats(
859 self,
860 scheduler_outputs: Optional[SchedulerOutputs] = None,
861 model_output: Optional[List[SamplerOutput]] = None) -> None:
862 """Forced log when no requests active."""
863 if self.log_stats:
842 - self.stat_logger.log(
843 - self._get_stats(scheduler_outputs, model_output))
864 + for logger in self.stat_loggers.values():
865 + logger.log(self._get_stats(scheduler_outputs, model_output))
866
867 def _get_stats(
868 self,
vllm/engine/metrics.py
+189 -104
@@ -1,21 +1,27 @@
1 import time
2 +from abc import ABC, abstractmethod
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING
5 from typing import Counter as CollectionsCounter
6 from typing import Dict, List, Optional, Protocol, Union
7
8 import numpy as np
8 -from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
9 - disable_created_metrics)
9 +import prometheus_client
10
11 +from vllm.executor.ray_utils import ray
12 from vllm.logger import init_logger
13
14 +if ray is not None:
15 + from ray.util import metrics as ray_metrics
16 +else:
17 + ray_metrics = None
18 +
19 if TYPE_CHECKING:
20 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22 logger = init_logger(__name__)
23
18 -disable_created_metrics()
24 +prometheus_client.disable_created_metrics()
25
26 # The begin-* and end* here are used by the documentation generator
27 # to extract the metrics definitions.
@@ -24,56 +30,55 @@ disable_created_metrics()
30 # begin-metrics-definitions
31 class Metrics:
32 labelname_finish_reason = "finished_reason"
33 + _base_library = prometheus_client
34
35 def __init__(self, labelnames: List[str], max_model_len: int):
36 # Unregister any existing vLLM collectors
30 - for collector in list(REGISTRY._collector_to_names):
31 - if hasattr(collector, "_name") and "vllm" in collector._name:
32 - REGISTRY.unregister(collector)
37 + self._unregister_vllm_metrics()
38
39 # Config Information
35 - self.info_cache_config = Info(
40 + self.info_cache_config = prometheus_client.Info(
41 name='vllm:cache_config',
42 documentation='information of cache_config')
43
44 # System stats
45 # Scheduler State
41 - self.gauge_scheduler_running = Gauge(
46 + self.gauge_scheduler_running = self._base_library.Gauge(
47 name="vllm:num_requests_running",
48 documentation="Number of requests currently running on GPU.",
49 labelnames=labelnames)
45 - self.gauge_scheduler_waiting = Gauge(
50 + self.gauge_scheduler_waiting = self._base_library.Gauge(
51 name="vllm:num_requests_waiting",
52 documentation="Number of requests waiting to be processed.",
53 labelnames=labelnames)
49 - self.gauge_scheduler_swapped = Gauge(
54 + self.gauge_scheduler_swapped = self._base_library.Gauge(
55 name="vllm:num_requests_swapped",
56 documentation="Number of requests swapped to CPU.",
57 labelnames=labelnames)
58 # KV Cache Usage in %
54 - self.gauge_gpu_cache_usage = Gauge(
59 + self.gauge_gpu_cache_usage = self._base_library.Gauge(
60 name="vllm:gpu_cache_usage_perc",
61 documentation="GPU KV-cache usage. 1 means 100 percent usage.",
62 labelnames=labelnames)
58 - self.gauge_cpu_cache_usage = Gauge(
63 + self.gauge_cpu_cache_usage = self._base_library.Gauge(
64 name="vllm:cpu_cache_usage_perc",
65 documentation="CPU KV-cache usage. 1 means 100 percent usage.",
66 labelnames=labelnames)
67
68 # Iteration stats
64 - self.counter_num_preemption = Counter(
69 + self.counter_num_preemption = self._base_library.Counter(
70 name="vllm:num_preemptions_total",
71 documentation="Cumulative number of preemption from the engine.",
72 labelnames=labelnames)
68 - self.counter_prompt_tokens = Counter(
73 + self.counter_prompt_tokens = self._base_library.Counter(
74 name="vllm:prompt_tokens_total",
75 documentation="Number of prefill tokens processed.",
76 labelnames=labelnames)
72 - self.counter_generation_tokens = Counter(
77 + self.counter_generation_tokens = self._base_library.Counter(
78 name="vllm:generation_tokens_total",
79 documentation="Number of generation tokens processed.",
80 labelnames=labelnames)
76 - self.histogram_time_to_first_token = Histogram(
81 + self.histogram_time_to_first_token = self._base_library.Histogram(
82 name="vllm:time_to_first_token_seconds",
83 documentation="Histogram of time to first token in seconds.",
84 labelnames=labelnames,
@@ -81,7 +86,7 @@ class Metrics:
86 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
87 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
88 ])
84 - self.histogram_time_per_output_token = Histogram(
89 + self.histogram_time_per_output_token = self._base_library.Histogram(
90 name="vllm:time_per_output_token_seconds",
91 documentation="Histogram of time per output token in seconds.",
92 labelnames=labelnames,
@@ -92,54 +97,77 @@ class Metrics:
97
98 # Request stats
99 # Latency
95 - self.histogram_e2e_time_request = Histogram(
100 + self.histogram_e2e_time_request = self._base_library.Histogram(
101 name="vllm:e2e_request_latency_seconds",
102 documentation="Histogram of end to end request latency in seconds.",
103 labelnames=labelnames,
104 buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
105 # Metadata
101 - self.histogram_num_prompt_tokens_request = Histogram(
106 + self.histogram_num_prompt_tokens_request = self._base_library.Histogram(
107 name="vllm:request_prompt_tokens",
108 documentation="Number of prefill tokens processed.",
109 labelnames=labelnames,
110 buckets=build_1_2_5_buckets(max_model_len),
111 )
107 - self.histogram_num_generation_tokens_request = Histogram(
108 - name="vllm:request_generation_tokens",
109 - documentation="Number of generation tokens processed.",
110 - labelnames=labelnames,
111 - buckets=build_1_2_5_buckets(max_model_len),
112 - )
113 - self.histogram_best_of_request = Histogram(
112 + self.histogram_num_generation_tokens_request = \
113 + self._base_library.Histogram(
114 + name="vllm:request_generation_tokens",
115 + documentation="Number of generation tokens processed.",
116 + labelnames=labelnames,
117 + buckets=build_1_2_5_buckets(max_model_len),
118 + )
119 + self.histogram_best_of_request = self._base_library.Histogram(
120 name="vllm:request_params_best_of",
121 documentation="Histogram of the best_of request parameter.",
122 labelnames=labelnames,
123 buckets=[1, 2, 5, 10, 20],
124 )
119 - self.histogram_n_request = Histogram(
125 + self.histogram_n_request = self._base_library.Histogram(
126 name="vllm:request_params_n",
127 documentation="Histogram of the n request parameter.",
128 labelnames=labelnames,
129 buckets=[1, 2, 5, 10, 20],
130 )
125 - self.counter_request_success = Counter(
131 + self.counter_request_success = self._base_library.Counter(
132 name="vllm:request_success_total",
133 documentation="Count of successfully processed requests.",
134 labelnames=labelnames + [Metrics.labelname_finish_reason])
135
136 # Deprecated in favor of vllm:prompt_tokens_total
131 - self.gauge_avg_prompt_throughput = Gauge(
137 + self.gauge_avg_prompt_throughput = self._base_library.Gauge(
138 name="vllm:avg_prompt_throughput_toks_per_s",
139 documentation="Average prefill throughput in tokens/s.",
140 labelnames=labelnames,
141 )
142 # Deprecated in favor of vllm:generation_tokens_total
137 - self.gauge_avg_generation_throughput = Gauge(
143 + self.gauge_avg_generation_throughput = self._base_library.Gauge(
144 name="vllm:avg_generation_throughput_toks_per_s",
145 documentation="Average generation throughput in tokens/s.",
146 labelnames=labelnames,
147 )
148
149 + def _unregister_vllm_metrics(self) -> None:
150 + for collector in list(self._base_library.REGISTRY._collector_to_names):
151 + if hasattr(collector, "_name") and "vllm" in collector._name:
152 + self._base_library.REGISTRY.unregister(collector)
153 +
154 +
155 +class RayMetrics(Metrics):
156 + """
157 + RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
158 + Provides the same metrics as Metrics but uses Ray's util.metrics library.
159 + """
160 + _base_library = ray_metrics
161 +
162 + def __init__(self, labelnames: List[str], max_model_len: int):
163 + if ray_metrics is None:
164 + raise ImportError("RayMetrics requires Ray to be installed.")
165 + super().__init__(labelnames, max_model_len)
166 +
167 + def _unregister_vllm_metrics(self) -> None:
168 + # No-op on purpose
169 + pass
170 +
171
172 # end-metrics-definitions
173
@@ -206,34 +234,136 @@ class SupportsMetricsInfo(Protocol):
234 ...
235
236
209 -class StatLogger:
210 - """StatLogger is used LLMEngine to log to Promethus and Stdout."""
237 +def local_interval_elapsed(now: float, last_log: float,
238 + local_interval: float) -> bool:
239 + elapsed_time = now - last_log
240 + return elapsed_time > local_interval
241 +
242 +
243 +def get_throughput(tracked_stats: List[int], now: float,
244 + last_log: float) -> float:
245 + return float(np.sum(tracked_stats) / (now - last_log))
246
212 - def __init__(self, local_interval: float, labels: Dict[str, str],
213 - max_model_len: int) -> None:
214 - # Metadata for logging locally.
215 - self.last_local_log = time.time()
216 - self.local_interval = local_interval
247
248 +class StatLoggerBase(ABC):
249 + """Base class for StatLogger."""
250 +
251 + def __init__(self, local_interval: float) -> None:
252 # Tracked stats over current local logging interval.
253 self.num_prompt_tokens: List[int] = []
254 self.num_generation_tokens: List[int] = []
255 + self.last_local_log = time.time()
256 + self.local_interval = local_interval
257 +
258 + @abstractmethod
259 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
260 + raise NotImplementedError
261 +
262 + @abstractmethod
263 + def log(self, stats: Stats) -> None:
264 + raise NotImplementedError
265
266 +
267 +class LoggingStatLogger(StatLoggerBase):
268 + """LoggingStatLogger is used in LLMEngine to log to Stdout."""
269 +
270 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
271 + raise NotImplementedError
272 +
273 + def log(self, stats: Stats) -> None:
274 + """Called by LLMEngine.
275 + Logs to Stdout every self.local_interval seconds."""
276 +
277 + # Save tracked stats for token counters.
278 + self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
279 + self.num_generation_tokens.append(stats.num_generation_tokens_iter)
280 +
281 + # Log locally every local_interval seconds.
282 + if local_interval_elapsed(stats.now, self.last_local_log,
283 + self.local_interval):
284 + # Compute summary metrics for tracked stats (and log them
285 + # to promethus if applicable).
286 + prompt_throughput = get_throughput(self.num_prompt_tokens,
287 + now=stats.now,
288 + last_log=self.last_local_log)
289 + generation_throughput = get_throughput(
290 + self.num_generation_tokens,
291 + now=stats.now,
292 + last_log=self.last_local_log)
293 +
294 + # Log to stdout.
295 + logger.info(
296 + "Avg prompt throughput: %.1f tokens/s, "
297 + "Avg generation throughput: %.1f tokens/s, "
298 + "Running: %d reqs, Swapped: %d reqs, "
299 + "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
300 + "CPU KV cache usage: %.1f%%.",
301 + prompt_throughput,
302 + generation_throughput,
303 + stats.num_running_sys,
304 + stats.num_swapped_sys,
305 + stats.num_waiting_sys,
306 + stats.gpu_cache_usage_sys * 100,
307 + stats.cpu_cache_usage_sys * 100,
308 + )
309 +
310 + # Reset tracked stats for next interval.
311 + self.num_prompt_tokens = []
312 + self.num_generation_tokens = []
313 + self.last_local_log = stats.now
314 +
315 + if stats.spec_decode_metrics is not None:
316 + logger.info(
317 + self._format_spec_decode_metrics_str(
318 + stats.spec_decode_metrics))
319 +
320 + def _format_spec_decode_metrics_str(
321 + self, metrics: "SpecDecodeWorkerMetrics") -> str:
322 +
323 + return ("Speculative metrics: "
324 + f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
325 + f"System efficiency: {metrics.system_efficiency:.3f}, "
326 + f"Number of speculative tokens: {metrics.num_spec_tokens}, "
327 + f"Number of accepted tokens: {metrics.accepted_tokens}, "
328 + f"Number of draft tokens tokens: {metrics.draft_tokens}, "
329 + f"Number of emitted tokens tokens: {metrics.emitted_tokens}.")
330 +
331 +
332 +class PrometheusStatLogger(StatLoggerBase):
333 + """PrometheusStatLogger is used LLMEngine to log to Promethus."""
334 + _metrics_cls = Metrics
335 +
336 + def __init__(self, local_interval: float, labels: Dict[str, str],
337 + max_model_len: int) -> None:
338 + super().__init__(local_interval)
339 # Prometheus metrics
340 self.labels = labels
224 - self.metrics = Metrics(labelnames=list(labels.keys()),
225 - max_model_len=max_model_len)
341 + self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
342 + max_model_len=max_model_len)
343
344 def info(self, type: str, obj: SupportsMetricsInfo) -> None:
345 if type == "cache_config":
346 self.metrics.info_cache_config.info(obj.metrics_info())
347
231 - def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
232 - return float(np.sum(tracked_stats) / (now - self.last_local_log))
348 + def _log_gauge(self, gauge, data: Union[int, float]) -> None:
349 + # Convenience function for logging to gauge.
350 + gauge.labels(**self.labels).set(data)
351
234 - def _local_interval_elapsed(self, now: float) -> bool:
235 - elapsed_time = now - self.last_local_log
236 - return elapsed_time > self.local_interval
352 + def _log_counter(self, counter, data: Union[int, float]) -> None:
353 + # Convenience function for logging to counter.
354 + counter.labels(**self.labels).inc(data)
355 +
356 + def _log_counter_labels(self, counter, data: CollectionsCounter,
357 + label_key: str) -> None:
358 + # Convenience function for collection counter of labels.
359 + for label, count in data.items():
360 + counter.labels(**{**self.labels, label_key: label}).inc(count)
361 +
362 + def _log_histogram(self, histogram, data: Union[List[int],
363 + List[float]]) -> None:
364 + # Convenience function for logging list to histogram.
365 + for datum in data:
366 + histogram.labels(**self.labels).observe(datum)
367
368 def _log_prometheus(self, stats: Stats) -> None:
369 # System state data
@@ -279,26 +409,6 @@ class StatLogger:
409 self._log_histogram(self.metrics.histogram_best_of_request,
410 stats.best_of_requests)
411
282 - def _log_gauge(self, gauge: Gauge, data: Union[int, float]) -> None:
283 - # Convenience function for logging to gauge.
284 - gauge.labels(**self.labels).set(data)
285 -
286 - def _log_counter(self, counter: Counter, data: Union[int, float]) -> None:
287 - # Convenience function for logging to counter.
288 - counter.labels(**self.labels).inc(data)
289 -
290 - def _log_counter_labels(self, counter: Counter, data: CollectionsCounter,
291 - label_key: str) -> None:
292 - # Convenience function for collection counter of labels.
293 - for label, count in data.items():
294 - counter.labels(**{**self.labels, label_key: label}).inc(count)
295 -
296 - def _log_histogram(self, histogram: Histogram,
297 - data: Union[List[int], List[float]]) -> None:
298 - # Convenience function for logging list to histogram.
299 - for datum in data:
300 - histogram.labels(**self.labels).observe(datum)
301 -
412 def _log_prometheus_interval(self, prompt_throughput: float,
413 generation_throughput: float) -> None:
414 # Logs metrics to prometheus that are computed every logging_interval.
@@ -313,11 +423,8 @@ class StatLogger:
423 self.metrics.gauge_avg_generation_throughput.labels(
424 **self.labels).set(generation_throughput)
425
316 - def log(self, stats: Stats) -> None:
317 - """Called by LLMEngine.
318 - Logs to prometheus and tracked stats every iteration.
319 - Logs to Stdout every self.local_interval seconds."""
320 -
426 + def log(self, stats: Stats):
427 + """Logs to prometheus and tracked stats every iteration."""
428 # Log to prometheus.
429 self._log_prometheus(stats)
430
@@ -326,50 +433,28 @@ class StatLogger:
433 self.num_generation_tokens.append(stats.num_generation_tokens_iter)
434
435 # Log locally every local_interval seconds.
329 - if self._local_interval_elapsed(stats.now):
436 + if local_interval_elapsed(stats.now, self.last_local_log,
437 + self.local_interval):
438 # Compute summary metrics for tracked stats (and log them
439 # to promethus if applicable).
332 - prompt_throughput = self._get_throughput(self.num_prompt_tokens,
333 - now=stats.now)
334 - generation_throughput = self._get_throughput(
335 - self.num_generation_tokens, now=stats.now)
440 + prompt_throughput = get_throughput(self.num_prompt_tokens,
441 + now=stats.now,
442 + last_log=self.last_local_log)
443 + generation_throughput = get_throughput(
444 + self.num_generation_tokens,
445 + now=stats.now,
446 + last_log=self.last_local_log)
447 +
448 self._log_prometheus_interval(
449 prompt_throughput=prompt_throughput,
450 generation_throughput=generation_throughput)
451
340 - # Log to stdout.
341 - logger.info(
342 - "Avg prompt throughput: %.1f tokens/s, "
343 - "Avg generation throughput: %.1f tokens/s, "
344 - "Running: %d reqs, Swapped: %d reqs, "
345 - "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
346 - "CPU KV cache usage: %.1f%%.",
347 - prompt_throughput,
348 - generation_throughput,
349 - stats.num_running_sys,
350 - stats.num_swapped_sys,
351 - stats.num_waiting_sys,
352 - stats.gpu_cache_usage_sys * 100,
353 - stats.cpu_cache_usage_sys * 100,
354 - )
355 -
452 # Reset tracked stats for next interval.
453 self.num_prompt_tokens = []
454 self.num_generation_tokens = []
455 self.last_local_log = stats.now
456
361 - if stats.spec_decode_metrics is not None:
362 - logger.info(
363 - self._format_spec_decode_metrics_str(
364 - stats.spec_decode_metrics))
365 -
366 - def _format_spec_decode_metrics_str(
367 - self, metrics: "SpecDecodeWorkerMetrics") -> str:
457
369 - return ("Speculative metrics: "
370 - f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
371 - f"System efficiency: {metrics.system_efficiency:.3f}, "
372 - f"Number of speculative tokens: {metrics.num_spec_tokens}, "
373 - f"Number of accepted tokens: {metrics.accepted_tokens}, "
374 - f"Number of draft tokens tokens: {metrics.draft_tokens}, "
375 - f"Number of emitted tokens tokens: {metrics.emitted_tokens}.")
458 +class RayPrometheusStatLogger(PrometheusStatLogger):
459 + """RayPrometheusStatLogger uses Ray metrics instead."""
460 + _metrics_cls = RayMetrics