A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] StatLoggers: cache spec decode metrics when they get collected. (#6645)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 24, 2024 at 01:05 UTC
2f808e69ab16a8775cd58849b15b465d4f11b92e
2 files changed
+122
-16
tests/metrics/test_metrics.py
+91
@@ -1,3 +1,4 @@
1
+import time
2
from typing import List
3
4
import pytest
@@ -10,6 +11,8 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.engine.metrics import RayPrometheusStatLogger
12
from vllm.sampling_params import SamplingParams
13
14
+from ..conftest import cleanup
15
+
16
MODELS = [
17
"facebook/opt-125m",
18
]
@@ -219,6 +222,94 @@ def test_metric_spec_decode(
222
"does not meet expectation")
223
224
225
+@pytest.mark.parametrize("model", MODELS)
226
+@pytest.mark.parametrize("dtype", ["half"])
227
+@pytest.mark.parametrize("max_tokens", [10])
228
+@pytest.mark.parametrize("log_interval", [1, 3, 5, 7])
229
+def test_metric_spec_decode_interval(
230
+ vllm_runner,
231
+ example_prompts,
232
+ model: str,
233
+ dtype: str,
234
+ max_tokens: int,
235
+ log_interval: int,
236
+) -> None:
237
+ k = 5
238
+
239
+ engine_args = EngineArgs(model=model,
240
+ dtype=dtype,
241
+ disable_log_stats=False,
242
+ gpu_memory_utilization=0.4,
243
+ speculative_model=model,
244
+ num_speculative_tokens=k,
245
+ use_v2_block_manager=True,
246
+ enforce_eager=True)
247
+
248
+ engine = LLMEngine.from_engine_args(engine_args)
249
+
250
+ try:
251
+
252
+ engine.add_request(
253
+ "request-id-0",
254
+ example_prompts[0],
255
+ SamplingParams(max_tokens=max_tokens),
256
+ )
257
+
258
+ # set log internal
259
+ stat_logger = engine.stat_loggers['prometheus']
260
+ stat_logger.local_interval = log_interval
261
+
262
+ # prefill
263
+ engine.step()
264
+
265
+ # wait for 5 seconds to ensure that spec decode metrics
266
+ # get triggered in first decode step
267
+ time.sleep(5)
268
+
269
+ # first decode step should trigger async collection of metrics
270
+ engine.step()
271
+
272
+ # wait one second to allow H2D transfer to finish
273
+ time.sleep(1)
274
+
275
+ # second decode step should now be able to collect the spec
276
+ # decode stats and the request should also be finished
277
+ engine.step()
278
+
279
+ # must have finisehd now
280
+ assert not engine.has_unfinished_requests()
281
+
282
+ # wait to ensure logging occurs
283
+ time.sleep(log_interval)
284
+
285
+ # force logging
286
+ engine.step()
287
+
288
+ # Note that the purpose of this test is to verify spec decode
289
+ # metrics instead of functional correctness, so the expected values
290
+ # are intended to be loose.
291
+ metric_name_to_expected_fn = {
292
+ "gauge_spec_decode_draft_acceptance_rate": lambda v: 0 <= v <= 1,
293
+ "gauge_spec_decode_efficiency": lambda v: 0 <= v <= 1,
294
+ "counter_spec_decode_num_accepted_tokens": lambda v: 0 <= v <= k,
295
+ "counter_spec_decode_num_draft_tokens": lambda v: v == k,
296
+ "counter_spec_decode_num_emitted_tokens":
297
+ lambda v: 0 <= v <= k + 1,
298
+ }
299
+
300
+ for metric_name, is_expected in metric_name_to_expected_fn.items():
301
+ metric_val = getattr(
302
+ stat_logger.metrics,
303
+ metric_name).labels(**stat_logger.labels)._value.get()
304
+ assert is_expected(metric_val), (
305
+ f"the value of metric {metric_name} ({metric_val}) "
306
+ "does not meet expectation")
307
+
308
+ finally:
309
+ del engine
310
+ cleanup()
311
+
312
+
313
def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
314
num_requests: int) -> None:
315
if disable_log_stats:
vllm/engine/metrics.py
+31
-16
@@ -355,6 +355,7 @@ class StatLoggerBase(ABC):
355
self.num_generation_tokens: List[int] = []
356
self.last_local_log = time.time()
357
self.local_interval = local_interval
358
+ self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
359
360
@abstractmethod
361
def info(self, type: str, obj: SupportsMetricsInfo) -> None:
@@ -364,6 +365,12 @@ class StatLoggerBase(ABC):
365
def log(self, stats: Stats) -> None:
366
raise NotImplementedError
367
368
+ def maybe_update_spec_decode_metrics(self, stats: Stats):
369
+ """Save spec decode metrics (since they are unlikely
370
+ to be emitted at same time as log interval)."""
371
+ if stats.spec_decode_metrics is not None:
372
+ self.spec_decode_metrics = stats.spec_decode_metrics
373
+
374
375
class LoggingStatLogger(StatLoggerBase):
376
"""LoggingStatLogger is used in LLMEngine to log to Stdout."""
@@ -379,6 +386,9 @@ class LoggingStatLogger(StatLoggerBase):
386
self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
387
self.num_generation_tokens.append(stats.num_generation_tokens_iter)
388
389
+ # Update spec decode metrics
390
+ self.maybe_update_spec_decode_metrics(stats)
391
+
392
# Log locally every local_interval seconds.
393
if local_interval_elapsed(stats.now, self.last_local_log,
394
self.local_interval):
@@ -408,15 +418,16 @@ class LoggingStatLogger(StatLoggerBase):
418
stats.cpu_cache_usage_sys * 100,
419
)
420
421
+ if self.spec_decode_metrics is not None:
422
+ logger.info(
423
+ self._format_spec_decode_metrics_str(
424
+ self.spec_decode_metrics))
425
+
426
# Reset tracked stats for next interval.
427
self.num_prompt_tokens = []
428
self.num_generation_tokens = []
429
self.last_local_log = stats.now
415
-
416
- if stats.spec_decode_metrics is not None:
417
- logger.info(
418
- self._format_spec_decode_metrics_str(
419
- stats.spec_decode_metrics))
430
+ self.spec_decode_metrics = None
431
432
def _format_spec_decode_metrics_str(
433
self, metrics: "SpecDecodeWorkerMetrics") -> str:
@@ -533,6 +544,9 @@ class PrometheusStatLogger(StatLoggerBase):
544
self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
545
self.num_generation_tokens.append(stats.num_generation_tokens_iter)
546
547
+ # Update spec decode metrics
548
+ self.maybe_update_spec_decode_metrics(stats)
549
+
550
# Log locally every local_interval seconds.
551
if local_interval_elapsed(stats.now, self.last_local_log,
552
self.local_interval):
@@ -550,26 +564,27 @@ class PrometheusStatLogger(StatLoggerBase):
564
prompt_throughput=prompt_throughput,
565
generation_throughput=generation_throughput)
566
553
- # Reset tracked stats for next interval.
554
- self.num_prompt_tokens = []
555
- self.num_generation_tokens = []
556
- self.last_local_log = stats.now
557
-
558
- if stats.spec_decode_metrics is not None:
567
+ if self.spec_decode_metrics is not None:
568
self._log_gauge(
569
self.metrics.gauge_spec_decode_draft_acceptance_rate,
561
- stats.spec_decode_metrics.draft_acceptance_rate)
570
+ self.spec_decode_metrics.draft_acceptance_rate)
571
self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
563
- stats.spec_decode_metrics.system_efficiency)
572
+ self.spec_decode_metrics.system_efficiency)
573
self._log_counter(
574
self.metrics.counter_spec_decode_num_accepted_tokens,
566
- stats.spec_decode_metrics.accepted_tokens)
575
+ self.spec_decode_metrics.accepted_tokens)
576
self._log_counter(
577
self.metrics.counter_spec_decode_num_draft_tokens,
569
- stats.spec_decode_metrics.draft_tokens)
578
+ self.spec_decode_metrics.draft_tokens)
579
self._log_counter(
580
self.metrics.counter_spec_decode_num_emitted_tokens,
572
- stats.spec_decode_metrics.emitted_tokens)
581
+ self.spec_decode_metrics.emitted_tokens)
582
+
583
+ # Reset tracked stats for next interval.
584
+ self.num_prompt_tokens = []
585
+ self.num_generation_tokens = []
586
+ self.last_local_log = stats.now
587
+ self.spec_decode_metrics = None
588
589
590
class RayPrometheusStatLogger(PrometheusStatLogger):