@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ebf778061d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

monitor metrics of tokens per step using cudagraph batchsizes (#11031)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 9, 2024 at 22:35 UTC ebf778061db4e67c6903f8d6e8ad97c3db0174d8
4 files changed +23 -13
tests/metrics/test_metrics.py
+1 -1
@@ -411,7 +411,7 @@ def test_engine_log_metrics_ray(
411 logger = _RayPrometheusStatLogger(
412 local_interval=0.5,
413 labels=dict(model_name=engine.model_config.served_model_name),
414 - max_model_len=engine.model_config.max_model_len)
414 + vllm_config=engine.vllm_config)
415 engine.add_logger("ray", logger)
416 for i, prompt in enumerate(example_prompts):
417 engine.add_request(
vllm/engine/llm_engine.py
+4 -2
@@ -232,6 +232,7 @@ class LLMEngine:
232 use_cached_outputs: bool = False,
233 ) -> None:
234
235 + self.vllm_config = vllm_config
236 self.model_config = vllm_config.model_config
237 self.cache_config = vllm_config.cache_config
238 self.lora_config = vllm_config.lora_config
@@ -385,13 +386,14 @@ class LLMEngine:
386 self.stat_loggers = {
387 "logging":
388 LoggingStatLogger(
388 - local_interval=_LOCAL_LOGGING_INTERVAL_SEC),
389 + local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
390 + vllm_config=vllm_config),
391 "prometheus":
392 PrometheusStatLogger(
393 local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
394 labels=dict(
395 model_name=self.model_config.served_model_name),
394 - max_model_len=self.model_config.max_model_len),
396 + vllm_config=vllm_config),
397 }
398 self.stat_loggers["prometheus"].info("cache_config",
399 self.cache_config)
vllm/engine/metrics.py
+16 -9
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Type, Union, cast
6 import numpy as np
7 import prometheus_client
8
9 +from vllm.config import VllmConfig
10 from vllm.engine.metrics_types import (StatLoggerBase, Stats,
11 SupportsMetricsInfo)
12 from vllm.executor.ray_utils import ray
@@ -44,10 +45,12 @@ class Metrics:
45 _counter_cls = prometheus_client.Counter
46 _histogram_cls = prometheus_client.Histogram
47
47 - def __init__(self, labelnames: List[str], max_model_len: int):
48 + def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
49 # Unregister any existing vLLM collectors (for CI/CD)
50 self._unregister_vllm_metrics()
51
52 + max_model_len = vllm_config.model_config.max_model_len
53 +
54 # System stats
55 # Scheduler State
56 self.gauge_scheduler_running = self._gauge_cls(
@@ -115,11 +118,15 @@ class Metrics:
118 name="vllm:tokens_total",
119 documentation="Number of prefill plus generation tokens processed.",
120 labelnames=labelnames)
121 + buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
122 + if not vllm_config.model_config.enforce_eager:
123 + buckets = vllm_config.compilation_config.capture_sizes.copy()
124 + buckets.sort()
125 self.histogram_iteration_tokens = self._histogram_cls(
126 name="vllm:iteration_tokens_total",
127 documentation="Histogram of number of tokens per engine_step.",
128 labelnames=labelnames,
122 - buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096])
129 + buckets=buckets)
130 self.histogram_time_to_first_token = self._histogram_cls(
131 name="vllm:time_to_first_token_seconds",
132 documentation="Histogram of time to first token in seconds.",
@@ -361,10 +368,10 @@ class RayMetrics(Metrics):
368 _histogram_cls: Type[prometheus_client.Histogram] = cast(
369 Type[prometheus_client.Histogram], _RayHistogramWrapper)
370
364 - def __init__(self, labelnames: List[str], max_model_len: int):
371 + def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
372 if ray_metrics is None:
373 raise ImportError("RayMetrics requires Ray to be installed.")
367 - super().__init__(labelnames, max_model_len)
374 + super().__init__(labelnames, vllm_config)
375
376 def _unregister_vllm_metrics(self) -> None:
377 # No-op on purpose
@@ -421,8 +428,8 @@ def get_throughput(tracked_stats: List[int], now: float,
428 class LoggingStatLogger(StatLoggerBase):
429 """LoggingStatLogger is used in LLMEngine to log to Stdout."""
430
424 - def __init__(self, *args, **kwargs) -> None:
425 - super().__init__(*args, **kwargs)
431 + def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
432 + super().__init__(local_interval, vllm_config)
433 self.last_prompt_throughput: Optional[float] = None
434 self.last_generation_throughput: Optional[float] = None
435
@@ -515,12 +522,12 @@ class PrometheusStatLogger(StatLoggerBase):
522 _gauge_cls = prometheus_client.Gauge
523
524 def __init__(self, local_interval: float, labels: Dict[str, str],
518 - max_model_len: int) -> None:
519 - super().__init__(local_interval)
525 + vllm_config: VllmConfig) -> None:
526 + super().__init__(local_interval, vllm_config)
527 # Prometheus metrics
528 self.labels = labels
529 self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
523 - max_model_len=max_model_len)
530 + vllm_config=vllm_config)
531
532 def _log_gauge(self, gauge, data: Union[int, float]) -> None:
533 # Convenience function for logging to gauge.
vllm/engine/metrics_types.py
+2 -1
@@ -16,6 +16,7 @@ from abc import ABC, abstractmethod
16 from dataclasses import dataclass
17 from typing import Dict, List, Optional, Protocol
18
19 +from vllm.config import VllmConfig
20 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22
@@ -77,7 +78,7 @@ class SupportsMetricsInfo(Protocol):
78 class StatLoggerBase(ABC):
79 """Base class for StatLogger."""
80
80 - def __init__(self, local_interval: float) -> None:
81 + def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
82 # Tracked stats over current local logging interval.
83 self.num_prompt_tokens: List[int] = []
84 self.num_generation_tokens: List[int] = []