A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
monitor metrics of tokens per step using cudagraph batchsizes (#11031)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 9, 2024 at 22:35 UTC
ebf778061db4e67c6903f8d6e8ad97c3db0174d8
4 files changed
+23
-13
tests/metrics/test_metrics.py
+1
-1
@@ -411,7 +411,7 @@ def test_engine_log_metrics_ray(
411
logger = _RayPrometheusStatLogger(
412
local_interval=0.5,
413
labels=dict(model_name=engine.model_config.served_model_name),
414
- max_model_len=engine.model_config.max_model_len)
414
+ vllm_config=engine.vllm_config)
415
engine.add_logger("ray", logger)
416
for i, prompt in enumerate(example_prompts):
417
engine.add_request(
vllm/engine/llm_engine.py
+4
-2
@@ -232,6 +232,7 @@ class LLMEngine:
232
use_cached_outputs: bool = False,
233
) -> None:
234
235
+ self.vllm_config = vllm_config
236
self.model_config = vllm_config.model_config
237
self.cache_config = vllm_config.cache_config
238
self.lora_config = vllm_config.lora_config
@@ -385,13 +386,14 @@ class LLMEngine:
386
self.stat_loggers = {
387
"logging":
388
LoggingStatLogger(
388
- local_interval=_LOCAL_LOGGING_INTERVAL_SEC),
389
+ local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
390
+ vllm_config=vllm_config),
391
"prometheus":
392
PrometheusStatLogger(
393
local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
394
labels=dict(
395
model_name=self.model_config.served_model_name),
394
- max_model_len=self.model_config.max_model_len),
396
+ vllm_config=vllm_config),
397
}
398
self.stat_loggers["prometheus"].info("cache_config",
399
self.cache_config)
vllm/engine/metrics.py
+16
-9
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Type, Union, cast
6
import numpy as np
7
import prometheus_client
8
9
+from vllm.config import VllmConfig
10
from vllm.engine.metrics_types import (StatLoggerBase, Stats,
11
SupportsMetricsInfo)
12
from vllm.executor.ray_utils import ray
@@ -44,10 +45,12 @@ class Metrics:
45
_counter_cls = prometheus_client.Counter
46
_histogram_cls = prometheus_client.Histogram
47
47
- def __init__(self, labelnames: List[str], max_model_len: int):
48
+ def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
49
# Unregister any existing vLLM collectors (for CI/CD)
50
self._unregister_vllm_metrics()
51
52
+ max_model_len = vllm_config.model_config.max_model_len
53
+
54
# System stats
55
# Scheduler State
56
self.gauge_scheduler_running = self._gauge_cls(
@@ -115,11 +118,15 @@ class Metrics:
118
name="vllm:tokens_total",
119
documentation="Number of prefill plus generation tokens processed.",
120
labelnames=labelnames)
121
+ buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
122
+ if not vllm_config.model_config.enforce_eager:
123
+ buckets = vllm_config.compilation_config.capture_sizes.copy()
124
+ buckets.sort()
125
self.histogram_iteration_tokens = self._histogram_cls(
126
name="vllm:iteration_tokens_total",
127
documentation="Histogram of number of tokens per engine_step.",
128
labelnames=labelnames,
122
- buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096])
129
+ buckets=buckets)
130
self.histogram_time_to_first_token = self._histogram_cls(
131
name="vllm:time_to_first_token_seconds",
132
documentation="Histogram of time to first token in seconds.",
@@ -361,10 +368,10 @@ class RayMetrics(Metrics):
368
_histogram_cls: Type[prometheus_client.Histogram] = cast(
369
Type[prometheus_client.Histogram], _RayHistogramWrapper)
370
364
- def __init__(self, labelnames: List[str], max_model_len: int):
371
+ def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
372
if ray_metrics is None:
373
raise ImportError("RayMetrics requires Ray to be installed.")
367
- super().__init__(labelnames, max_model_len)
374
+ super().__init__(labelnames, vllm_config)
375
376
def _unregister_vllm_metrics(self) -> None:
377
# No-op on purpose
@@ -421,8 +428,8 @@ def get_throughput(tracked_stats: List[int], now: float,
428
class LoggingStatLogger(StatLoggerBase):
429
"""LoggingStatLogger is used in LLMEngine to log to Stdout."""
430
424
- def __init__(self, *args, **kwargs) -> None:
425
- super().__init__(*args, **kwargs)
431
+ def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
432
+ super().__init__(local_interval, vllm_config)
433
self.last_prompt_throughput: Optional[float] = None
434
self.last_generation_throughput: Optional[float] = None
435
@@ -515,12 +522,12 @@ class PrometheusStatLogger(StatLoggerBase):
522
_gauge_cls = prometheus_client.Gauge
523
524
def __init__(self, local_interval: float, labels: Dict[str, str],
518
- max_model_len: int) -> None:
519
- super().__init__(local_interval)
525
+ vllm_config: VllmConfig) -> None:
526
+ super().__init__(local_interval, vllm_config)
527
# Prometheus metrics
528
self.labels = labels
529
self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
523
- max_model_len=max_model_len)
530
+ vllm_config=vllm_config)
531
532
def _log_gauge(self, gauge, data: Union[int, float]) -> None:
533
# Convenience function for logging to gauge.
vllm/engine/metrics_types.py
+2
-1
@@ -16,6 +16,7 @@ from abc import ABC, abstractmethod
16
from dataclasses import dataclass
17
from typing import Dict, List, Optional, Protocol
18
19
+from vllm.config import VllmConfig
20
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
21
22
@@ -77,7 +78,7 @@ class SupportsMetricsInfo(Protocol):
78
class StatLoggerBase(ABC):
79
"""Base class for StatLogger."""
80
80
- def __init__(self, local_interval: float) -> None:
81
+ def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
82
# Tracked stats over current local logging interval.
83
self.num_prompt_tokens: List[int] = []
84
self.num_generation_tokens: List[int] = []