@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9289e577ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

add cache_config's info to prometheus metrics. (#3100)

Allen.Dou committed Feb 29, 2024 at 14:15 UTC 9289e577ec185bd9feb2c03bb86b82f1bf9bb633
3 files changed +14 -1
vllm/config.py
+4
@@ -308,6 +308,10 @@ class CacheConfig:
308 self.num_gpu_blocks = None
309 self.num_cpu_blocks = None
310
311 + def metrics_info(self):
312 + # convert cache_config to dict(key: str, value:str) for prometheus metrics info
313 + return {key: str(value) for key, value in self.__dict__.items()}
314 +
315 def _verify_args(self) -> None:
316 if self.gpu_memory_utilization > 1.0:
317 raise ValueError(
vllm/engine/llm_engine.py
+1
@@ -138,6 +138,7 @@ class LLMEngine:
138 self.stat_logger = StatLogger(
139 local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
140 labels=dict(model_name=model_config.model))
141 + self.stat_logger.info("cache_config", self.cache_config)
142
143 self.forward_dag = None
144 if USE_RAY_COMPILED_DAG:
vllm/engine/metrics.py
+9 -1
@@ -1,5 +1,5 @@
1 from vllm.logger import init_logger
2 -from prometheus_client import Counter, Gauge, Histogram, REGISTRY, disable_created_metrics
2 +from prometheus_client import Counter, Gauge, Histogram, Info, REGISTRY, disable_created_metrics
3
4 import time
5 import numpy as np
@@ -23,6 +23,10 @@ class Metrics:
23 if hasattr(collector, "_name") and "vllm" in collector._name:
24 REGISTRY.unregister(collector)
25
26 + self.info_cache_config = Info(
27 + name='vllm:cache_config',
28 + documentation='information of cache_config')
29 +
30 # System stats
31 self.gauge_scheduler_running = Gauge(
32 name="vllm:num_requests_running",
@@ -128,6 +132,10 @@ class StatLogger:
132 self.labels = labels
133 self.metrics = Metrics(labelnames=list(labels.keys()))
134
135 + def info(self, type: str, obj: object) -> None:
136 + if type == "cache_config":
137 + self.metrics.info_cache_config.info(obj.metrics_info())
138 +
139 def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
140 return float(np.sum(tracked_stats) / (now - self.last_local_log))
141