A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
add cache_config's info to prometheus metrics. (#3100)
Allen.Dou committed
Feb 29, 2024 at 14:15 UTC
9289e577ec185bd9feb2c03bb86b82f1bf9bb633
3 files changed
+14
-1
vllm/config.py
+4
@@ -308,6 +308,10 @@ class CacheConfig:
308
self.num_gpu_blocks = None
309
self.num_cpu_blocks = None
310
311
+ def metrics_info(self):
312
+ # convert cache_config to dict(key: str, value:str) for prometheus metrics info
313
+ return {key: str(value) for key, value in self.__dict__.items()}
314
+
315
def _verify_args(self) -> None:
316
if self.gpu_memory_utilization > 1.0:
317
raise ValueError(
vllm/engine/llm_engine.py
+1
@@ -138,6 +138,7 @@ class LLMEngine:
138
self.stat_logger = StatLogger(
139
local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
140
labels=dict(model_name=model_config.model))
141
+ self.stat_logger.info("cache_config", self.cache_config)
142
143
self.forward_dag = None
144
if USE_RAY_COMPILED_DAG:
vllm/engine/metrics.py
+9
-1
@@ -1,5 +1,5 @@
1
from vllm.logger import init_logger
2
-from prometheus_client import Counter, Gauge, Histogram, REGISTRY, disable_created_metrics
2
+from prometheus_client import Counter, Gauge, Histogram, Info, REGISTRY, disable_created_metrics
3
4
import time
5
import numpy as np
@@ -23,6 +23,10 @@ class Metrics:
23
if hasattr(collector, "_name") and "vllm" in collector._name:
24
REGISTRY.unregister(collector)
25
26
+ self.info_cache_config = Info(
27
+ name='vllm:cache_config',
28
+ documentation='information of cache_config')
29
+
30
# System stats
31
self.gauge_scheduler_running = Gauge(
32
name="vllm:num_requests_running",
@@ -128,6 +132,10 @@ class StatLogger:
132
self.labels = labels
133
self.metrics = Metrics(labelnames=list(labels.keys()))
134
135
+ def info(self, type: str, obj: object) -> None:
136
+ if type == "cache_config":
137
+ self.metrics.info_cache_config.info(obj.metrics_info())
138
+
139
def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
140
return float(np.sum(tracked_stats) / (now - self.last_local_log))
141