A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[V1][Metrics] Support `vllm:cache_config_info` (#13299)
Mark McLoughlin committed
Feb 22, 2025 at 08:20 UTC
1cd981da4f90e1c313cbfdacbeef2fe417581828
5 files changed
+32
-12
tests/entrypoints/openai/test_metrics.py
+1
@@ -230,6 +230,7 @@ EXPECTED_METRICS_V1 = [
230
"vllm:prompt_tokens_total",
231
"vllm:generation_tokens_total",
232
"vllm:iteration_tokens_total",
233
+ "vllm:cache_config_info",
234
"vllm:request_success_total",
235
"vllm:request_prompt_tokens_sum",
236
"vllm:request_prompt_tokens_bucket",
vllm/config.py
+6
@@ -88,6 +88,12 @@ class SupportsHash(Protocol):
88
...
89
90
91
+class SupportsMetricsInfo(Protocol):
92
+
93
+ def metrics_info(self) -> Dict[str, str]:
94
+ ...
95
+
96
+
97
class ModelImpl(str, enum.Enum):
98
AUTO = "auto"
99
VLLM = "vllm"
vllm/engine/metrics.py
+2
-3
@@ -8,9 +8,8 @@ from typing import Dict, List, Optional, Type, Union, cast
8
import numpy as np
9
import prometheus_client
10
11
-from vllm.config import VllmConfig
12
-from vllm.engine.metrics_types import (StatLoggerBase, Stats,
13
- SupportsMetricsInfo)
11
+from vllm.config import SupportsMetricsInfo, VllmConfig
12
+from vllm.engine.metrics_types import StatLoggerBase, Stats
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15
vllm/engine/metrics_types.py
+2
-8
@@ -15,9 +15,9 @@ do this in Python code and lazily import prometheus_client.
15
import time
16
from abc import ABC, abstractmethod
17
from dataclasses import dataclass
18
-from typing import Dict, List, Optional, Protocol
18
+from typing import List, Optional
19
20
-from vllm.config import VllmConfig
20
+from vllm.config import SupportsMetricsInfo, VllmConfig
21
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
23
@@ -70,12 +70,6 @@ class Stats:
70
spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
71
72
73
-class SupportsMetricsInfo(Protocol):
74
-
75
- def metrics_info(self) -> Dict[str, str]:
76
- ...
77
-
78
-
73
class StatLoggerBase(ABC):
74
"""Base class for StatLogger."""
75
vllm/v1/metrics/loggers.py
+21
-1
@@ -7,7 +7,7 @@ from typing import Dict, List
7
import numpy as np
8
import prometheus_client
9
10
-from vllm.config import VllmConfig
10
+from vllm.config import SupportsMetricsInfo, VllmConfig
11
from vllm.logger import init_logger
12
from vllm.v1.core.kv_cache_utils import PrefixCachingMetrics
13
from vllm.v1.engine import FinishReason
@@ -228,6 +228,26 @@ class PrometheusStatLogger(StatLoggerBase):
228
buckets=request_latency_buckets,
229
labelnames=labelnames).labels(*labelvalues)
230
231
+ self.log_metrics_info("cache_config", vllm_config.cache_config)
232
+
233
+ def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
234
+ metrics_info = config_obj.metrics_info()
235
+
236
+ name, documentation = None, None
237
+ if type == "cache_config":
238
+ name = "vllm:cache_config_info"
239
+ documentation = "Information of the LLMEngine CacheConfig"
240
+ assert name is not None, f"Unknown metrics info type {type}"
241
+
242
+ # Info type metrics are syntactic sugar for a gauge permanently set to 1
243
+ # Since prometheus multiprocessing mode does not support Info, emulate
244
+ # info here with a gauge.
245
+ info_gauge = prometheus_client.Gauge(
246
+ name=name,
247
+ documentation=documentation,
248
+ labelnames=metrics_info.keys()).labels(**metrics_info)
249
+ info_gauge.set(1)
250
+
251
def log(self, scheduler_stats: SchedulerStats,
252
iteration_stats: IterationStats):
253
"""Log to prometheus."""