A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 21:30 UTC
[Metrics] Add Prometheus counters for Model FLOPs Utilization (MFU) (#30950)
Export the existing Model FLOPs Utilization (MFU) metrics via Prometheus. `--enable-mfu-metrics` is required for these to be exposed. Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Feb 23, 2026 at 15:01 UTC
5cc7c4452e48b4492c47ff7e130751d7a786dbf9
5 files changed
+109
-1
docs/mkdocs/hooks/generate_metrics.py
+1
@@ -22,6 +22,7 @@ METRIC_SOURCE_FILES = [
22
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py",
23
"output": "nixl_connector.inc.md",
24
},
25
+ {"path": "vllm/v1/metrics/perf.py", "output": "perf.inc.md"},
26
]
27
28
docs/usage/metrics.md
+6
@@ -45,6 +45,12 @@ The following metrics are exposed:
45
46
--8<-- "docs/generated/metrics/nixl_connector.inc.md"
47
48
+## Model Flops Utilization (MFU) Performance Metrics
49
+
50
+These metrics are available via `--enable-mfu-metrics`:
51
+
52
+--8<-- "docs/generated/metrics/perf.inc.md"
53
+
54
## Deprecation Policy
55
56
Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1`
vllm/v1/metrics/loggers.py
+8
-1
@@ -19,7 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
19
from vllm.logger import init_logger
20
from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group
21
from vllm.v1.engine import FinishReason
22
-from vllm.v1.metrics.perf import PerfMetricsLogging
22
+from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm
23
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
24
from vllm.v1.metrics.stats import (
25
CachingMetrics,
@@ -392,6 +392,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
392
_histogram_cls = Histogram
393
_spec_decoding_cls = SpecDecodingProm
394
_kv_connector_cls = KVConnectorPrometheus
395
+ _perf_metrics_cls = PerfMetricsProm
396
397
def __init__(
398
self, vllm_config: VllmConfig, engine_indexes: list[int] | None = None
@@ -424,6 +425,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
425
self.kv_connector_prom = self._kv_connector_cls(
426
vllm_config, labelnames, per_engine_labelvalues
427
)
428
+ self.perf_metrics_prom = self._perf_metrics_cls(
429
+ vllm_config, labelnames, per_engine_labelvalues
430
+ )
431
432
#
433
# Scheduler state
@@ -1065,6 +1069,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1069
scheduler_stats.kv_connector_stats, engine_idx
1070
)
1071
1072
+ if scheduler_stats.perf_stats is not None:
1073
+ self.perf_metrics_prom.observe(scheduler_stats.perf_stats, engine_idx)
1074
+
1075
if (
1076
self.kv_cache_metrics_enabled
1077
and scheduler_stats.kv_cache_eviction_events
vllm/v1/metrics/perf.py
+82
@@ -13,6 +13,7 @@ from collections.abc import Iterable
13
from dataclasses import asdict, dataclass
14
from typing import Any, Protocol
15
16
+import prometheus_client
17
import torch
18
from pydantic import BaseModel, Field, ValidationError, model_validator
19
from typing_extensions import Self
@@ -1233,6 +1234,87 @@ class PerfMetricsLogging:
1234
self.reset()
1235
1236
1237
+#### Prometheus Integration ####
1238
+
1239
+
1240
+class PerfMetricsProm:
1241
+ """Record performance metrics in Prometheus.
1242
+
1243
+ Average TFLOPS (tera floating-point operations per second) can be
1244
+ calculated using a PromQL query:
1245
+
1246
+ rate(vllm:estimated_flops_per_gpu_total[1m]) / 1e12
1247
+
1248
+ Average memory bandwidth in GB/s can be calculated using:
1249
+
1250
+ (rate(vllm:estimated_read_bytes_per_gpu_total[1m]) +
1251
+ rate(vllm:estimated_write_bytes_per_gpu_total[1m])) / 1e9
1252
+ """
1253
+
1254
+ _counter_cls = prometheus_client.Counter
1255
+
1256
+ def __init__(
1257
+ self,
1258
+ vllm_config: VllmConfig,
1259
+ labelnames: list[str],
1260
+ per_engine_labelvalues: dict[int, list[object]],
1261
+ ):
1262
+ counter_flops = self._counter_cls(
1263
+ name="vllm:estimated_flops_per_gpu_total",
1264
+ documentation=(
1265
+ "Estimated number of floating point operations per GPU "
1266
+ "(for Model Flops Utilization calculations)."
1267
+ ),
1268
+ labelnames=labelnames,
1269
+ )
1270
+ self.counter_flops = make_per_engine(counter_flops, per_engine_labelvalues)
1271
+
1272
+ counter_read_bytes = self._counter_cls(
1273
+ name="vllm:estimated_read_bytes_per_gpu_total",
1274
+ documentation=(
1275
+ "Estimated number of bytes read from memory per GPU "
1276
+ "(for Model Flops Utilization calculations)."
1277
+ ),
1278
+ labelnames=labelnames,
1279
+ )
1280
+ self.counter_read_bytes = make_per_engine(
1281
+ counter_read_bytes, per_engine_labelvalues
1282
+ )
1283
+
1284
+ counter_write_bytes = self._counter_cls(
1285
+ name="vllm:estimated_write_bytes_per_gpu_total",
1286
+ documentation=(
1287
+ "Estimated number of bytes written to memory per GPU "
1288
+ "(for Model Flops Utilization calculations)."
1289
+ ),
1290
+ labelnames=labelnames,
1291
+ )
1292
+ self.counter_write_bytes = make_per_engine(
1293
+ counter_write_bytes, per_engine_labelvalues
1294
+ )
1295
+
1296
+ def observe(self, perf_stats: PerfStats, engine_idx: int = 0):
1297
+ if not (
1298
+ perf_stats.num_flops_per_gpu
1299
+ or perf_stats.num_read_bytes_per_gpu
1300
+ or perf_stats.num_write_bytes_per_gpu
1301
+ ):
1302
+ return
1303
+ self.counter_flops[engine_idx].inc(perf_stats.num_flops_per_gpu)
1304
+ self.counter_read_bytes[engine_idx].inc(perf_stats.num_read_bytes_per_gpu)
1305
+ self.counter_write_bytes[engine_idx].inc(perf_stats.num_write_bytes_per_gpu)
1306
+
1307
+
1308
+def make_per_engine(
1309
+ counter: prometheus_client.Counter, per_engine_labelvalues: dict[int, list[object]]
1310
+):
1311
+ """Create a counter for each label value."""
1312
+ return {
1313
+ idx: counter.labels(*labelvalues)
1314
+ for idx, labelvalues in per_engine_labelvalues.items()
1315
+ }
1316
+
1317
+
1318
## util functions
1319
1320
vllm/v1/metrics/ray_wrappers.py
+12
@@ -4,6 +4,7 @@ import time
4
5
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorPrometheus
6
from vllm.v1.metrics.loggers import PrometheusStatLogger
7
+from vllm.v1.metrics.perf import PerfMetricsProm
8
from vllm.v1.spec_decode.metrics import SpecDecodingProm
9
10
try:
@@ -179,6 +180,16 @@ class RayKVConnectorPrometheus(KVConnectorPrometheus):
180
_histogram_cls = RayHistogramWrapper
181
182
183
+class RayPerfMetricsProm(PerfMetricsProm):
184
+ """
185
+ RayPerfMetricsProm is used by RayMetrics to log Ray
186
+ metrics. Provides the same MFU metrics as PerfMetricsProm
187
+ uses Ray's util.metrics library.
188
+ """
189
+
190
+ _counter_cls = RayCounterWrapper
191
+
192
+
193
class RayPrometheusStatLogger(PrometheusStatLogger):
194
"""RayPrometheusStatLogger uses Ray metrics instead."""
195
@@ -187,6 +198,7 @@ class RayPrometheusStatLogger(PrometheusStatLogger):
198
_histogram_cls = RayHistogramWrapper
199
_spec_decoding_cls = RaySpecDecodingProm
200
_kv_connector_cls = RayKVConnectorPrometheus
201
+ _perf_metrics_cls = RayPerfMetricsProm
202
203
@staticmethod
204
def _unregister_vllm_metrics():