| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Metrics for dispatched sleep-mode frontend operations.""" |
| 4 | |
| 5 | from collections.abc import Iterator |
| 6 | from contextlib import contextmanager |
| 7 | from threading import Lock |
| 8 | from time import perf_counter |
| 9 | from typing import Literal |
| 10 | |
| 11 | from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram |
| 12 | |
| 13 | Operation = Literal["sleep", "release_kv_cache_memory", "wake"] |
| 14 | BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600) |
| 15 | |
| 16 | |
| 17 | class SleepModeOperationMetrics: |
| 18 | def __init__(self, registry: CollectorRegistry | None = None): |
| 19 | registry = REGISTRY if registry is None else registry |
| 20 | self.duration = Histogram( |
| 21 | "vllm:sleep_mode_operation_duration_seconds", |
| 22 | "Duration of one sleep-mode operation.", |
| 23 | ["operation"], |
| 24 | buckets=BUCKETS, |
| 25 | registry=registry, |
| 26 | ) |
| 27 | self.in_flight = Gauge( |
| 28 | "vllm:sleep_mode_operations_in_flight", |
| 29 | "Sleep-mode operations currently awaited.", |
| 30 | ["operation"], |
| 31 | multiprocess_mode="livesum", |
| 32 | registry=registry, |
| 33 | ) |
| 34 | |
| 35 | @contextmanager |
| 36 | def record(self, operation: Operation) -> Iterator[None]: |
| 37 | active = self.in_flight.labels(operation) |
| 38 | started = perf_counter() |
| 39 | active.inc() |
| 40 | try: |
| 41 | yield |
| 42 | finally: |
| 43 | active.dec() |
| 44 | self.duration.labels(operation).observe(perf_counter() - started) |
| 45 | |
| 46 | |
| 47 | _metrics: SleepModeOperationMetrics | None = None |
| 48 | _metrics_lock = Lock() |
| 49 | |
| 50 | |
| 51 | def sleep_mode_operation_metrics() -> SleepModeOperationMetrics: |
| 52 | global _metrics |
| 53 | if _metrics is None: |
| 54 | with _metrics_lock: |
| 55 | if _metrics is None: |
| 56 | _metrics = SleepModeOperationMetrics() |
| 57 | return _metrics |