A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[MISC] Add lora requests to metrics (#9477)
Co-authored-by: Kunjan Patel <kunjanp_google_com@vllm.us-central1-a.c.kunjanp-gke-dev-2.internal>
Kunjan committed
Oct 18, 2024 at 13:50 UTC
9bb10a7d276e085c72f2545cea1a3565937e7b22
3 files changed
+54
-2
vllm/engine/llm_engine.py
+23
-1
@@ -1,4 +1,5 @@
1
import time
2
+from collections import Counter as collectionsCounter
3
from collections import deque
4
from contextlib import contextmanager
5
from dataclasses import dataclass
@@ -1617,6 +1618,25 @@ class LLMEngine:
1618
n_requests: List[int] = []
1619
finished_reason_requests: List[str] = []
1620
1621
+ # Lora requests
1622
+ running_lora_adapters = dict(
1623
+ collectionsCounter([
1624
+ running_request.lora_request.lora_name
1625
+ for scheduler in self.scheduler
1626
+ for running_request in scheduler.running
1627
+ if running_request.lora_request
1628
+ ]))
1629
+ waiting_lora_adapters = dict(
1630
+ collectionsCounter([
1631
+ waiting_request.lora_request.lora_name
1632
+ for scheduler in self.scheduler
1633
+ for waiting_request in scheduler.waiting
1634
+ if waiting_request.lora_request
1635
+ ]))
1636
+ max_lora_stat = "0"
1637
+ if self.lora_config:
1638
+ max_lora_stat = str(self.lora_config.max_loras)
1639
+
1640
# NOTE: This loop assumes prefill seq_groups are before
1641
# decode seq_groups in scheduled_seq_groups.
1642
if scheduler_outputs is not None:
@@ -1738,7 +1758,9 @@ class LLMEngine:
1758
num_generation_tokens_requests=num_generation_tokens_requests,
1759
n_requests=n_requests,
1760
finished_reason_requests=finished_reason_requests,
1741
- )
1761
+ max_lora=str(max_lora_stat),
1762
+ waiting_lora_adapters=list(waiting_lora_adapters.keys()),
1763
+ running_lora_adapters=list(running_lora_adapters.keys()))
1764
1765
def add_lora(self, lora_request: LoRARequest) -> bool:
1766
return self.model_executor.add_lora(lora_request)
vllm/engine/metrics.py
+28
-1
@@ -34,7 +34,11 @@ class Metrics:
34
See https://prometheus.github.io/client_python/multiprocess/ for more
35
details on limitations.
36
"""
37
+
38
labelname_finish_reason = "finished_reason"
39
+ labelname_waiting_lora_adapters = "waiting_lora_adapters"
40
+ labelname_running_lora_adapters = "running_lora_adapters"
41
+ labelname_max_lora = "max_lora"
42
_gauge_cls = prometheus_client.Gauge
43
_counter_cls = prometheus_client.Counter
44
_histogram_cls = prometheus_client.Histogram
@@ -55,6 +59,16 @@ class Metrics:
59
documentation="Number of requests waiting to be processed.",
60
labelnames=labelnames,
61
multiprocess_mode="sum")
62
+ self.gauge_lora_info = self._gauge_cls(
63
+ name="vllm:lora_requests_info",
64
+ documentation="Running stats on lora requests.",
65
+ labelnames=[
66
+ self.labelname_running_lora_adapters,
67
+ self.labelname_max_lora,
68
+ self.labelname_waiting_lora_adapters,
69
+ ],
70
+ multiprocess_mode="livemostrecent",
71
+ )
72
self.gauge_scheduler_swapped = self._gauge_cls(
73
name="vllm:num_requests_swapped",
74
documentation="Number of requests swapped to CPU.",
@@ -426,6 +440,9 @@ class PrometheusStatLogger(StatLoggerBase):
440
for datum in data:
441
histogram.labels(**self.labels).observe(datum)
442
443
+ def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
444
+ gauge.labels(**data).set(1)
445
+
446
def _log_prometheus(self, stats: Stats) -> None:
447
# System state data
448
self._log_gauge(self.metrics.gauge_scheduler_running,
@@ -442,7 +459,17 @@ class PrometheusStatLogger(StatLoggerBase):
459
stats.cpu_prefix_cache_hit_rate)
460
self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
461
stats.gpu_prefix_cache_hit_rate)
445
-
462
+ # Including max-lora in metric, in future this property of lora
463
+ # config maybe extended to be dynamic.
464
+ lora_info = {
465
+ self.metrics.labelname_running_lora_adapters:
466
+ ",".join(stats.running_lora_adapters),
467
+ self.metrics.labelname_waiting_lora_adapters:
468
+ ",".join(stats.waiting_lora_adapters),
469
+ self.metrics.labelname_max_lora:
470
+ stats.max_lora,
471
+ }
472
+ self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
473
# Iteration level data
474
self._log_counter(self.metrics.counter_num_preemption,
475
stats.num_preemption_iter)
vllm/engine/metrics_types.py
+3
@@ -51,6 +51,9 @@ class Stats:
51
num_generation_tokens_requests: List[int]
52
n_requests: List[int]
53
finished_reason_requests: List[str]
54
+ waiting_lora_adapters: List[str]
55
+ running_lora_adapters: List[str]
56
+ max_lora: str
57
58
spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
59