@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9bb10a7d27
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[MISC] Add lora requests to metrics (#9477)

Co-authored-by: Kunjan Patel <kunjanp_google_com@vllm.us-central1-a.c.kunjanp-gke-dev-2.internal>

Kunjan committed Oct 18, 2024 at 13:50 UTC 9bb10a7d276e085c72f2545cea1a3565937e7b22
3 files changed +54 -2
vllm/engine/llm_engine.py
+23 -1
@@ -1,4 +1,5 @@
1 import time
2 +from collections import Counter as collectionsCounter
3 from collections import deque
4 from contextlib import contextmanager
5 from dataclasses import dataclass
@@ -1617,6 +1618,25 @@ class LLMEngine:
1618 n_requests: List[int] = []
1619 finished_reason_requests: List[str] = []
1620
1621 + # Lora requests
1622 + running_lora_adapters = dict(
1623 + collectionsCounter([
1624 + running_request.lora_request.lora_name
1625 + for scheduler in self.scheduler
1626 + for running_request in scheduler.running
1627 + if running_request.lora_request
1628 + ]))
1629 + waiting_lora_adapters = dict(
1630 + collectionsCounter([
1631 + waiting_request.lora_request.lora_name
1632 + for scheduler in self.scheduler
1633 + for waiting_request in scheduler.waiting
1634 + if waiting_request.lora_request
1635 + ]))
1636 + max_lora_stat = "0"
1637 + if self.lora_config:
1638 + max_lora_stat = str(self.lora_config.max_loras)
1639 +
1640 # NOTE: This loop assumes prefill seq_groups are before
1641 # decode seq_groups in scheduled_seq_groups.
1642 if scheduler_outputs is not None:
@@ -1738,7 +1758,9 @@ class LLMEngine:
1758 num_generation_tokens_requests=num_generation_tokens_requests,
1759 n_requests=n_requests,
1760 finished_reason_requests=finished_reason_requests,
1741 - )
1761 + max_lora=str(max_lora_stat),
1762 + waiting_lora_adapters=list(waiting_lora_adapters.keys()),
1763 + running_lora_adapters=list(running_lora_adapters.keys()))
1764
1765 def add_lora(self, lora_request: LoRARequest) -> bool:
1766 return self.model_executor.add_lora(lora_request)
vllm/engine/metrics.py
+28 -1
@@ -34,7 +34,11 @@ class Metrics:
34 See https://prometheus.github.io/client_python/multiprocess/ for more
35 details on limitations.
36 """
37 +
38 labelname_finish_reason = "finished_reason"
39 + labelname_waiting_lora_adapters = "waiting_lora_adapters"
40 + labelname_running_lora_adapters = "running_lora_adapters"
41 + labelname_max_lora = "max_lora"
42 _gauge_cls = prometheus_client.Gauge
43 _counter_cls = prometheus_client.Counter
44 _histogram_cls = prometheus_client.Histogram
@@ -55,6 +59,16 @@ class Metrics:
59 documentation="Number of requests waiting to be processed.",
60 labelnames=labelnames,
61 multiprocess_mode="sum")
62 + self.gauge_lora_info = self._gauge_cls(
63 + name="vllm:lora_requests_info",
64 + documentation="Running stats on lora requests.",
65 + labelnames=[
66 + self.labelname_running_lora_adapters,
67 + self.labelname_max_lora,
68 + self.labelname_waiting_lora_adapters,
69 + ],
70 + multiprocess_mode="livemostrecent",
71 + )
72 self.gauge_scheduler_swapped = self._gauge_cls(
73 name="vllm:num_requests_swapped",
74 documentation="Number of requests swapped to CPU.",
@@ -426,6 +440,9 @@ class PrometheusStatLogger(StatLoggerBase):
440 for datum in data:
441 histogram.labels(**self.labels).observe(datum)
442
443 + def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
444 + gauge.labels(**data).set(1)
445 +
446 def _log_prometheus(self, stats: Stats) -> None:
447 # System state data
448 self._log_gauge(self.metrics.gauge_scheduler_running,
@@ -442,7 +459,17 @@ class PrometheusStatLogger(StatLoggerBase):
459 stats.cpu_prefix_cache_hit_rate)
460 self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
461 stats.gpu_prefix_cache_hit_rate)
445 -
462 + # Including max-lora in metric, in future this property of lora
463 + # config maybe extended to be dynamic.
464 + lora_info = {
465 + self.metrics.labelname_running_lora_adapters:
466 + ",".join(stats.running_lora_adapters),
467 + self.metrics.labelname_waiting_lora_adapters:
468 + ",".join(stats.waiting_lora_adapters),
469 + self.metrics.labelname_max_lora:
470 + stats.max_lora,
471 + }
472 + self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
473 # Iteration level data
474 self._log_counter(self.metrics.counter_num_preemption,
475 stats.num_preemption_iter)
vllm/engine/metrics_types.py
+3
@@ -51,6 +51,9 @@ class Stats:
51 num_generation_tokens_requests: List[int]
52 n_requests: List[int]
53 finished_reason_requests: List[str]
54 + waiting_lora_adapters: List[str]
55 + running_lora_adapters: List[str]
56 + max_lora: str
57
58 spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
59