@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc640ee8bc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Rust Frontend][Metrics] Export `vllm:lora_requests_info` from frontend (#45030)

Signed-off-by: Will Eaton <weaton@redhat.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Will Eaton committed Jun 11, 2026 at 09:45 UTC cc640ee8bc1e61d333ecec039b2e3f143f9d4066
8 files changed +383 -36
rust/Cargo.lock
+1
@@ -5834,6 +5834,7 @@ dependencies = [
5834 name = "vllm-metrics"
5835 version = "0.1.0"
5836 dependencies = [
5837 + "itertools 0.14.0",
5838 "prometheus-client",
5839 ]
5840
rust/src/engine-core-client/src/client.rs
+2 -1
@@ -442,9 +442,10 @@ impl EngineCoreClient {
442 );
443
444 let request_id = req.request_id.clone();
445 + let lora_name = req.lora_request.as_ref().map(|lora| lora.lora_name.clone());
446 let data_parallel_rank = req.data_parallel_rank;
447 let (engine_id, rx) =
447 - self.inner.register_request(request_id.clone(), data_parallel_rank)?;
448 + self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?;
449
450 let result: Result<()> = async {
451 if let Some(coordinator) = self.coordinator.as_ref() {
rust/src/engine-core-client/src/client/imp.rs
+20 -4
@@ -1,4 +1,4 @@
1 -use std::collections::BTreeMap;
1 +use std::collections::{BTreeMap, BTreeSet};
2 use std::sync::Arc;
3
4 use arc_swap::ArcSwapOption;
@@ -13,7 +13,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
13 use crate::client::stream::EngineCoreStreamOutput;
14 use crate::client::{AbortCause, AbortRequest};
15 use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
16 -use crate::metrics::record_scheduler_stats;
16 +use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
17 use crate::protocol::stats::SchedulerStats;
18 use crate::protocol::utility::UtilityOutput;
19 use crate::protocol::{
@@ -58,17 +58,19 @@ impl ClientInner {
58 /// per-request output channel bound to its `request_id`.
59 ///
60 /// When `data_parallel_rank` is provided, the request is routed to that
61 - /// specific engine rank, bypassing load balancing.
61 + /// specific engine rank, bypassing load balancing. `lora_name` is the
62 + /// request's LoRA adapter, tracked for `vllm:lora_requests_info`.
63 pub fn register_request(
64 &self,
65 request_id: String,
66 + lora_name: Option<String>,
67 data_parallel_rank: Option<u32>,
68 ) -> Result<(EngineId, OutputReceiver)> {
69 let mut registry = self.request_reg.lock();
70 if registry.is_closed() {
71 return Err(self.closed_error());
72 }
71 - registry.register(request_id, data_parallel_rank)
73 + registry.register(request_id, lora_name, data_parallel_rank)
74 }
75
76 /// Allocate the next utility `call_id` and register its waiting receiver.
@@ -131,6 +133,12 @@ impl ClientInner {
133 self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
134 }
135
136 + /// Snapshot the adapter names of tracked LoRA requests as
137 + /// (running, waiting) sets.
138 + pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
139 + self.request_reg.lock().lora_adapter_states()
140 + }
141 +
142 /// Close all active request streams and utility calls with the first
143 /// persistent health error.
144 pub fn close_registries(&self, error: Arc<Error>) {
@@ -303,6 +311,8 @@ pub(crate) async fn run_output_dispatcher_loop(
311 inner: Arc<ClientInner>,
312 mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
313 ) {
314 + let mut lora_info = LoraInfoExporter::default();
315 +
316 let result: Result<()> = async {
317 loop {
318 let outputs = match output_rx.recv().await {
@@ -357,6 +367,12 @@ pub(crate) async fn run_output_dispatcher_loop(
367 scheduler_stats,
368 );
369 }
370 +
371 + // The engine's scheduler stats never carry adapter names;
372 + // the gauge is derived from the registry's frontend-side
373 + // request tracking instead.
374 + let (running, waiting) = inner.lora_adapter_states();
375 + lora_info.update(&METRICS.scheduler, running, waiting);
376 }
377 ClassifiedEngineCoreOutputs::Utility(utility) => {
378 let call_id = utility.output.call_id;
rust/src/engine-core-client/src/client/state.rs
+210 -25
@@ -1,4 +1,4 @@
1 -use std::collections::{BTreeMap, HashMap};
1 +use std::collections::{BTreeMap, BTreeSet, HashMap};
2 use std::sync::atomic::{AtomicU64, Ordering};
3
4 use tokio::sync::{mpsc, oneshot};
@@ -7,9 +7,9 @@ use tracing::trace;
7 use crate::EngineId;
8 use crate::client::stream::EngineCoreStreamOutput;
9 use crate::error::{Error, Result};
10 -use crate::protocol::EngineCoreOutput;
10 use crate::protocol::stats::SchedulerStats;
11 use crate::protocol::utility::UtilityOutput;
12 +use crate::protocol::{EngineCoreEventType, EngineCoreOutput};
13 use crate::transport::ConnectedEngine;
14
15 pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -21,6 +21,25 @@ pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
21 struct TrackedRequest {
22 sender: OutputSender,
23 engine_id: EngineId,
24 + lora: Option<LoraRequestState>,
25 +}
26 +
27 +/// Frontend-side view of one LoRA request's scheduling phase.
28 +///
29 +/// The engine's `SchedulerStats` does not carry adapter names, so
30 +/// `vllm:lora_requests_info` must be derived from per-request lifecycle events
31 +/// observed by this client, mirroring `LoRARequestStates` in the Python
32 +/// frontend (`vllm/v1/engine/output_processor.py`).
33 +#[derive(Debug)]
34 +struct LoraRequestState {
35 + adapter_name: String,
36 + phase: LoraPhase,
37 +}
38 +
39 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40 +enum LoraPhase {
41 + Waiting,
42 + Running,
43 }
44
45 /// The latest real scheduler-side load snapshot observed from one engine.
@@ -105,6 +124,7 @@ impl RequestRegistry {
124 pub fn register(
125 &mut self,
126 request_id: String,
127 + lora_name: Option<String>,
128 data_parallel_rank: Option<u32>,
129 ) -> Result<(EngineId, OutputReceiver)> {
130 if self.requests.contains_key(&request_id) {
@@ -118,6 +138,10 @@ impl RequestRegistry {
138 TrackedRequest {
139 sender: tx,
140 engine_id: engine_id.clone(),
141 + lora: lora_name.map(|adapter_name| LoraRequestState {
142 + adapter_name,
143 + phase: LoraPhase::Waiting,
144 + }),
145 },
146 );
147
@@ -171,6 +195,7 @@ impl RequestRegistry {
195 /// Obtain the stream sender for one output. If it indicates the request is
196 /// finished, it will be removed from the registry.
197 pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
198 + self.apply_lora_events(output);
199 if output.finished() {
200 self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
201 } else {
@@ -180,6 +205,43 @@ impl RequestRegistry {
205 }
206 }
207
208 + /// Advance the request's LoRA scheduling phase from the engine-core events
209 + /// attached to one output, mirroring the Python frontend's
210 + /// `LoRARequestStates.update_from_events`.
211 + fn apply_lora_events(&mut self, output: &EngineCoreOutput) {
212 + let Some(events) = output.events.as_ref() else {
213 + return;
214 + };
215 + let Some(lora) = self
216 + .requests
217 + .get_mut(output.request_id.as_str())
218 + .and_then(|tracked| tracked.lora.as_mut())
219 + else {
220 + return;
221 + };
222 + for event in events {
223 + lora.phase = match event.r#type {
224 + EngineCoreEventType::Queued | EngineCoreEventType::Preempted => LoraPhase::Waiting,
225 + EngineCoreEventType::Scheduled => LoraPhase::Running,
226 + };
227 + }
228 + }
229 +
230 + /// Snapshot the adapter names of tracked LoRA requests as
231 + /// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
232 + pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
233 + let mut running = BTreeSet::new();
234 + let mut waiting = BTreeSet::new();
235 + for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
236 + let set = match lora.phase {
237 + LoraPhase::Running => &mut running,
238 + LoraPhase::Waiting => &mut waiting,
239 + };
240 + set.insert(lora.adapter_name.clone());
241 + }
242 + (running, waiting)
243 + }
244 +
245 /// Obtain stream senders for a whole engine output batch under one
246 /// registry lock. Finished outputs are removed before returning.
247 pub fn senders_for_outputs<'a>(
@@ -336,11 +398,16 @@ impl UtilityRegistry {
398
399 #[cfg(test)]
400 mod tests {
339 - use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
401 + use std::collections::BTreeSet;
402 +
403 use crate::EngineId;
341 - use crate::client::state::EngineLoadSnapshot;
404 + use crate::client::state::{
405 + EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
406 + };
407 use crate::mock_engine::default_ready_response;
343 - use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
408 + use crate::protocol::{
409 + EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
410 + };
411 use crate::transport::ConnectedEngine;
412
413 fn connected_engine(engine_id: EngineId) -> ConnectedEngine {
@@ -350,11 +417,36 @@ mod tests {
417 }
418 }
419
420 + fn output_with_events(
421 + request_id: &str,
422 + events: &[EngineCoreEventType],
423 + finish_reason: Option<EngineCoreFinishReason>,
424 + ) -> EngineCoreOutput {
425 + EngineCoreOutput {
426 + request_id: request_id.to_string(),
427 + events: Some(
428 + events
429 + .iter()
430 + .map(|event_type| EngineCoreEvent {
431 + r#type: *event_type,
432 + timestamp: 0.0,
433 + })
434 + .collect(),
435 + ),
436 + finish_reason,
437 + ..Default::default()
438 + }
439 + }
440 +
441 + fn adapter_names(values: &[&str]) -> BTreeSet<String> {
442 + values.iter().map(|name| (*name).to_string()).collect()
443 + }
444 +
445 #[test]
446 fn registry_rejects_duplicate_request_ids() {
447 let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
356 - registry.register("req-1".to_string(), None).unwrap();
357 - let error = registry.register("req-1".to_string(), None).unwrap_err();
448 + registry.register("req-1".to_string(), None, None).unwrap();
449 + let error = registry.register("req-1".to_string(), None, None).unwrap_err();
450 assert!(matches!(
451 error,
452 crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
@@ -364,7 +456,7 @@ mod tests {
456 #[test]
457 fn registry_removes_finished_request_on_output() {
458 let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
367 - registry.register("req-1".to_string(), None).unwrap();
459 + registry.register("req-1".to_string(), None, None).unwrap();
460
461 let sender = registry.sender_for_output(&EngineCoreOutput {
462 request_id: "req-1".to_string(),
@@ -376,11 +468,104 @@ mod tests {
468 assert!(!registry.contains("req-1"));
469 }
470
471 + #[test]
472 + fn registry_tracks_lora_phases_from_engine_events() {
473 + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
474 + registry
475 + .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
476 + .unwrap();
477 + registry.register("req-plain".to_string(), None, None).unwrap();
478 +
479 + // Registered but not yet scheduled: counted as waiting. The non-LoRA
480 + // request never shows up.
481 + assert_eq!(
482 + registry.lora_adapter_states(),
483 + (adapter_names(&[]), adapter_names(&["adapter-a"]))
484 + );
485 +
486 + // Queued then scheduled in one output: running.
487 + drop(registry.sender_for_output(&output_with_events(
488 + "req-lora",
489 + &[EngineCoreEventType::Queued, EngineCoreEventType::Scheduled],
490 + None,
491 + )));
492 + assert_eq!(
493 + registry.lora_adapter_states(),
494 + (adapter_names(&["adapter-a"]), adapter_names(&[]))
495 + );
496 +
497 + // Preempted: back to waiting.
498 + drop(registry.sender_for_output(&output_with_events(
499 + "req-lora",
500 + &[EngineCoreEventType::Preempted],
501 + None,
502 + )));
503 + assert_eq!(
504 + registry.lora_adapter_states(),
505 + (adapter_names(&[]), adapter_names(&["adapter-a"]))
506 + );
507 +
508 + // Finished: dropped from tracking entirely.
509 + drop(registry.sender_for_output(&output_with_events(
510 + "req-lora",
511 + &[EngineCoreEventType::Scheduled],
512 + Some(EngineCoreFinishReason::Stop),
513 + )));
514 + assert_eq!(
515 + registry.lora_adapter_states(),
516 + (adapter_names(&[]), adapter_names(&[]))
517 + );
518 + }
519 +
520 + #[test]
521 + fn registry_unions_lora_adapters_across_requests() {
522 + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
523 + registry
524 + .register("req-a1".to_string(), Some("adapter-a".to_string()), None)
525 + .unwrap();
526 + registry
527 + .register("req-a2".to_string(), Some("adapter-a".to_string()), None)
528 + .unwrap();
529 + registry
530 + .register("req-b".to_string(), Some("adapter-b".to_string()), None)
531 + .unwrap();
532 +
533 + // One of adapter-a's requests starts running while the other waits:
534 + // the adapter appears in both sets.
535 + drop(registry.sender_for_output(&output_with_events(
536 + "req-a1",
537 + &[EngineCoreEventType::Scheduled],
538 + None,
539 + )));
540 + assert_eq!(
541 + registry.lora_adapter_states(),
542 + (
543 + adapter_names(&["adapter-a"]),
544 + adapter_names(&["adapter-a", "adapter-b"])
545 + )
546 + );
547 + }
548 +
549 + #[test]
550 + fn registry_drops_lora_tracking_on_abort() {
551 + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
552 + registry
553 + .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
554 + .unwrap();
555 +
556 + drop(registry.finish_many(&["req-lora".to_string()]));
557 +
558 + assert_eq!(
559 + registry.lora_adapter_states(),
560 + (adapter_names(&[]), adapter_names(&[]))
561 + );
562 + }
563 +
564 #[test]
565 fn registry_closes_all_requests_on_failure() {
566 let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
382 - registry.register("req-1".to_string(), None).unwrap();
383 - registry.register("req-2".to_string(), None).unwrap();
567 + registry.register("req-1".to_string(), None, None).unwrap();
568 + registry.register("req-2".to_string(), None, None).unwrap();
569
570 let senders = registry.close();
571
@@ -396,9 +581,9 @@ mod tests {
581 connected_engine(engine_0.clone()),
582 connected_engine(engine_1.clone()),
583 ]);
399 - let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
400 - let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
401 - let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
584 + let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
585 + let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
586 + let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
587
588 assert_eq!(chosen_0, engine_0);
589 assert_eq!(chosen_1, engine_1);
@@ -425,9 +610,9 @@ mod tests {
610 connected_engine(engine_1.clone()),
611 ]);
612
428 - let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
429 - let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
430 - let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
613 + let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
614 + let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
615 + let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
616
617 assert_eq!(chosen_0, engine_0);
618 assert_eq!(chosen_1, engine_1);
@@ -494,7 +679,7 @@ mod tests {
679 }
680 ));
681
497 - let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap();
682 + let (chosen, _) = registry.register("req-stats".to_string(), None, None).unwrap();
683 assert_eq!(chosen, engine_1);
684 }
685
@@ -510,15 +695,15 @@ mod tests {
695 ]);
696
697 // Explicitly target rank 2 (third engine).
513 - let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap();
698 + let (chosen, _) = registry.register("req-1".to_string(), None, Some(2)).unwrap();
699 assert_eq!(chosen, engine_2);
700
701 // Explicitly target rank 0 (first engine).
517 - let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap();
702 + let (chosen, _) = registry.register("req-2".to_string(), None, Some(0)).unwrap();
703 assert_eq!(chosen, engine_0);
704
705 // Explicitly target rank 1.
521 - let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap();
706 + let (chosen, _) = registry.register("req-3".to_string(), None, Some(1)).unwrap();
707 assert_eq!(chosen, engine_1);
708 }
709
@@ -532,11 +717,11 @@ mod tests {
717 ]);
718
719 // Load-balance: first two go to engine_0 and engine_1.
535 - registry.register("req-lb-0".to_string(), None).unwrap();
720 + registry.register("req-lb-0".to_string(), None, None).unwrap();
721
722 // Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
723 // But with dp_rank=0, it should still go to engine_0.
539 - let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap();
724 + let (chosen, _) = registry.register("req-dp".to_string(), None, Some(0)).unwrap();
725 assert_eq!(chosen, engine_0);
726 }
727
@@ -547,7 +732,7 @@ mod tests {
732 connected_engine(EngineId::from_engine_index(1)),
733 ]);
734
550 - let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
735 + let error = registry.register("req-1".to_string(), None, Some(2)).unwrap_err();
736 assert!(matches!(
737 error,
738 crate::error::Error::InvalidDataParallelRank {
@@ -562,10 +747,10 @@ mod tests {
747 let engine_0 = EngineId::from_engine_index(0);
748 let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
749
565 - let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
750 + let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
751 assert_eq!(chosen, engine_0);
752
568 - let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err();
753 + let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
754 assert!(matches!(
755 error,
756 crate::error::Error::InvalidDataParallelRank {
rust/src/engine-core-client/src/metrics.rs
+116 -1
@@ -1,4 +1,10 @@
1 -use vllm_metrics::{EngineLabels, EnginePositionLabels, SchedulerMetrics, WaitingReasonLabels};
1 +use std::collections::BTreeSet;
2 +use std::time::{SystemTime, UNIX_EPOCH};
3 +
4 +use vllm_metrics::{
5 + EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics,
6 + WaitingReasonLabels,
7 +};
8
9 use crate::protocol::stats::SchedulerStats;
10
@@ -129,3 +135,112 @@ pub(crate) fn record_scheduler_stats(
135 }
136 }
137 }
138 +
139 +/// Exports `vllm:lora_requests_info` as a single series covering all LoRA
140 +/// requests tracked by this client across every engine in the replica.
141 +///
142 +/// The engine's `SchedulerStats` never carries adapter names: the Python
143 +/// frontend fills them in from per-request lifecycle events tracked by
144 +/// `LoRARequestStates` in `vllm/v1/engine/output_processor.py`. The Rust
145 +/// frontend mirrors that, deriving the sets from the request registry.
146 +#[derive(Default)]
147 +pub(crate) struct LoraInfoExporter {
148 + current: Option<LoraInfoLabels>,
149 +}
150 +
151 +impl LoraInfoExporter {
152 + pub(crate) fn update(
153 + &mut self,
154 + metrics: &SchedulerMetrics,
155 + running: BTreeSet<String>,
156 + waiting: BTreeSet<String>,
157 + ) {
158 + let next = (!running.is_empty() || !waiting.is_empty()).then_some(LoraInfoLabels {
159 + running_lora_adapters: LoraAdapterNames(running),
160 + waiting_lora_adapters: LoraAdapterNames(waiting),
161 + });
162 +
163 + if self.current != next
164 + && let Some(prev) = &self.current
165 + {
166 + metrics.lora_info.remove(prev);
167 + }
168 +
169 + // Python sets this gauge to the current time on every record.
170 + if let Some(labels) = &next {
171 + metrics.lora_info.get_or_create(labels).set(now_unix_secs());
172 + }
173 +
174 + self.current = next;
175 + }
176 +}
177 +
178 +fn now_unix_secs() -> f64 {
179 + SystemTime::now()
180 + .duration_since(UNIX_EPOCH)
181 + .map(|d| d.as_secs_f64())
182 + .unwrap_or(0.0)
183 +}
184 +
185 +#[cfg(test)]
186 +mod tests {
187 + use std::collections::BTreeSet;
188 +
189 + use expect_test::expect;
190 + use vllm_metrics::Metrics;
191 +
192 + use crate::metrics::LoraInfoExporter;
193 +
194 + fn names(values: &[&str]) -> BTreeSet<String> {
195 + values.iter().map(|name| (*name).to_string()).collect()
196 + }
197 +
198 + /// The `lora_requests_info` series with the non-deterministic timestamp
199 + /// value replaced by `<ts>`, one line per series.
200 + fn lora_series(rendered: &str) -> String {
201 + rendered
202 + .lines()
203 + .filter(|l| l.starts_with("vllm:lora_requests_info{"))
204 + .map(|l| match l.rsplit_once("} ") {
205 + Some((labels, _value)) => format!("{labels}}} <ts>"),
206 + None => l.to_string(),
207 + })
208 + .collect::<Vec<_>>()
209 + .join("\n")
210 + }
211 +
212 + #[test]
213 + fn lora_info_emits_clears_stale_and_drains() {
214 + let metrics = Metrics::new();
215 + let mut exporter = LoraInfoExporter::default();
216 +
217 + // No adapters: nothing emitted.
218 + exporter.update(&metrics.scheduler, names(&[]), names(&[]));
219 + expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
220 +
221 + // Two running (sorted), one waiting.
222 + exporter.update(&metrics.scheduler, names(&["b", "a"]), names(&["c"]));
223 + expect![[
224 + r#"vllm:lora_requests_info{running_lora_adapters="a,b",waiting_lora_adapters="c"} <ts>"#
225 + ]]
226 + .assert_eq(&lora_series(&metrics.render().unwrap()));
227 +
228 + // "c" gets scheduled and "d" arrives: the stale series is replaced.
229 + exporter.update(&metrics.scheduler, names(&["a", "b", "c"]), names(&["d"]));
230 + expect![[
231 + r#"vllm:lora_requests_info{running_lora_adapters="a,b,c",waiting_lora_adapters="d"} <ts>"#
232 + ]]
233 + .assert_eq(&lora_series(&metrics.render().unwrap()));
234 +
235 + // Everything but "d" finishes.
236 + exporter.update(&metrics.scheduler, names(&["d"]), names(&[]));
237 + expect![[
238 + r#"vllm:lora_requests_info{running_lora_adapters="d",waiting_lora_adapters=""} <ts>"#
239 + ]]
240 + .assert_eq(&lora_series(&metrics.render().unwrap()));
241 +
242 + // All requests done: series removed entirely.
243 + exporter.update(&metrics.scheduler, names(&[]), names(&[]));
244 + expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
245 + }
246 +}
rust/src/engine-core-client/src/protocol/stats.rs
-4
@@ -181,10 +181,6 @@ pub struct SchedulerStats {
181 pub spec_decoding_stats: Option<SpecDecodingStats>,
182 /// Connector-specific KV transfer stats, kept opaque for now.
183 pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
184 - /// Waiting request counts per LoRA adapter.
185 - pub waiting_lora_adapters: BTreeMap<String, u64>,
186 - /// Running request counts per LoRA adapter.
187 - pub running_lora_adapters: BTreeMap<String, u64>,
184 /// CUDA graph runtime stats when graph metrics are enabled.
185 pub cudagraph_stats: Option<CudagraphStat>,
186 /// Estimated MFU/performance stats, when enabled.
rust/src/metrics/Cargo.toml
+1
@@ -5,6 +5,7 @@ edition.workspace = true
5 license.workspace = true
6
7 [dependencies]
8 +itertools.workspace = true
9 prometheus-client.workspace = true
10
11 [lints]
rust/src/metrics/src/scheduler.rs
+33 -1
@@ -1,4 +1,7 @@
1 -use prometheus_client::encoding::EncodeLabelSet;
1 +use std::collections::BTreeSet;
2 +
3 +use itertools::Itertools as _;
4 +use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder};
5 use prometheus_client::metrics::family::Family;
6 use prometheus_client::metrics::histogram::Histogram;
7 use prometheus_client::registry::Registry;
@@ -42,6 +45,23 @@ pub struct WaitingReasonLabels {
45 pub reason: &'static str,
46 }
47
48 +/// Adapter names encoded as a deterministic comma-joined Prometheus label value.
49 +#[derive(Clone, Debug, Hash, PartialEq, Eq)]
50 +pub struct LoraAdapterNames(pub BTreeSet<String>);
51 +
52 +impl EncodeLabelValue for LoraAdapterNames {
53 + fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> {
54 + EncodeLabelValue::encode(&self.0.iter().join(","), encoder)
55 + }
56 +}
57 +
58 +/// Labels for `vllm:lora_requests_info`.
59 +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
60 +pub struct LoraInfoLabels {
61 + pub running_lora_adapters: LoraAdapterNames,
62 + pub waiting_lora_adapters: LoraAdapterNames,
63 +}
64 +
65 /// Scheduler/batch-scoped Prometheus families exported from `SchedulerStats`.
66 pub struct SchedulerMetrics {
67 // Scheduler state gauges.
@@ -50,6 +70,10 @@ pub struct SchedulerMetrics {
70 pub scheduler_waiting_by_reason: Family<WaitingReasonLabels, U64Gauge>,
71 pub kv_cache_usage: Family<EngineLabels, F64Gauge>,
72
73 + /// `vllm:lora_requests_info`. Value is the emit-time unix timestamp in
74 + /// seconds.
75 + pub lora_info: Family<LoraInfoLabels, F64Gauge>,
76 +
77 // Prefix-cache counters, including the connector-backed external cache path.
78 pub prefix_cache_queries: Family<EngineLabels, U64Counter>,
79 pub prefix_cache_hits: Family<EngineLabels, U64Counter>,
@@ -109,6 +133,13 @@ impl SchedulerMetrics {
133 kv_cache_usage.clone(),
134 );
135
136 + let lora_info = Family::default();
137 + registry.register(
138 + "vllm:lora_requests_info",
139 + "Running stats on lora requests.",
140 + lora_info.clone(),
141 + );
142 +
143 // Prefix-cache counters, including the connector-backed external cache path.
144 let prefix_cache_queries = Family::default();
145 registry.register(
@@ -219,6 +250,7 @@ impl SchedulerMetrics {
250 scheduler_waiting,
251 scheduler_waiting_by_reason,
252 kv_cache_usage,
253 + lora_info,
254 prefix_cache_queries,
255 prefix_cache_hits,
256 external_prefix_cache_queries,