[Rust Frontend][Metrics] Add per-request preemption histogram (#57033)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: reidliu41 <reid201711@gmail.com>
Reid committed
Sep 16, 2026 at 10:51 UTC
0673de980055a4283fcc152fc1ecc837ff534f1d
3 files changed
+33
rust/src/llm/src/request_metrics.rs
+6
@@ -45,6 +45,7 @@ pub(crate) struct RequestMetricsTracker {
45
last_token_ts: f64,
46
first_token_latency: f64,
47
num_generation_tokens: u32,
48
+ num_preemptions: u64,
49
latest_num_cached_tokens: u32,
50
}
51
@@ -69,6 +70,7 @@ struct RequestMetricHandles {
70
request_max_num_generation_tokens: HistogramMetric,
71
request_params_max_tokens: HistogramMetric,
72
request_params_n: HistogramMetric,
73
+ request_num_preemptions: HistogramMetric,
74
request_prefill_kv_computed_tokens: HistogramMetric,
75
time_to_first_token_seconds: HistogramMetric,
76
inter_token_latency_seconds: HistogramMetric,
@@ -104,6 +106,7 @@ impl RequestMetricsTracker {
106
last_token_ts: 0.0,
107
first_token_latency: 0.0,
108
num_generation_tokens: 0,
109
+ num_preemptions: 0,
110
latest_num_cached_tokens: 0,
111
}
112
}
@@ -184,6 +187,7 @@ impl RequestMetricsTracker {
187
self.handles.request_params_max_tokens.observe(max_tokens_param as f64);
188
}
189
self.handles.request_params_n.observe(self.n_param as f64);
190
+ self.handles.request_num_preemptions.observe(self.num_preemptions as f64);
191
self.handles
192
.request_prefill_kv_computed_tokens
193
.observe(prefill_kv_computed_tokens as f64);
@@ -223,6 +227,7 @@ impl RequestMetricsTracker {
227
}
228
}
229
EngineCoreEventType::Preempted => {
230
+ self.num_preemptions += 1;
231
self.handles.num_preemptions.inc();
232
}
233
}
@@ -276,6 +281,7 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
281
.get_or_create_owned(&labels),
282
request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels),
283
request_params_n: metrics.request_params_n.get_or_create_owned(&labels),
284
+ request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels),
285
request_prefill_kv_computed_tokens: metrics
286
.request_prefill_kv_computed_tokens
287
.get_or_create_owned(&labels),
rust/src/llm/tests/generate.rs
+12
@@ -790,6 +790,12 @@ async fn generate_records_request_metrics_in_prometheus_output() {
790
assert!(rendered.contains(&format!(
791
"vllm:num_preemptions_total{{model_name=\"{model_name}\",engine=\"4\"}} 1"
792
)));
793
+ assert!(rendered.contains(&format!(
794
+ "vllm:request_num_preemptions_sum{{model_name=\"{model_name}\",engine=\"4\"}} 1.0"
795
+ )));
796
+ assert!(rendered.contains(&format!(
797
+ "vllm:request_num_preemptions_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
798
+ )));
799
assert!(rendered.contains(&format!(
800
"vllm:time_to_first_token_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
801
)));
@@ -883,6 +889,12 @@ async fn dropping_stream_records_abort_terminal_request_metrics() {
889
assert!(rendered.contains(&format!(
890
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
891
)));
892
+ assert!(rendered.contains(&format!(
893
+ "vllm:request_num_preemptions_sum{{model_name=\"{model_name}\",engine=\"5\"}} 0.0"
894
+ )));
895
+ assert!(rendered.contains(&format!(
896
+ "vllm:request_num_preemptions_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
897
+ )));
898
899
llm.shutdown().await.unwrap();
900
}
rust/src/metrics/src/request.rs
+15
@@ -21,6 +21,7 @@ const REQUEST_LATENCY_BUCKETS: [f64; 21] = [
21
480.0, 960.0, 1920.0, 7680.0,
22
];
23
const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0];
24
+const REQUEST_NUM_PREEMPTIONS_BUCKETS: [f64; 7] = [1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0];
25
const ITERATION_TOKENS_BUCKETS: [f64; 13] = [
26
1.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0, 16384.0,
27
];
@@ -69,6 +70,10 @@ fn request_params_n_histogram() -> Histogram {
70
Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied())
71
}
72
73
+fn request_num_preemptions_histogram() -> Histogram {
74
+ Histogram::new(REQUEST_NUM_PREEMPTIONS_BUCKETS.iter().copied())
75
+}
76
+
77
fn iteration_tokens_histogram() -> Histogram {
78
Histogram::new(ITERATION_TOKENS_BUCKETS.iter().copied())
79
}
@@ -110,6 +115,7 @@ pub struct RequestMetrics {
115
pub request_max_num_generation_tokens: HistogramFamily,
116
pub request_params_max_tokens: HistogramFamily,
117
pub request_params_n: HistogramFamily,
118
+ pub request_num_preemptions: HistogramFamily,
119
pub request_prefill_kv_computed_tokens: HistogramFamily,
120
pub time_to_first_token_seconds: HistogramFamily,
121
pub inter_token_latency_seconds: HistogramFamily,
@@ -216,6 +222,14 @@ impl RequestMetrics {
222
request_params_n.clone(),
223
);
224
225
+ let request_num_preemptions =
226
+ Family::new_with_constructor(request_num_preemptions_histogram as fn() -> Histogram);
227
+ registry.register(
228
+ "vllm:request_num_preemptions",
229
+ "Histogram of the number of times a request was preempted.",
230
+ request_num_preemptions.clone(),
231
+ );
232
+
233
let request_prefill_kv_computed_tokens =
234
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
235
registry.register(
@@ -302,6 +316,7 @@ impl RequestMetrics {
316
request_max_num_generation_tokens,
317
request_params_max_tokens,
318
request_params_n,
319
+ request_num_preemptions,
320
request_prefill_kv_computed_tokens,
321
time_to_first_token_seconds,
322
inter_token_latency_seconds,