A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 03:30 UTC
[Metrics] Add request_id to FinishedRequestStats to enable correlation between metrics and requests (#39710)
Enables external `StatLogger` plugins to correlate per-request metrics with request-level context. Also, this is a pre-requisite for Prometheus exemplars in #30972. Signed-off-by: Csrayz <33659823+Csrayz@users.noreply.github.com>
Csrayz committed
Apr 15, 2026 at 19:24 UTC
68be0f853ed0cb131468e1f9062b05d8d7a4ab34
3 files changed
+12
tests/v1/metrics/test_stats.py
+8
@@ -26,6 +26,7 @@ def test_prefill_kv_computed_with_cache():
26
# Case 1: With prefix cache (1200 tokens cached)
27
iteration_stats.update_from_finished_request(
28
finish_reason=FinishReason.STOP,
29
+ request_id="test-req-001",
30
num_prompt_tokens=10000,
31
max_tokens_param=100,
32
req_stats=req_stats,
@@ -35,6 +36,7 @@ def test_prefill_kv_computed_with_cache():
36
finished_req = iteration_stats.finished_requests[0]
37
assert finished_req.num_prompt_tokens == 10000
38
assert finished_req.num_cached_tokens == 1200
39
+ assert finished_req.request_id == "test-req-001"
40
41
# Verify calculation: prefill KV = prompt tokens - cached tokens
42
prefill_kv_computed = finished_req.num_prompt_tokens - max(
@@ -55,6 +57,7 @@ def test_prefill_kv_computed_no_cache():
57
# Case 2: No prefix cache
58
iteration_stats.update_from_finished_request(
59
finish_reason=FinishReason.STOP,
60
+ request_id="test-req-002",
61
num_prompt_tokens=2000,
62
max_tokens_param=100,
63
req_stats=req_stats,
@@ -64,6 +67,7 @@ def test_prefill_kv_computed_no_cache():
67
finished_req = iteration_stats.finished_requests[0]
68
assert finished_req.num_prompt_tokens == 2000
69
assert finished_req.num_cached_tokens == 0
70
+ assert finished_req.request_id == "test-req-002"
71
72
# Verify calculation: prefill KV = full prompt when no cache
73
prefill_kv_computed = finished_req.num_prompt_tokens - max(
@@ -84,6 +88,7 @@ def test_prefill_kv_computed_edge_cases():
88
# Case 3: Negative num_cached_tokens (shouldn't happen, but handle gracefully)
89
iteration_stats.update_from_finished_request(
90
finish_reason=FinishReason.STOP,
91
+ request_id="test-req-003",
92
num_prompt_tokens=100,
93
max_tokens_param=10,
94
req_stats=req_stats,
@@ -96,11 +101,13 @@ def test_prefill_kv_computed_edge_cases():
101
finished_req.num_cached_tokens, 0
102
)
103
assert prefill_kv_computed == 100 # Should treat negative as 0
104
+ assert finished_req.request_id == "test-req-003"
105
106
# Case 4: All tokens cached (shouldn't happen in practice)
107
iteration_stats2 = IterationStats()
108
iteration_stats2.update_from_finished_request(
109
finish_reason=FinishReason.STOP,
110
+ request_id="test-req-004",
111
num_prompt_tokens=100,
112
max_tokens_param=10,
113
req_stats=req_stats,
@@ -112,6 +119,7 @@ def test_prefill_kv_computed_edge_cases():
119
finished_req2.num_cached_tokens, 0
120
)
121
assert prefill_kv_computed2 == 0 # All cached, nothing computed
122
+ assert finished_req2.request_id == "test-req-004"
123
124
125
def test_prompt_token_stats_all_computed():
vllm/v1/engine/output_processor.py
+1
@@ -799,6 +799,7 @@ class OutputProcessor:
799
assert req_state.stats is not None
800
iteration_stats.update_from_finished_request(
801
finish_reason=finish_reason,
802
+ request_id=req_state.external_req_id,
803
num_prompt_tokens=req_state.prompt_len,
804
max_tokens_param=req_state.max_tokens_param,
805
req_stats=req_state.stats,
vllm/v1/metrics/stats.py
+3
@@ -225,6 +225,7 @@ class FinishedRequestStats:
225
"""Stats associated with a finished request."""
226
227
finish_reason: "FinishReason"
228
+ request_id: str | None = None
229
e2e_latency: float = 0.0
230
num_prompt_tokens: int = 0
231
num_generation_tokens: int = 0
@@ -427,6 +428,7 @@ class IterationStats:
428
def update_from_finished_request(
429
self,
430
finish_reason: "FinishReason",
431
+ request_id: str,
432
num_prompt_tokens: int,
433
max_tokens_param: int | None,
434
req_stats: RequestStateStats,
@@ -458,6 +460,7 @@ class IterationStats:
460
461
finished_req = FinishedRequestStats(
462
finish_reason=finish_reason,
463
+ request_id=request_id,
464
e2e_latency=e2e_latency,
465
num_prompt_tokens=num_prompt_tokens,
466
num_generation_tokens=req_stats.num_generation_tokens,