A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 05:45 UTC
[Metrics] Log number of preempted requests (#28522)
Add tracking and periodic logging for the number of preempted requests in the metrics logger. This helps monitor system behavior under load. Signed-off-by: Yining Liu <610lyn@gmail.com>
lyn610 committed
Nov 14, 2025 at 17:47 UTC
ecf8230d4d196566a76c907949d6569b1ff176ad
1 file changed
+19
-4
vllm/v1/metrics/loggers.py
+19
-4
@@ -118,12 +118,14 @@ class LoggingStatLogger(StatLoggerBase):
118
self.num_prompt_tokens: int = 0
119
self.num_generation_tokens: int = 0
120
self.num_corrupted_reqs: int = 0
121
+ self.num_preemptions: int = 0
122
123
def _track_iteration_stats(self, iteration_stats: IterationStats):
124
# Save tracked stats for token counters.
125
self.num_prompt_tokens += iteration_stats.num_prompt_tokens
126
self.num_generation_tokens += iteration_stats.num_generation_tokens
127
self.num_corrupted_reqs += iteration_stats.num_corrupted_reqs
128
+ self.num_preemptions += iteration_stats.num_preempted_reqs
129
130
def _get_throughput(self, tracked_stats: int, now: float) -> float:
131
# Compute summary metrics for tracked stats
@@ -196,18 +198,31 @@ class LoggingStatLogger(StatLoggerBase):
198
"Avg generation throughput: %.1f tokens/s",
199
"Running: %d reqs",
200
"Waiting: %d reqs",
199
- "GPU KV cache usage: %.1f%%",
200
- "Prefix cache hit rate: %.1f%%",
201
]
202
log_args = [
203
self.last_prompt_throughput,
204
self.last_generation_throughput,
205
self.last_scheduler_stats.num_running_reqs,
206
self.last_scheduler_stats.num_waiting_reqs,
207
- self.last_scheduler_stats.kv_cache_usage * 100,
208
- self.prefix_caching_metrics.hit_rate * 100,
207
]
208
209
+ if self.num_preemptions > 0:
210
+ log_parts.append("Preemptions: %d")
211
+ log_args.append(self.num_preemptions)
212
+
213
+ log_parts.extend(
214
+ [
215
+ "GPU KV cache usage: %.1f%%",
216
+ "Prefix cache hit rate: %.1f%%",
217
+ ]
218
+ )
219
+ log_args.extend(
220
+ [
221
+ self.last_scheduler_stats.kv_cache_usage * 100,
222
+ self.prefix_caching_metrics.hit_rate * 100,
223
+ ]
224
+ )
225
+
226
if envs.VLLM_COMPUTE_NANS_IN_LOGITS:
227
log_parts.append("Corrupted: %d reqs")
228
log_args.append(self.num_corrupted_reqs)