A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[V1][Metrics] Handle preemptions (#13169)
Mark McLoughlin committed
Feb 27, 2025 at 04:04 UTC
cd711c48b29a37f2bc4929bfe8291ab3107af505
5 files changed
+48
-19
tests/entrypoints/openai/test_metrics.py
+1
@@ -227,6 +227,7 @@ EXPECTED_METRICS_V1 = [
227
"vllm:gpu_cache_usage_perc",
228
"vllm:gpu_prefix_cache_queries",
229
"vllm:gpu_prefix_cache_hits",
230
+ "vllm:num_preemptions_total",
231
"vllm:prompt_tokens_total",
232
"vllm:generation_tokens_total",
233
"vllm:iteration_tokens_total",
vllm/v1/core/scheduler.py
+9
-1
@@ -164,6 +164,7 @@ class Scheduler:
164
self.kv_cache_manager.free(preempted_req)
165
preempted_req.status = RequestStatus.PREEMPTED
166
preempted_req.num_computed_tokens = 0
167
+ self.request_preempted(preempted_req, scheduled_timestamp)
168
169
self.waiting.appendleft(preempted_req)
170
preempted_reqs.append(preempted_req)
@@ -281,9 +282,9 @@ class Scheduler:
282
self.waiting.popleft()
283
self.running.append(request)
284
self.scheduled_req_ids.add(request.request_id)
285
+ self.request_scheduled(request, scheduled_timestamp)
286
if request.status == RequestStatus.WAITING:
287
scheduled_new_reqs.append(request)
286
- self.request_scheduled(request, scheduled_timestamp)
288
elif request.status == RequestStatus.PREEMPTED:
289
scheduled_resumed_reqs.append(request)
290
else:
@@ -675,6 +676,13 @@ class Scheduler:
676
EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
677
timestamp))
678
679
+ def request_preempted(self, request: Request, timestamp: float):
680
+ if not self.log_stats:
681
+ return
682
+ request.events.append(
683
+ EngineCoreEvent.new_event(EngineCoreEventType.PREEMPTED,
684
+ timestamp))
685
+
686
def make_stats(self) -> Optional[SchedulerStats]:
687
if not self.log_stats:
688
return None
vllm/v1/engine/__init__.py
+1
@@ -65,6 +65,7 @@ class EngineCoreEventType(enum.IntEnum):
65
"""The type of engine core request event."""
66
QUEUED = 1
67
SCHEDULED = 2
68
+ PREEMPTED = 3
69
70
71
class EngineCoreEvent(msgspec.Struct):
vllm/v1/metrics/loggers.py
+15
-9
@@ -132,6 +132,11 @@ class PrometheusStatLogger(StatLoggerBase):
132
"GPU prefix cache hits, in terms of number of cached blocks.",
133
labelnames=labelnames).labels(*labelvalues)
134
135
+ self.counter_num_preempted_reqs = prometheus_client.Counter(
136
+ name="vllm:num_preemptions_total",
137
+ documentation="Cumulative number of preemption from the engine.",
138
+ labelnames=labelnames).labels(*labelvalues)
139
+
140
self.counter_prompt_tokens = prometheus_client.Counter(
141
name="vllm:prompt_tokens_total",
142
documentation="Number of prefill tokens processed.",
@@ -282,6 +287,7 @@ class PrometheusStatLogger(StatLoggerBase):
287
self.counter_gpu_prefix_cache_hits.inc(
288
scheduler_stats.prefix_cache_stats.hits)
289
290
+ self.counter_num_preempted_reqs.inc(iteration_stats.num_preempted_reqs)
291
self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
292
self.counter_generation_tokens.inc(
293
iteration_stats.num_generation_tokens)
@@ -289,10 +295,19 @@ class PrometheusStatLogger(StatLoggerBase):
295
iteration_stats.num_prompt_tokens + \
296
iteration_stats.num_generation_tokens)
297
298
+ for ttft in iteration_stats.time_to_first_tokens_iter:
299
+ self.histogram_time_to_first_token.observe(ttft)
300
+ for tpot in iteration_stats.time_per_output_tokens_iter:
301
+ self.histogram_time_per_output_token.observe(tpot)
302
+
303
for finished_request in iteration_stats.finished_requests:
304
self.counter_request_success[finished_request.finish_reason].inc()
305
self.histogram_e2e_time_request.observe(
306
finished_request.e2e_latency)
307
+ self.histogram_queue_time_request.observe(
308
+ finished_request.queued_time)
309
+ self.histogram_prefill_time_request.observe(
310
+ finished_request.prefill_time)
311
self.histogram_inference_time_request.observe(
312
finished_request.inference_time)
313
self.histogram_decode_time_request.observe(
@@ -302,15 +317,6 @@ class PrometheusStatLogger(StatLoggerBase):
317
self.histogram_num_generation_tokens_request.observe(
318
finished_request.num_generation_tokens)
319
305
- for ttft in iteration_stats.time_to_first_tokens_iter:
306
- self.histogram_time_to_first_token.observe(ttft)
307
- for tpot in iteration_stats.time_per_output_tokens_iter:
308
- self.histogram_time_per_output_token.observe(tpot)
309
- for queue_time in iteration_stats.queue_times_iter:
310
- self.histogram_queue_time_request.observe(queue_time)
311
- for prefill_time in iteration_stats.prefill_times_iter:
312
- self.histogram_prefill_time_request.observe(prefill_time)
313
-
320
if self.gauge_lora_info is not None:
321
running_lora_adapters = \
322
",".join(iteration_stats.running_lora_adapters.keys())
vllm/v1/metrics/stats.py
+22
-9
@@ -67,6 +67,8 @@ class FinishedRequestStats:
67
e2e_latency: float = 0.0
68
num_prompt_tokens: int = 0
69
num_generation_tokens: int = 0
70
+ queued_time: float = 0.0
71
+ prefill_time: float = 0.0
72
inference_time: float = 0.0
73
decode_time: float = 0.0
74
@@ -78,11 +80,10 @@ class IterationStats:
80
self.iteration_timestamp = time.time()
81
self.num_generation_tokens = 0
82
self.num_prompt_tokens = 0
83
+ self.num_preempted_reqs = 0
84
self.finished_requests: List[FinishedRequestStats] = []
85
self.time_to_first_tokens_iter: List[float] = []
86
self.time_per_output_tokens_iter: List[float] = []
84
- self.queue_times_iter: List[float] = []
85
- self.prefill_times_iter: List[float] = []
87
self.waiting_lora_adapters: Dict[str, int] = {}
88
self.running_lora_adapters: Dict[str, int] = {}
89
@@ -122,9 +123,6 @@ class IterationStats:
123
if is_prefilling:
124
# TODO: re-enable no-output-for-partial-prefills invariant as above
125
if num_new_generation_tokens > 0:
125
- prefill_interval = \
126
- engine_core_timestamp - req_stats.scheduled_ts
127
- self.prefill_times_iter.append(prefill_interval)
126
req_stats.first_token_ts = engine_core_timestamp
127
else:
128
tpot = engine_core_timestamp - req_stats.last_token_ts
@@ -145,24 +143,39 @@ class IterationStats:
143
if lora_stats is not None:
144
lora_stats.waiting_requests.add(req_id)
145
elif event.type == EngineCoreEventType.SCHEDULED:
148
- queued_interval = event.timestamp - req_stats.queued_ts
149
- self.queue_times_iter.append(queued_interval)
150
- req_stats.scheduled_ts = event.timestamp
146
+ if req_stats.scheduled_ts == 0.0: # ignore preemptions
147
+ req_stats.scheduled_ts = event.timestamp
148
LoRARequestStates.scheduled_request(lora_stats, req_id)
149
+ elif event.type == EngineCoreEventType.PREEMPTED:
150
+ self.num_preempted_reqs += 1
151
152
def update_from_finished_request(self, finish_reason: "FinishReason",
153
request_output: "RequestOutput",
154
req_stats: RequestStateStats):
155
e2e_latency = self._time_since(req_stats.arrival_time)
156
158
- inference_time = req_stats.last_token_ts - req_stats.scheduled_ts
157
+ # Queued interval is from first QUEUED event to first SCHEDULED
158
+ queued_time = req_stats.scheduled_ts - req_stats.queued_ts
159
+
160
+ # Prefill interval is from first SCHEDULED to first NEW_TOKEN
161
+ # Any preemptions during prefill is included in the interval
162
+ prefill_time = req_stats.first_token_ts - req_stats.scheduled_ts
163
+
164
+ # Decode interval is from first NEW_TOKEN to last NEW_TOKEN
165
+ # Any preemptions during decode are included
166
decode_time = req_stats.last_token_ts - req_stats.first_token_ts
167
168
+ # Inference interval is from first SCHEDULED to last NEW_TOKEN
169
+ # Any preemptions during prefill or decode are included
170
+ inference_time = req_stats.last_token_ts - req_stats.scheduled_ts
171
+
172
finished_req = \
173
FinishedRequestStats(finish_reason=finish_reason,
174
e2e_latency=e2e_latency,
175
num_prompt_tokens=len(request_output.prompt_token_ids),
176
num_generation_tokens=req_stats.num_generation_tokens,
177
+ queued_time=queued_time,
178
+ prefill_time=prefill_time,
179
inference_time=inference_time,
180
decode_time=decode_time)
181
self.finished_requests.append(finished_req)