A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[v1][stats][1/n] Add RequestStatsUpdate and RequestStats types (#10907)
Signed-off-by: rickyx <rickyx@anyscale.com>
Ricky Xu committed
Jan 21, 2025 at 11:51 UTC
132a1321004bb994e2260fcc02a3c312bd3b1fe0
3 files changed
+749
tests/v1/test_stats.py
new
+300
@@ -0,0 +1,300 @@
1
+import pytest
2
+
3
+from vllm.sampling_params import SamplingParams
4
+from vllm.v1.stats.common import RequestStats, RequestStatsUpdate
5
+
6
+
7
+def make_update(
8
+ request_id: str,
9
+ update_type: RequestStatsUpdate.Type,
10
+ monotonic_ts_s: float,
11
+ **kwargs,
12
+):
13
+ if update_type == RequestStatsUpdate.Type.INPUT_PROCESSED:
14
+ kwargs.setdefault("sampling_params", SamplingParams(n=1))
15
+ kwargs.setdefault("num_prompt_tokens", 10)
16
+ elif update_type == RequestStatsUpdate.Type.PREFILLING:
17
+ kwargs.setdefault("num_computed_tokens", 10)
18
+ kwargs.setdefault("num_cached_tokens", 10)
19
+ elif update_type == RequestStatsUpdate.Type.DETOKENIZED:
20
+ kwargs.setdefault("num_new_tokens", 10)
21
+ elif update_type == RequestStatsUpdate.Type.FINISHED:
22
+ kwargs.setdefault("finish_reason", "test_reason")
23
+
24
+ return RequestStatsUpdate(
25
+ request_id=request_id,
26
+ type=update_type,
27
+ monotonic_ts_s=monotonic_ts_s,
28
+ **kwargs,
29
+ )
30
+
31
+
32
+def test_invalid_request_update():
33
+ request_id = "test_request"
34
+ update_specific_required_fields = {
35
+ RequestStatsUpdate.Type.INPUT_PROCESSED: [
36
+ "sampling_params",
37
+ "num_prompt_tokens",
38
+ ],
39
+ RequestStatsUpdate.Type.PREFILLING: [
40
+ "num_computed_tokens",
41
+ "num_cached_tokens",
42
+ ],
43
+ RequestStatsUpdate.Type.DETOKENIZED: ["num_new_tokens"],
44
+ RequestStatsUpdate.Type.FINISHED: ["finish_reason"],
45
+ }
46
+
47
+ # Missing a required field should raise an assertion error.
48
+ for update_type in RequestStatsUpdate.Type:
49
+ required_fields = update_specific_required_fields.get(update_type, [])
50
+
51
+ # Try to miss one of the required fields.
52
+ kwargs = {field: object() for field in required_fields}
53
+ for field in required_fields:
54
+ copy_kwargs = kwargs.copy()
55
+ copy_kwargs.pop(field)
56
+ with pytest.raises(ValueError):
57
+ RequestStatsUpdate(
58
+ request_id=request_id,
59
+ type=update_type,
60
+ **copy_kwargs,
61
+ )
62
+
63
+
64
+def test_invalid_request_update_transition():
65
+ # Test invalid transition type.
66
+ for src in RequestStatsUpdate.Type:
67
+ for dst in RequestStatsUpdate.Type:
68
+ if dst not in RequestStatsUpdate._VALID_TRANSITIONS[src]:
69
+ with pytest.raises(AssertionError):
70
+ RequestStatsUpdate.check_valid_update(
71
+ make_update(
72
+ update_type=dst,
73
+ request_id="test_request",
74
+ monotonic_ts_s=1,
75
+ ),
76
+ last_update_type=src,
77
+ last_updated_ts_s=0,
78
+ )
79
+ else:
80
+ RequestStatsUpdate.check_valid_update(
81
+ make_update(
82
+ request_id="test_request",
83
+ update_type=dst,
84
+ monotonic_ts_s=1,
85
+ ),
86
+ last_update_type=src,
87
+ last_updated_ts_s=0,
88
+ )
89
+
90
+ # Test invalid timestamp.
91
+ with pytest.raises(AssertionError):
92
+ RequestStatsUpdate.check_valid_update(
93
+ make_update(
94
+ request_id="test_request",
95
+ update_type=RequestStatsUpdate.Type.ARRIVED,
96
+ monotonic_ts_s=1,
97
+ ),
98
+ last_update_type=None,
99
+ last_updated_ts_s=2,
100
+ )
101
+
102
+
103
+def test_lifecycle_updates():
104
+ request_id = "test_request"
105
+ stats = RequestStats(request_id=request_id)
106
+
107
+ # Test the below scenario:
108
+ arrived_ts = 0
109
+ input_processed_ts = 1
110
+ queued_ts = 2
111
+ prefilling_ts = 3
112
+ decoded_ts = 5
113
+ detokenized_ts = 6
114
+ decoded_2_ts = 7
115
+ detokenized_2_ts = 8
116
+ preempted_ts = 9
117
+ resumed_ts = 10
118
+ decoded_3_ts = 11
119
+ detokenized_3_ts = 12
120
+ finished_ts = 13
121
+
122
+ # Test ARRIVED
123
+ arrived_update = RequestStatsUpdate(
124
+ request_id=request_id,
125
+ type=RequestStatsUpdate.Type.ARRIVED,
126
+ monotonic_ts_s=arrived_ts,
127
+ )
128
+ stats.update_from(arrived_update)
129
+ assert stats.arrival_ts_s == arrived_ts
130
+ assert stats.last_updated_ts_s == arrived_ts
131
+
132
+ # Test INPUT_PROCESSED
133
+ sampling_params = SamplingParams(n=1)
134
+ input_processed_update = RequestStatsUpdate(
135
+ request_id=request_id,
136
+ type=RequestStatsUpdate.Type.INPUT_PROCESSED,
137
+ monotonic_ts_s=input_processed_ts,
138
+ sampling_params=sampling_params,
139
+ num_prompt_tokens=6,
140
+ )
141
+ stats.update_from(input_processed_update)
142
+ assert stats.input_processor_end_ts_s == input_processed_ts
143
+ assert stats.last_updated_ts_s == input_processed_ts
144
+ assert stats.num_prompt_tokens == 6
145
+ assert stats.sampling_params == sampling_params
146
+
147
+ assert stats.first_token_ts_s is None
148
+ assert stats.prefill_ts_s is None
149
+
150
+ # Test QUEUED
151
+ queued_update = RequestStatsUpdate(
152
+ request_id=request_id,
153
+ type=RequestStatsUpdate.Type.QUEUED,
154
+ monotonic_ts_s=queued_ts,
155
+ )
156
+ stats.update_from(queued_update)
157
+ assert stats.queued_ts_s == queued_ts
158
+ assert stats.last_updated_ts_s == queued_ts
159
+
160
+ # Test PREFILLING
161
+ prefilling_update = RequestStatsUpdate(
162
+ request_id=request_id,
163
+ type=RequestStatsUpdate.Type.PREFILLING,
164
+ monotonic_ts_s=prefilling_ts,
165
+ num_computed_tokens=3,
166
+ num_cached_tokens=1,
167
+ )
168
+ stats.update_from(prefilling_update)
169
+ assert stats.prefill_ts_s == prefilling_ts
170
+ assert stats.num_computed_tokens == 3
171
+ assert stats.num_cached_tokens == 1
172
+ assert stats.queue_duration_s == prefilling_ts - queued_ts
173
+
174
+ # Test DECODING
175
+ decoded_update = RequestStatsUpdate(
176
+ request_id=request_id,
177
+ type=RequestStatsUpdate.Type.DECODING,
178
+ monotonic_ts_s=decoded_ts,
179
+ )
180
+ stats.update_from(decoded_update)
181
+ assert stats.last_updated_ts_s == decoded_ts
182
+
183
+ # Test DETOKENIZED
184
+ detokenized_update = RequestStatsUpdate(
185
+ request_id=request_id,
186
+ type=RequestStatsUpdate.Type.DETOKENIZED,
187
+ monotonic_ts_s=detokenized_ts,
188
+ num_new_tokens=1,
189
+ )
190
+ stats.update_from(detokenized_update)
191
+ assert stats.last_updated_ts_s == detokenized_ts
192
+ assert stats.num_output_tokens == 1
193
+ # Since arrival
194
+ assert stats.first_token_latency_s == detokenized_ts - arrived_ts
195
+ # Since first scheduled
196
+ assert stats.prefill_latency_s == detokenized_ts - prefilling_ts
197
+
198
+ # Test another DECODING and DETOKENIZED should
199
+ # yield correct inter token latency
200
+ decoded_update = RequestStatsUpdate(
201
+ request_id=request_id,
202
+ type=RequestStatsUpdate.Type.DECODING,
203
+ monotonic_ts_s=decoded_2_ts,
204
+ )
205
+ stats.update_from(decoded_update)
206
+
207
+ detokenized_update = RequestStatsUpdate(
208
+ request_id=request_id,
209
+ type=RequestStatsUpdate.Type.DETOKENIZED,
210
+ monotonic_ts_s=detokenized_2_ts,
211
+ num_new_tokens=1,
212
+ )
213
+ stats.update_from(detokenized_update)
214
+ assert stats.output_token_latency_s_lst == [
215
+ detokenized_2_ts - detokenized_ts,
216
+ ]
217
+ assert stats.num_output_tokens == 2
218
+
219
+ # Test PREEMPTED
220
+ preempted_update = RequestStatsUpdate(
221
+ request_id=request_id,
222
+ type=RequestStatsUpdate.Type.PREEMPTED,
223
+ monotonic_ts_s=preempted_ts,
224
+ )
225
+ stats.update_from(preempted_update)
226
+ assert stats.last_updated_ts_s == preempted_ts
227
+ assert stats.preempted_ts_s_lst == [preempted_ts]
228
+ # States should be reset
229
+ assert stats.num_computed_tokens == 0
230
+ assert stats.num_cached_tokens == 0
231
+ # These states should not be reset
232
+ assert stats.num_output_tokens == 2
233
+ assert stats.output_token_latency_s_lst == [
234
+ detokenized_2_ts - detokenized_ts,
235
+ ]
236
+ assert stats.prefill_latency_s == prefilling_ts - arrived_ts
237
+ assert stats.num_prompt_tokens == 6
238
+ assert stats.prefill_start_ts_s_lst == [prefilling_ts]
239
+
240
+ # Test resumed
241
+ resumed_update = RequestStatsUpdate(
242
+ request_id=request_id,
243
+ type=RequestStatsUpdate.Type.PREFILLING,
244
+ monotonic_ts_s=resumed_ts,
245
+ num_computed_tokens=6,
246
+ num_cached_tokens=2,
247
+ )
248
+ stats.update_from(resumed_update)
249
+ # prefill timestamp should not be updated since it's a resumed prefill
250
+ assert stats.prefill_ts_s == prefilling_ts
251
+ assert stats.num_computed_tokens == 6
252
+ assert stats.num_cached_tokens == 2
253
+ assert stats.prefill_start_ts_s_lst == [
254
+ prefilling_ts,
255
+ resumed_ts,
256
+ ]
257
+ assert stats.last_updated_ts_s == resumed_ts
258
+
259
+ # Test another DECODED/DETOKENIZED should yield correct first token latency.
260
+ decoded_update = RequestStatsUpdate(
261
+ request_id=request_id,
262
+ type=RequestStatsUpdate.Type.DECODING,
263
+ monotonic_ts_s=decoded_3_ts,
264
+ )
265
+ detokenized_update = RequestStatsUpdate(
266
+ request_id=request_id,
267
+ type=RequestStatsUpdate.Type.DETOKENIZED,
268
+ monotonic_ts_s=detokenized_3_ts,
269
+ num_new_tokens=1,
270
+ )
271
+ stats.update_from(decoded_update)
272
+ stats.update_from(detokenized_update)
273
+ assert stats.first_token_ts_s == detokenized_ts - arrived_ts
274
+ assert stats.num_output_tokens == 3
275
+ assert stats.output_token_latency_s_lst == [
276
+ detokenized_2_ts - detokenized_ts,
277
+ detokenized_3_ts - detokenized_2_ts,
278
+ ]
279
+
280
+ # Test FINISHED
281
+ finished_update = RequestStatsUpdate(
282
+ request_id=request_id,
283
+ type=RequestStatsUpdate.Type.FINISHED,
284
+ monotonic_ts_s=finished_ts,
285
+ finish_reason="test_reason",
286
+ )
287
+ stats.update_from(finished_update)
288
+ assert stats.last_updated_ts_s == finished_ts
289
+ assert stats.e2e_latency_s == finished_ts - arrived_ts
290
+ assert stats.inference_latency_s == finished_ts - prefilling_ts
291
+ assert stats.prefill_latency_s == detokenized_ts - prefilling_ts
292
+ assert stats.decode_latency_s == finished_ts - detokenized_ts
293
+ assert stats.first_token_latency_s == detokenized_ts - arrived_ts
294
+ assert stats.queue_duration_s == prefilling_ts - queued_ts
295
+ assert stats.is_finished
296
+ assert stats.finish_reason == "test_reason"
297
+
298
+ # TODO(rickyx): Add model forward/execute time.
299
+ assert stats.model_forward_duration_s == 0.0
300
+ assert stats.model_execute_duration_s == 0.0
vllm/v1/stats/__init__.py
vllm/v1/stats/common.py
new
+449
@@ -0,0 +1,449 @@
1
+import time
2
+from dataclasses import dataclass
3
+from dataclasses import field as dataclass_field
4
+from enum import IntEnum
5
+from typing import ClassVar, Dict, List, Optional, Set
6
+
7
+import msgspec
8
+from msgspec import field as msgspec_field
9
+
10
+from vllm.sampling_params import SamplingParams
11
+
12
+
13
+class RequestStatsUpdate(msgspec.Struct,
14
+ array_like=True,
15
+ omit_defaults=True,
16
+ gc=False):
17
+ """
18
+ An update to the request stats.
19
+
20
+ This represents a stats update at a specific timestamp with metadata
21
+ associated with the update.
22
+
23
+ NOTE: since there might be multiple processes generating updates at
24
+ different parts of the engine (e.g. input processor, scheduler, engine core,
25
+ etc.), we use the monotonic timestamp to record the update to compute any
26
+ intervals, and explicit wall-clock timestamp should be used for timestamps.
27
+
28
+ WARNING: This assumes stats are generated in a single machine. If there are
29
+ potentially multiple machines, one should always generate the stats updates
30
+ on one single machine or use something else.
31
+ """
32
+
33
+ class Type(IntEnum):
34
+ """See `RequestStats` for the lifecycle of a request."""
35
+
36
+ # Request arrived at the engine frontend.
37
+ ARRIVED = 0
38
+ # Input processed by the input processor.
39
+ INPUT_PROCESSED = 1
40
+ # Queued on the engine core.
41
+ QUEUED = 2
42
+ # Scheduled running prefill by the scheduler.
43
+ # A request could be running a new prefill on the prompt tokens or
44
+ # a resumed prefill on the original prefill tokens + generated output
45
+ # tokens before preemption.
46
+ PREFILLING = 3
47
+ # Preempted by the scheduler.
48
+ PREEMPTED = 4
49
+ # Output token is generated by the engine core.
50
+ DECODING = 5
51
+ # Token detokenized by the detokenizer.
52
+ # We will record the timestamp for each output token, as well as the
53
+ # finish reason.
54
+ DETOKENIZED = 6
55
+ # Request finishes (or aborts).
56
+ FINISHED = 7
57
+
58
+ """
59
+ Valid state updates:
60
+ ARRIVED
61
+ │
62
+ ├──────► INPUT_PROCESSED ──────► QUEUED ──────► PREFILLING ◄────┐
63
+ │ │ │ │ │
64
+ │ │ │ ▼ │
65
+ │ │ │ -──► DECODING │
66
+ │ │ │ | │ │
67
+ │ │ │ | ▼ │
68
+ │ │ │ └─ DETOKENIZED │
69
+ │ │ │ │ │
70
+ │ │ │ ▼ │
71
+ │ ▼ ▼ PREEMPTED ◄──────┘
72
+ │ │ │ │
73
+ └──────────────┴───────────────────┴──────────────┴
74
+ │
75
+ ▼
76
+ FINISHED (All could go to FINISHED)
77
+ """
78
+ _VALID_TRANSITIONS: ClassVar[Dict[Type, Set[Type]]] = {
79
+ Type.ARRIVED: {
80
+ Type.INPUT_PROCESSED,
81
+ Type.FINISHED,
82
+ },
83
+ Type.INPUT_PROCESSED: {
84
+ Type.QUEUED,
85
+ Type.FINISHED,
86
+ },
87
+ Type.QUEUED: {
88
+ Type.PREFILLING,
89
+ Type.FINISHED,
90
+ },
91
+ Type.PREFILLING: {
92
+ Type.DECODING,
93
+ Type.PREEMPTED,
94
+ Type.FINISHED,
95
+ },
96
+ Type.DECODING: {
97
+ Type.DETOKENIZED,
98
+ Type.FINISHED,
99
+ },
100
+ Type.DETOKENIZED: {
101
+ Type.DECODING,
102
+ Type.PREEMPTED,
103
+ Type.FINISHED,
104
+ },
105
+ Type.PREEMPTED: {Type.PREFILLING, Type.FINISHED},
106
+ Type.FINISHED: set(),
107
+ }
108
+
109
+ request_id: str
110
+
111
+ type: Type
112
+
113
+ # Timestamp when the update is recorded. This is used to record time
114
+ # intervals between events rather than wall clock time.
115
+ monotonic_ts_s: float = msgspec_field(
116
+ default_factory=lambda: time.monotonic())
117
+
118
+ ############################################################
119
+ # Metadata associated with the update.
120
+ ############################################################
121
+ # For input_processed. Metadata needed for stats logging.
122
+ num_prompt_tokens: Optional[int] = None
123
+ sampling_params: Optional[SamplingParams] = None
124
+
125
+ # For running.
126
+ # Number of tokens computed when scheduled to run.
127
+ num_computed_tokens: Optional[int] = None
128
+ # Number of cached tokens when scheduled to run.
129
+ num_cached_tokens: Optional[int] = None
130
+
131
+ # For decoded.
132
+ # The number of new output tokens generated.
133
+ num_new_tokens: Optional[int] = None
134
+
135
+ # For both detokenized and decoded.
136
+ # Finished reason.
137
+ finish_reason: Optional[str] = None
138
+
139
+ # Non-optional fields for each update type.
140
+ _REQUIRED_FIELDS: ClassVar[Dict[Type, List[str]]] = {
141
+ Type.INPUT_PROCESSED: ["num_prompt_tokens", "sampling_params"],
142
+ Type.PREFILLING: ["num_computed_tokens", "num_cached_tokens"],
143
+ Type.DETOKENIZED: ["num_new_tokens"],
144
+ Type.FINISHED: ["finish_reason"],
145
+ }
146
+
147
+ def __post_init__(self):
148
+ required_fields = self._REQUIRED_FIELDS.get(self.type, [])
149
+ for field in required_fields:
150
+ if getattr(self, field) is None:
151
+ raise ValueError(
152
+ f"Field {field} is required for update type {self.type}.")
153
+
154
+ @staticmethod
155
+ def check_valid_update(
156
+ update: "RequestStatsUpdate",
157
+ last_update_type: Optional[Type],
158
+ last_updated_ts_s: Optional[float],
159
+ ):
160
+ if last_update_type is None:
161
+ assert update.type == RequestStatsUpdate.Type.ARRIVED
162
+ else:
163
+ valid_cur_update_types = RequestStatsUpdate._VALID_TRANSITIONS[
164
+ last_update_type]
165
+ assert update.type in valid_cur_update_types, (
166
+ f"Invalid update type: {update.type} for last_update_type: "
167
+ f"{last_update_type}.")
168
+
169
+ if last_updated_ts_s is not None:
170
+ assert update.monotonic_ts_s >= last_updated_ts_s, (
171
+ "Update timestamp must be monotonically increasing, but "
172
+ f"last_updated_ts_s={last_updated_ts_s} and "
173
+ f"update.monotonic_ts_s={update.monotonic_ts_s}.")
174
+
175
+
176
+@dataclass
177
+class RequestStats:
178
+ """Stats associated with a request (`Request`)."""
179
+
180
+ ############################################################
181
+ # Metadata
182
+ ############################################################
183
+ request_id: str
184
+ sampling_params: Optional[SamplingParams] = None
185
+ num_prompt_tokens: Optional[int] = None
186
+
187
+ ############################################################
188
+ # Metrics and Stats
189
+ ############################################################
190
+ # Timestamp when the request was last updated.
191
+ last_updated_ts_s: Optional[float] = None
192
+
193
+ # Last update stats type.
194
+ last_update_type: Optional[RequestStatsUpdate.Type] = None
195
+
196
+ # Timestamp when the request arrived at the llm engine.
197
+ arrival_ts_s: Optional[float] = None
198
+
199
+ # Number of tokens cached. When part of the request prefix is cached,
200
+ # this will be set.
201
+ num_cached_tokens: int = 0
202
+
203
+ # Number of tokens computed.
204
+ num_computed_tokens: int = 0
205
+
206
+ # The timestamp when the request become waiting in the queue.
207
+ queued_ts_s: Optional[float] = None
208
+
209
+ # When the input processor is completed.
210
+ input_processor_end_ts_s: Optional[float] = None
211
+
212
+ # A sorted list of timestamps when the request was scheduled to prefill.
213
+ # This could be when:
214
+ # 1. the request is newly scheduled, so it's a new prefill.
215
+ # 2. the request was preempted and resumed. It is equivalent to running
216
+ # a prefill of the original prefill tokens + generated output tokens
217
+ # before preemption.
218
+ prefill_start_ts_s_lst: List[float] = dataclass_field(default_factory=list)
219
+
220
+ # A list of timestamps when a token is decoded by the engine core.
221
+ decoding_ts_s_lst: List[float] = dataclass_field(default_factory=list)
222
+
223
+ # A sorted list of timestamps for each output token.
224
+ output_token_ts_s_lst: List[float] = dataclass_field(default_factory=list)
225
+
226
+ # First token's timestamp.
227
+ first_token_ts_s: Optional[float] = None
228
+
229
+ # TODO(rickyx): we need model runner to surface these.
230
+ model_forward_duration_s: float = 0.0
231
+ # Includes model forward, block/sync across workers, cpu-gpu sync time
232
+ # and sampling time.
233
+ model_execute_duration_s: float = 0.0
234
+
235
+ # A sorted list of timestamps when the request was preempted at the
236
+ # scheduler.
237
+ # TODO(rickyx): right now, we don't actually have a good high-level
238
+ # metric to measure the impact of preemption other than observation of
239
+ # large P99 TPOT. Ideally we could quantify the impact of preemption by
240
+ # measuring the number of tokens re-computed due to preemption.
241
+ preempted_ts_s_lst: List[float] = dataclass_field(default_factory=list)
242
+
243
+ # Timestamp when the request was finished at the engine core.
244
+ finished_ts_s: Optional[float] = None
245
+
246
+ # Finish reason.
247
+ finish_reason: Optional[str] = None
248
+
249
+ ############################################################
250
+ # Derived properties.
251
+ ############################################################
252
+ @property
253
+ def prefill_ts_s(self) -> Optional[float]:
254
+ """The timestamp when the request started prefilling.
255
+ Since a request could be preempted in decoding and later resumed
256
+ to prefill the decoded tokens, we use the first prefill start timestamp.
257
+ """
258
+ return (self.prefill_start_ts_s_lst[0]
259
+ if self.prefill_start_ts_s_lst else None)
260
+
261
+ @property
262
+ def e2e_latency_s(self) -> Optional[float]:
263
+ if self.finished_ts_s is None or self.arrival_ts_s is None:
264
+ return None
265
+ assert self.finished_ts_s >= self.arrival_ts_s
266
+ return self.finished_ts_s - self.arrival_ts_s
267
+
268
+ @property
269
+ def queue_duration_s(self) -> Optional[float]:
270
+ """How long the request was waiting to run."""
271
+ if self.queued_ts_s is None or self.prefill_ts_s is None:
272
+ # Either not queued or not running yet.
273
+ return None
274
+ assert self.queued_ts_s <= self.prefill_ts_s
275
+ return self.prefill_ts_s - self.queued_ts_s
276
+
277
+ @property
278
+ def inference_latency_s(self) -> Optional[float]:
279
+ """How long the request was running inference
280
+ (prefill and decode)."""
281
+ if self.finished_ts_s is None or self.prefill_ts_s is None:
282
+ return None
283
+ assert self.finished_ts_s >= self.prefill_ts_s
284
+ return self.finished_ts_s - self.prefill_ts_s
285
+
286
+ @property
287
+ def first_token_latency_s(self) -> Optional[float]:
288
+ if self.first_token_ts_s is None or self.arrival_ts_s is None:
289
+ return None
290
+ assert self.first_token_ts_s >= self.arrival_ts_s
291
+ return self.first_token_ts_s - self.arrival_ts_s
292
+
293
+ @property
294
+ def prefill_latency_s(self) -> Optional[float]:
295
+ if self.first_token_ts_s is None or self.prefill_ts_s is None:
296
+ return None
297
+ assert self.first_token_ts_s >= self.prefill_ts_s
298
+ return self.first_token_ts_s - self.prefill_ts_s
299
+
300
+ @property
301
+ def decode_latency_s(self) -> Optional[float]:
302
+ if self.e2e_latency_s is None or self.first_token_latency_s is None:
303
+ return None
304
+ assert self.e2e_latency_s >= self.first_token_latency_s
305
+ return self.e2e_latency_s - self.first_token_latency_s
306
+
307
+ @property
308
+ def output_token_latency_s_lst(self) -> List[float]:
309
+ if len(self.output_token_ts_s_lst) == 0:
310
+ return []
311
+ latency_s_lst = []
312
+ for i in range(1, len(self.output_token_ts_s_lst)):
313
+ assert (self.output_token_ts_s_lst[i] >=
314
+ self.output_token_ts_s_lst[i - 1])
315
+ latency_s = (self.output_token_ts_s_lst[i] -
316
+ self.output_token_ts_s_lst[i - 1])
317
+ latency_s_lst.append(latency_s)
318
+ return latency_s_lst
319
+
320
+ @property
321
+ def num_output_tokens(self) -> int:
322
+ return len(self.output_token_ts_s_lst)
323
+
324
+ @property
325
+ def is_finished(self) -> bool:
326
+ return self.finished_ts_s is not None
327
+
328
+ def update_from(self, update: "RequestStatsUpdate"):
329
+ RequestStatsUpdate.check_valid_update(update, self.last_update_type,
330
+ self.last_updated_ts_s)
331
+ ts = update.monotonic_ts_s
332
+ self.last_updated_ts_s = ts
333
+ self.last_update_type = update.type
334
+ if update.type == RequestStatsUpdate.Type.ARRIVED:
335
+ self.arrival_ts_s = ts
336
+ elif update.type == RequestStatsUpdate.Type.INPUT_PROCESSED:
337
+ self.input_processor_end_ts_s = ts
338
+ self.sampling_params = update.sampling_params
339
+ self.num_prompt_tokens = update.num_prompt_tokens
340
+ elif update.type == RequestStatsUpdate.Type.QUEUED:
341
+ self.queued_ts_s = ts
342
+ elif update.type == RequestStatsUpdate.Type.PREFILLING:
343
+ self.prefill_start_ts_s_lst.append(ts)
344
+ self.num_cached_tokens = update.num_cached_tokens
345
+ self.num_computed_tokens = update.num_computed_tokens
346
+ elif update.type == RequestStatsUpdate.Type.PREEMPTED:
347
+ self._reset_for_preemption(ts)
348
+ elif update.type == RequestStatsUpdate.Type.DECODING:
349
+ self.decoding_ts_s_lst.append(ts)
350
+ elif update.type == RequestStatsUpdate.Type.DETOKENIZED:
351
+ self._record_detokenized_output(
352
+ ts,
353
+ update.num_new_tokens,
354
+ )
355
+ elif update.type == RequestStatsUpdate.Type.FINISHED:
356
+ self.finished_ts_s = ts
357
+ self.finish_reason = update.finish_reason
358
+ else:
359
+ raise ValueError(f"Unknown update type: {update.type}")
360
+
361
+ def _record_detokenized_output(
362
+ self,
363
+ ts_s: float,
364
+ num_new_tokens: int,
365
+ ):
366
+ # Update if first output token is generated.
367
+ if len(self.output_token_ts_s_lst) == 0:
368
+ self.first_token_ts_s = ts_s
369
+ assert (
370
+ self.prefill_ts_s is not None
371
+ ), "Request must be running before generating output tokens."
372
+
373
+ # Some X new tokens were generated at the ts.
374
+ self.output_token_ts_s_lst.extend([ts_s] * num_new_tokens)
375
+
376
+ def _reset_for_preemption(self, ts_s: float):
377
+ self.preempted_ts_s_lst.append(ts_s)
378
+ # Reset the computed tokens since it might restart the prefill.
379
+ self.num_computed_tokens = 0
380
+ # Cached token count might also change when resumed.
381
+ self.num_cached_tokens = 0
382
+ # These stats don't change since they happen before request running.
383
+ # - arrival_ts_s
384
+ # - input_processor_end_ts_s
385
+ # - sampling_params
386
+ # - num_prompt_tokens
387
+ # - first_token_ts_s
388
+ #
389
+ # These stats are accumulated over preemptions:
390
+ # - output_token_ts_s_lst
391
+ # - prefill_start_ts_s_lst (after preemption, it will prefill the
392
+ # original prefill tokens and any output tokens generated before
393
+ # preemption.)
394
+
395
+
396
+@dataclass
397
+class KVCacheStats:
398
+ # KV Cache Usage in %
399
+ gpu_cache_usage_sys: float = 0.0
400
+ gpu_prefix_cache_hit_rate: float = 0.0
401
+
402
+
403
+@dataclass
404
+class SchedulerStats:
405
+ """Stats associated with the scheduler."""
406
+
407
+ # Number of requests currently running.
408
+ num_running_reqs: int = 0
409
+ # Number of requests currently waiting.
410
+ num_waiting_reqs: int = 0
411
+
412
+ kv_cache_stats: KVCacheStats = dataclass_field(
413
+ default_factory=KVCacheStats)
414
+
415
+
416
+@dataclass
417
+class EngineCoreProcessStats:
418
+ """Stats associated with the engine core process."""
419
+
420
+ # Number of requests currently in the input queue. None if the engine core
421
+ # is not running in multiprocess mode.
422
+ input_queue_size: Optional[int] = None
423
+ # Number of outputs currently in the output queue. None if the engine core
424
+ # is not running in multiprocess mode.
425
+ output_queue_size: Optional[int] = None
426
+
427
+
428
+class EngineCoreStatsSnapshot(msgspec.Struct,
429
+ array_like=True,
430
+ omit_defaults=True,
431
+ gc=False):
432
+ """
433
+ A snapshot of the EngineCore's current stats over a period of time.
434
+ """
435
+
436
+ # Snapshot of the scheduler stats.
437
+ scheduler_stats: SchedulerStats = msgspec_field(
438
+ default_factory=SchedulerStats)
439
+
440
+ # Per request stats updates.
441
+ requests_stats_updates: List[RequestStatsUpdate] = msgspec_field(
442
+ default_factory=list)
443
+
444
+ # Engine core's queue stats.
445
+ engine_core_process_stats: EngineCoreProcessStats = msgspec_field(
446
+ default_factory=EngineCoreProcessStats)
447
+
448
+ # TODO(rickyx): Add other components' stats,
449
+ # e.g. model runner/worker and etc.