@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 132a132100
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[v1][stats][1/n] Add RequestStatsUpdate and RequestStats types (#10907)

Signed-off-by: rickyx <rickyx@anyscale.com>

Ricky Xu committed Jan 21, 2025 at 11:51 UTC 132a1321004bb994e2260fcc02a3c312bd3b1fe0
3 files changed +749
tests/v1/test_stats.py new
+300
@@ -0,0 +1,300 @@
1 +import pytest
2 +
3 +from vllm.sampling_params import SamplingParams
4 +from vllm.v1.stats.common import RequestStats, RequestStatsUpdate
5 +
6 +
7 +def make_update(
8 + request_id: str,
9 + update_type: RequestStatsUpdate.Type,
10 + monotonic_ts_s: float,
11 + **kwargs,
12 +):
13 + if update_type == RequestStatsUpdate.Type.INPUT_PROCESSED:
14 + kwargs.setdefault("sampling_params", SamplingParams(n=1))
15 + kwargs.setdefault("num_prompt_tokens", 10)
16 + elif update_type == RequestStatsUpdate.Type.PREFILLING:
17 + kwargs.setdefault("num_computed_tokens", 10)
18 + kwargs.setdefault("num_cached_tokens", 10)
19 + elif update_type == RequestStatsUpdate.Type.DETOKENIZED:
20 + kwargs.setdefault("num_new_tokens", 10)
21 + elif update_type == RequestStatsUpdate.Type.FINISHED:
22 + kwargs.setdefault("finish_reason", "test_reason")
23 +
24 + return RequestStatsUpdate(
25 + request_id=request_id,
26 + type=update_type,
27 + monotonic_ts_s=monotonic_ts_s,
28 + **kwargs,
29 + )
30 +
31 +
32 +def test_invalid_request_update():
33 + request_id = "test_request"
34 + update_specific_required_fields = {
35 + RequestStatsUpdate.Type.INPUT_PROCESSED: [
36 + "sampling_params",
37 + "num_prompt_tokens",
38 + ],
39 + RequestStatsUpdate.Type.PREFILLING: [
40 + "num_computed_tokens",
41 + "num_cached_tokens",
42 + ],
43 + RequestStatsUpdate.Type.DETOKENIZED: ["num_new_tokens"],
44 + RequestStatsUpdate.Type.FINISHED: ["finish_reason"],
45 + }
46 +
47 + # Missing a required field should raise an assertion error.
48 + for update_type in RequestStatsUpdate.Type:
49 + required_fields = update_specific_required_fields.get(update_type, [])
50 +
51 + # Try to miss one of the required fields.
52 + kwargs = {field: object() for field in required_fields}
53 + for field in required_fields:
54 + copy_kwargs = kwargs.copy()
55 + copy_kwargs.pop(field)
56 + with pytest.raises(ValueError):
57 + RequestStatsUpdate(
58 + request_id=request_id,
59 + type=update_type,
60 + **copy_kwargs,
61 + )
62 +
63 +
64 +def test_invalid_request_update_transition():
65 + # Test invalid transition type.
66 + for src in RequestStatsUpdate.Type:
67 + for dst in RequestStatsUpdate.Type:
68 + if dst not in RequestStatsUpdate._VALID_TRANSITIONS[src]:
69 + with pytest.raises(AssertionError):
70 + RequestStatsUpdate.check_valid_update(
71 + make_update(
72 + update_type=dst,
73 + request_id="test_request",
74 + monotonic_ts_s=1,
75 + ),
76 + last_update_type=src,
77 + last_updated_ts_s=0,
78 + )
79 + else:
80 + RequestStatsUpdate.check_valid_update(
81 + make_update(
82 + request_id="test_request",
83 + update_type=dst,
84 + monotonic_ts_s=1,
85 + ),
86 + last_update_type=src,
87 + last_updated_ts_s=0,
88 + )
89 +
90 + # Test invalid timestamp.
91 + with pytest.raises(AssertionError):
92 + RequestStatsUpdate.check_valid_update(
93 + make_update(
94 + request_id="test_request",
95 + update_type=RequestStatsUpdate.Type.ARRIVED,
96 + monotonic_ts_s=1,
97 + ),
98 + last_update_type=None,
99 + last_updated_ts_s=2,
100 + )
101 +
102 +
103 +def test_lifecycle_updates():
104 + request_id = "test_request"
105 + stats = RequestStats(request_id=request_id)
106 +
107 + # Test the below scenario:
108 + arrived_ts = 0
109 + input_processed_ts = 1
110 + queued_ts = 2
111 + prefilling_ts = 3
112 + decoded_ts = 5
113 + detokenized_ts = 6
114 + decoded_2_ts = 7
115 + detokenized_2_ts = 8
116 + preempted_ts = 9
117 + resumed_ts = 10
118 + decoded_3_ts = 11
119 + detokenized_3_ts = 12
120 + finished_ts = 13
121 +
122 + # Test ARRIVED
123 + arrived_update = RequestStatsUpdate(
124 + request_id=request_id,
125 + type=RequestStatsUpdate.Type.ARRIVED,
126 + monotonic_ts_s=arrived_ts,
127 + )
128 + stats.update_from(arrived_update)
129 + assert stats.arrival_ts_s == arrived_ts
130 + assert stats.last_updated_ts_s == arrived_ts
131 +
132 + # Test INPUT_PROCESSED
133 + sampling_params = SamplingParams(n=1)
134 + input_processed_update = RequestStatsUpdate(
135 + request_id=request_id,
136 + type=RequestStatsUpdate.Type.INPUT_PROCESSED,
137 + monotonic_ts_s=input_processed_ts,
138 + sampling_params=sampling_params,
139 + num_prompt_tokens=6,
140 + )
141 + stats.update_from(input_processed_update)
142 + assert stats.input_processor_end_ts_s == input_processed_ts
143 + assert stats.last_updated_ts_s == input_processed_ts
144 + assert stats.num_prompt_tokens == 6
145 + assert stats.sampling_params == sampling_params
146 +
147 + assert stats.first_token_ts_s is None
148 + assert stats.prefill_ts_s is None
149 +
150 + # Test QUEUED
151 + queued_update = RequestStatsUpdate(
152 + request_id=request_id,
153 + type=RequestStatsUpdate.Type.QUEUED,
154 + monotonic_ts_s=queued_ts,
155 + )
156 + stats.update_from(queued_update)
157 + assert stats.queued_ts_s == queued_ts
158 + assert stats.last_updated_ts_s == queued_ts
159 +
160 + # Test PREFILLING
161 + prefilling_update = RequestStatsUpdate(
162 + request_id=request_id,
163 + type=RequestStatsUpdate.Type.PREFILLING,
164 + monotonic_ts_s=prefilling_ts,
165 + num_computed_tokens=3,
166 + num_cached_tokens=1,
167 + )
168 + stats.update_from(prefilling_update)
169 + assert stats.prefill_ts_s == prefilling_ts
170 + assert stats.num_computed_tokens == 3
171 + assert stats.num_cached_tokens == 1
172 + assert stats.queue_duration_s == prefilling_ts - queued_ts
173 +
174 + # Test DECODING
175 + decoded_update = RequestStatsUpdate(
176 + request_id=request_id,
177 + type=RequestStatsUpdate.Type.DECODING,
178 + monotonic_ts_s=decoded_ts,
179 + )
180 + stats.update_from(decoded_update)
181 + assert stats.last_updated_ts_s == decoded_ts
182 +
183 + # Test DETOKENIZED
184 + detokenized_update = RequestStatsUpdate(
185 + request_id=request_id,
186 + type=RequestStatsUpdate.Type.DETOKENIZED,
187 + monotonic_ts_s=detokenized_ts,
188 + num_new_tokens=1,
189 + )
190 + stats.update_from(detokenized_update)
191 + assert stats.last_updated_ts_s == detokenized_ts
192 + assert stats.num_output_tokens == 1
193 + # Since arrival
194 + assert stats.first_token_latency_s == detokenized_ts - arrived_ts
195 + # Since first scheduled
196 + assert stats.prefill_latency_s == detokenized_ts - prefilling_ts
197 +
198 + # Test another DECODING and DETOKENIZED should
199 + # yield correct inter token latency
200 + decoded_update = RequestStatsUpdate(
201 + request_id=request_id,
202 + type=RequestStatsUpdate.Type.DECODING,
203 + monotonic_ts_s=decoded_2_ts,
204 + )
205 + stats.update_from(decoded_update)
206 +
207 + detokenized_update = RequestStatsUpdate(
208 + request_id=request_id,
209 + type=RequestStatsUpdate.Type.DETOKENIZED,
210 + monotonic_ts_s=detokenized_2_ts,
211 + num_new_tokens=1,
212 + )
213 + stats.update_from(detokenized_update)
214 + assert stats.output_token_latency_s_lst == [
215 + detokenized_2_ts - detokenized_ts,
216 + ]
217 + assert stats.num_output_tokens == 2
218 +
219 + # Test PREEMPTED
220 + preempted_update = RequestStatsUpdate(
221 + request_id=request_id,
222 + type=RequestStatsUpdate.Type.PREEMPTED,
223 + monotonic_ts_s=preempted_ts,
224 + )
225 + stats.update_from(preempted_update)
226 + assert stats.last_updated_ts_s == preempted_ts
227 + assert stats.preempted_ts_s_lst == [preempted_ts]
228 + # States should be reset
229 + assert stats.num_computed_tokens == 0
230 + assert stats.num_cached_tokens == 0
231 + # These states should not be reset
232 + assert stats.num_output_tokens == 2
233 + assert stats.output_token_latency_s_lst == [
234 + detokenized_2_ts - detokenized_ts,
235 + ]
236 + assert stats.prefill_latency_s == prefilling_ts - arrived_ts
237 + assert stats.num_prompt_tokens == 6
238 + assert stats.prefill_start_ts_s_lst == [prefilling_ts]
239 +
240 + # Test resumed
241 + resumed_update = RequestStatsUpdate(
242 + request_id=request_id,
243 + type=RequestStatsUpdate.Type.PREFILLING,
244 + monotonic_ts_s=resumed_ts,
245 + num_computed_tokens=6,
246 + num_cached_tokens=2,
247 + )
248 + stats.update_from(resumed_update)
249 + # prefill timestamp should not be updated since it's a resumed prefill
250 + assert stats.prefill_ts_s == prefilling_ts
251 + assert stats.num_computed_tokens == 6
252 + assert stats.num_cached_tokens == 2
253 + assert stats.prefill_start_ts_s_lst == [
254 + prefilling_ts,
255 + resumed_ts,
256 + ]
257 + assert stats.last_updated_ts_s == resumed_ts
258 +
259 + # Test another DECODED/DETOKENIZED should yield correct first token latency.
260 + decoded_update = RequestStatsUpdate(
261 + request_id=request_id,
262 + type=RequestStatsUpdate.Type.DECODING,
263 + monotonic_ts_s=decoded_3_ts,
264 + )
265 + detokenized_update = RequestStatsUpdate(
266 + request_id=request_id,
267 + type=RequestStatsUpdate.Type.DETOKENIZED,
268 + monotonic_ts_s=detokenized_3_ts,
269 + num_new_tokens=1,
270 + )
271 + stats.update_from(decoded_update)
272 + stats.update_from(detokenized_update)
273 + assert stats.first_token_ts_s == detokenized_ts - arrived_ts
274 + assert stats.num_output_tokens == 3
275 + assert stats.output_token_latency_s_lst == [
276 + detokenized_2_ts - detokenized_ts,
277 + detokenized_3_ts - detokenized_2_ts,
278 + ]
279 +
280 + # Test FINISHED
281 + finished_update = RequestStatsUpdate(
282 + request_id=request_id,
283 + type=RequestStatsUpdate.Type.FINISHED,
284 + monotonic_ts_s=finished_ts,
285 + finish_reason="test_reason",
286 + )
287 + stats.update_from(finished_update)
288 + assert stats.last_updated_ts_s == finished_ts
289 + assert stats.e2e_latency_s == finished_ts - arrived_ts
290 + assert stats.inference_latency_s == finished_ts - prefilling_ts
291 + assert stats.prefill_latency_s == detokenized_ts - prefilling_ts
292 + assert stats.decode_latency_s == finished_ts - detokenized_ts
293 + assert stats.first_token_latency_s == detokenized_ts - arrived_ts
294 + assert stats.queue_duration_s == prefilling_ts - queued_ts
295 + assert stats.is_finished
296 + assert stats.finish_reason == "test_reason"
297 +
298 + # TODO(rickyx): Add model forward/execute time.
299 + assert stats.model_forward_duration_s == 0.0
300 + assert stats.model_execute_duration_s == 0.0
vllm/v1/stats/__init__.py
vllm/v1/stats/common.py new
+449
@@ -0,0 +1,449 @@
1 +import time
2 +from dataclasses import dataclass
3 +from dataclasses import field as dataclass_field
4 +from enum import IntEnum
5 +from typing import ClassVar, Dict, List, Optional, Set
6 +
7 +import msgspec
8 +from msgspec import field as msgspec_field
9 +
10 +from vllm.sampling_params import SamplingParams
11 +
12 +
13 +class RequestStatsUpdate(msgspec.Struct,
14 + array_like=True,
15 + omit_defaults=True,
16 + gc=False):
17 + """
18 + An update to the request stats.
19 +
20 + This represents a stats update at a specific timestamp with metadata
21 + associated with the update.
22 +
23 + NOTE: since there might be multiple processes generating updates at
24 + different parts of the engine (e.g. input processor, scheduler, engine core,
25 + etc.), we use the monotonic timestamp to record the update to compute any
26 + intervals, and explicit wall-clock timestamp should be used for timestamps.
27 +
28 + WARNING: This assumes stats are generated in a single machine. If there are
29 + potentially multiple machines, one should always generate the stats updates
30 + on one single machine or use something else.
31 + """
32 +
33 + class Type(IntEnum):
34 + """See `RequestStats` for the lifecycle of a request."""
35 +
36 + # Request arrived at the engine frontend.
37 + ARRIVED = 0
38 + # Input processed by the input processor.
39 + INPUT_PROCESSED = 1
40 + # Queued on the engine core.
41 + QUEUED = 2
42 + # Scheduled running prefill by the scheduler.
43 + # A request could be running a new prefill on the prompt tokens or
44 + # a resumed prefill on the original prefill tokens + generated output
45 + # tokens before preemption.
46 + PREFILLING = 3
47 + # Preempted by the scheduler.
48 + PREEMPTED = 4
49 + # Output token is generated by the engine core.
50 + DECODING = 5
51 + # Token detokenized by the detokenizer.
52 + # We will record the timestamp for each output token, as well as the
53 + # finish reason.
54 + DETOKENIZED = 6
55 + # Request finishes (or aborts).
56 + FINISHED = 7
57 +
58 + """
59 + Valid state updates:
60 + ARRIVED
61 + │
62 + ├──────► INPUT_PROCESSED ──────► QUEUED ──────► PREFILLING ◄────┐
63 + │ │ │ │ │
64 + │ │ │ ▼ │
65 + │ │ │ -──► DECODING │
66 + │ │ │ | │ │
67 + │ │ │ | ▼ │
68 + │ │ │ └─ DETOKENIZED │
69 + │ │ │ │ │
70 + │ │ │ ▼ │
71 + │ ▼ ▼ PREEMPTED ◄──────┘
72 + │ │ │ │
73 + └──────────────┴───────────────────┴──────────────┴
74 + │
75 + ▼
76 + FINISHED (All could go to FINISHED)
77 + """
78 + _VALID_TRANSITIONS: ClassVar[Dict[Type, Set[Type]]] = {
79 + Type.ARRIVED: {
80 + Type.INPUT_PROCESSED,
81 + Type.FINISHED,
82 + },
83 + Type.INPUT_PROCESSED: {
84 + Type.QUEUED,
85 + Type.FINISHED,
86 + },
87 + Type.QUEUED: {
88 + Type.PREFILLING,
89 + Type.FINISHED,
90 + },
91 + Type.PREFILLING: {
92 + Type.DECODING,
93 + Type.PREEMPTED,
94 + Type.FINISHED,
95 + },
96 + Type.DECODING: {
97 + Type.DETOKENIZED,
98 + Type.FINISHED,
99 + },
100 + Type.DETOKENIZED: {
101 + Type.DECODING,
102 + Type.PREEMPTED,
103 + Type.FINISHED,
104 + },
105 + Type.PREEMPTED: {Type.PREFILLING, Type.FINISHED},
106 + Type.FINISHED: set(),
107 + }
108 +
109 + request_id: str
110 +
111 + type: Type
112 +
113 + # Timestamp when the update is recorded. This is used to record time
114 + # intervals between events rather than wall clock time.
115 + monotonic_ts_s: float = msgspec_field(
116 + default_factory=lambda: time.monotonic())
117 +
118 + ############################################################
119 + # Metadata associated with the update.
120 + ############################################################
121 + # For input_processed. Metadata needed for stats logging.
122 + num_prompt_tokens: Optional[int] = None
123 + sampling_params: Optional[SamplingParams] = None
124 +
125 + # For running.
126 + # Number of tokens computed when scheduled to run.
127 + num_computed_tokens: Optional[int] = None
128 + # Number of cached tokens when scheduled to run.
129 + num_cached_tokens: Optional[int] = None
130 +
131 + # For decoded.
132 + # The number of new output tokens generated.
133 + num_new_tokens: Optional[int] = None
134 +
135 + # For both detokenized and decoded.
136 + # Finished reason.
137 + finish_reason: Optional[str] = None
138 +
139 + # Non-optional fields for each update type.
140 + _REQUIRED_FIELDS: ClassVar[Dict[Type, List[str]]] = {
141 + Type.INPUT_PROCESSED: ["num_prompt_tokens", "sampling_params"],
142 + Type.PREFILLING: ["num_computed_tokens", "num_cached_tokens"],
143 + Type.DETOKENIZED: ["num_new_tokens"],
144 + Type.FINISHED: ["finish_reason"],
145 + }
146 +
147 + def __post_init__(self):
148 + required_fields = self._REQUIRED_FIELDS.get(self.type, [])
149 + for field in required_fields:
150 + if getattr(self, field) is None:
151 + raise ValueError(
152 + f"Field {field} is required for update type {self.type}.")
153 +
154 + @staticmethod
155 + def check_valid_update(
156 + update: "RequestStatsUpdate",
157 + last_update_type: Optional[Type],
158 + last_updated_ts_s: Optional[float],
159 + ):
160 + if last_update_type is None:
161 + assert update.type == RequestStatsUpdate.Type.ARRIVED
162 + else:
163 + valid_cur_update_types = RequestStatsUpdate._VALID_TRANSITIONS[
164 + last_update_type]
165 + assert update.type in valid_cur_update_types, (
166 + f"Invalid update type: {update.type} for last_update_type: "
167 + f"{last_update_type}.")
168 +
169 + if last_updated_ts_s is not None:
170 + assert update.monotonic_ts_s >= last_updated_ts_s, (
171 + "Update timestamp must be monotonically increasing, but "
172 + f"last_updated_ts_s={last_updated_ts_s} and "
173 + f"update.monotonic_ts_s={update.monotonic_ts_s}.")
174 +
175 +
176 +@dataclass
177 +class RequestStats:
178 + """Stats associated with a request (`Request`)."""
179 +
180 + ############################################################
181 + # Metadata
182 + ############################################################
183 + request_id: str
184 + sampling_params: Optional[SamplingParams] = None
185 + num_prompt_tokens: Optional[int] = None
186 +
187 + ############################################################
188 + # Metrics and Stats
189 + ############################################################
190 + # Timestamp when the request was last updated.
191 + last_updated_ts_s: Optional[float] = None
192 +
193 + # Last update stats type.
194 + last_update_type: Optional[RequestStatsUpdate.Type] = None
195 +
196 + # Timestamp when the request arrived at the llm engine.
197 + arrival_ts_s: Optional[float] = None
198 +
199 + # Number of tokens cached. When part of the request prefix is cached,
200 + # this will be set.
201 + num_cached_tokens: int = 0
202 +
203 + # Number of tokens computed.
204 + num_computed_tokens: int = 0
205 +
206 + # The timestamp when the request become waiting in the queue.
207 + queued_ts_s: Optional[float] = None
208 +
209 + # When the input processor is completed.
210 + input_processor_end_ts_s: Optional[float] = None
211 +
212 + # A sorted list of timestamps when the request was scheduled to prefill.
213 + # This could be when:
214 + # 1. the request is newly scheduled, so it's a new prefill.
215 + # 2. the request was preempted and resumed. It is equivalent to running
216 + # a prefill of the original prefill tokens + generated output tokens
217 + # before preemption.
218 + prefill_start_ts_s_lst: List[float] = dataclass_field(default_factory=list)
219 +
220 + # A list of timestamps when a token is decoded by the engine core.
221 + decoding_ts_s_lst: List[float] = dataclass_field(default_factory=list)
222 +
223 + # A sorted list of timestamps for each output token.
224 + output_token_ts_s_lst: List[float] = dataclass_field(default_factory=list)
225 +
226 + # First token's timestamp.
227 + first_token_ts_s: Optional[float] = None
228 +
229 + # TODO(rickyx): we need model runner to surface these.
230 + model_forward_duration_s: float = 0.0
231 + # Includes model forward, block/sync across workers, cpu-gpu sync time
232 + # and sampling time.
233 + model_execute_duration_s: float = 0.0
234 +
235 + # A sorted list of timestamps when the request was preempted at the
236 + # scheduler.
237 + # TODO(rickyx): right now, we don't actually have a good high-level
238 + # metric to measure the impact of preemption other than observation of
239 + # large P99 TPOT. Ideally we could quantify the impact of preemption by
240 + # measuring the number of tokens re-computed due to preemption.
241 + preempted_ts_s_lst: List[float] = dataclass_field(default_factory=list)
242 +
243 + # Timestamp when the request was finished at the engine core.
244 + finished_ts_s: Optional[float] = None
245 +
246 + # Finish reason.
247 + finish_reason: Optional[str] = None
248 +
249 + ############################################################
250 + # Derived properties.
251 + ############################################################
252 + @property
253 + def prefill_ts_s(self) -> Optional[float]:
254 + """The timestamp when the request started prefilling.
255 + Since a request could be preempted in decoding and later resumed
256 + to prefill the decoded tokens, we use the first prefill start timestamp.
257 + """
258 + return (self.prefill_start_ts_s_lst[0]
259 + if self.prefill_start_ts_s_lst else None)
260 +
261 + @property
262 + def e2e_latency_s(self) -> Optional[float]:
263 + if self.finished_ts_s is None or self.arrival_ts_s is None:
264 + return None
265 + assert self.finished_ts_s >= self.arrival_ts_s
266 + return self.finished_ts_s - self.arrival_ts_s
267 +
268 + @property
269 + def queue_duration_s(self) -> Optional[float]:
270 + """How long the request was waiting to run."""
271 + if self.queued_ts_s is None or self.prefill_ts_s is None:
272 + # Either not queued or not running yet.
273 + return None
274 + assert self.queued_ts_s <= self.prefill_ts_s
275 + return self.prefill_ts_s - self.queued_ts_s
276 +
277 + @property
278 + def inference_latency_s(self) -> Optional[float]:
279 + """How long the request was running inference
280 + (prefill and decode)."""
281 + if self.finished_ts_s is None or self.prefill_ts_s is None:
282 + return None
283 + assert self.finished_ts_s >= self.prefill_ts_s
284 + return self.finished_ts_s - self.prefill_ts_s
285 +
286 + @property
287 + def first_token_latency_s(self) -> Optional[float]:
288 + if self.first_token_ts_s is None or self.arrival_ts_s is None:
289 + return None
290 + assert self.first_token_ts_s >= self.arrival_ts_s
291 + return self.first_token_ts_s - self.arrival_ts_s
292 +
293 + @property
294 + def prefill_latency_s(self) -> Optional[float]:
295 + if self.first_token_ts_s is None or self.prefill_ts_s is None:
296 + return None
297 + assert self.first_token_ts_s >= self.prefill_ts_s
298 + return self.first_token_ts_s - self.prefill_ts_s
299 +
300 + @property
301 + def decode_latency_s(self) -> Optional[float]:
302 + if self.e2e_latency_s is None or self.first_token_latency_s is None:
303 + return None
304 + assert self.e2e_latency_s >= self.first_token_latency_s
305 + return self.e2e_latency_s - self.first_token_latency_s
306 +
307 + @property
308 + def output_token_latency_s_lst(self) -> List[float]:
309 + if len(self.output_token_ts_s_lst) == 0:
310 + return []
311 + latency_s_lst = []
312 + for i in range(1, len(self.output_token_ts_s_lst)):
313 + assert (self.output_token_ts_s_lst[i] >=
314 + self.output_token_ts_s_lst[i - 1])
315 + latency_s = (self.output_token_ts_s_lst[i] -
316 + self.output_token_ts_s_lst[i - 1])
317 + latency_s_lst.append(latency_s)
318 + return latency_s_lst
319 +
320 + @property
321 + def num_output_tokens(self) -> int:
322 + return len(self.output_token_ts_s_lst)
323 +
324 + @property
325 + def is_finished(self) -> bool:
326 + return self.finished_ts_s is not None
327 +
328 + def update_from(self, update: "RequestStatsUpdate"):
329 + RequestStatsUpdate.check_valid_update(update, self.last_update_type,
330 + self.last_updated_ts_s)
331 + ts = update.monotonic_ts_s
332 + self.last_updated_ts_s = ts
333 + self.last_update_type = update.type
334 + if update.type == RequestStatsUpdate.Type.ARRIVED:
335 + self.arrival_ts_s = ts
336 + elif update.type == RequestStatsUpdate.Type.INPUT_PROCESSED:
337 + self.input_processor_end_ts_s = ts
338 + self.sampling_params = update.sampling_params
339 + self.num_prompt_tokens = update.num_prompt_tokens
340 + elif update.type == RequestStatsUpdate.Type.QUEUED:
341 + self.queued_ts_s = ts
342 + elif update.type == RequestStatsUpdate.Type.PREFILLING:
343 + self.prefill_start_ts_s_lst.append(ts)
344 + self.num_cached_tokens = update.num_cached_tokens
345 + self.num_computed_tokens = update.num_computed_tokens
346 + elif update.type == RequestStatsUpdate.Type.PREEMPTED:
347 + self._reset_for_preemption(ts)
348 + elif update.type == RequestStatsUpdate.Type.DECODING:
349 + self.decoding_ts_s_lst.append(ts)
350 + elif update.type == RequestStatsUpdate.Type.DETOKENIZED:
351 + self._record_detokenized_output(
352 + ts,
353 + update.num_new_tokens,
354 + )
355 + elif update.type == RequestStatsUpdate.Type.FINISHED:
356 + self.finished_ts_s = ts
357 + self.finish_reason = update.finish_reason
358 + else:
359 + raise ValueError(f"Unknown update type: {update.type}")
360 +
361 + def _record_detokenized_output(
362 + self,
363 + ts_s: float,
364 + num_new_tokens: int,
365 + ):
366 + # Update if first output token is generated.
367 + if len(self.output_token_ts_s_lst) == 0:
368 + self.first_token_ts_s = ts_s
369 + assert (
370 + self.prefill_ts_s is not None
371 + ), "Request must be running before generating output tokens."
372 +
373 + # Some X new tokens were generated at the ts.
374 + self.output_token_ts_s_lst.extend([ts_s] * num_new_tokens)
375 +
376 + def _reset_for_preemption(self, ts_s: float):
377 + self.preempted_ts_s_lst.append(ts_s)
378 + # Reset the computed tokens since it might restart the prefill.
379 + self.num_computed_tokens = 0
380 + # Cached token count might also change when resumed.
381 + self.num_cached_tokens = 0
382 + # These stats don't change since they happen before request running.
383 + # - arrival_ts_s
384 + # - input_processor_end_ts_s
385 + # - sampling_params
386 + # - num_prompt_tokens
387 + # - first_token_ts_s
388 + #
389 + # These stats are accumulated over preemptions:
390 + # - output_token_ts_s_lst
391 + # - prefill_start_ts_s_lst (after preemption, it will prefill the
392 + # original prefill tokens and any output tokens generated before
393 + # preemption.)
394 +
395 +
396 +@dataclass
397 +class KVCacheStats:
398 + # KV Cache Usage in %
399 + gpu_cache_usage_sys: float = 0.0
400 + gpu_prefix_cache_hit_rate: float = 0.0
401 +
402 +
403 +@dataclass
404 +class SchedulerStats:
405 + """Stats associated with the scheduler."""
406 +
407 + # Number of requests currently running.
408 + num_running_reqs: int = 0
409 + # Number of requests currently waiting.
410 + num_waiting_reqs: int = 0
411 +
412 + kv_cache_stats: KVCacheStats = dataclass_field(
413 + default_factory=KVCacheStats)
414 +
415 +
416 +@dataclass
417 +class EngineCoreProcessStats:
418 + """Stats associated with the engine core process."""
419 +
420 + # Number of requests currently in the input queue. None if the engine core
421 + # is not running in multiprocess mode.
422 + input_queue_size: Optional[int] = None
423 + # Number of outputs currently in the output queue. None if the engine core
424 + # is not running in multiprocess mode.
425 + output_queue_size: Optional[int] = None
426 +
427 +
428 +class EngineCoreStatsSnapshot(msgspec.Struct,
429 + array_like=True,
430 + omit_defaults=True,
431 + gc=False):
432 + """
433 + A snapshot of the EngineCore's current stats over a period of time.
434 + """
435 +
436 + # Snapshot of the scheduler stats.
437 + scheduler_stats: SchedulerStats = msgspec_field(
438 + default_factory=SchedulerStats)
439 +
440 + # Per request stats updates.
441 + requests_stats_updates: List[RequestStatsUpdate] = msgspec_field(
442 + default_factory=list)
443 +
444 + # Engine core's queue stats.
445 + engine_core_process_stats: EngineCoreProcessStats = msgspec_field(
446 + default_factory=EngineCoreProcessStats)
447 +
448 + # TODO(rickyx): Add other components' stats,
449 + # e.g. model runner/worker and etc.