A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Metrics] Add several request timing histograms (#12644)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Feb 11, 2025 at 15:14 UTC
75e6e145164c8e47a97b6e29654fe81b2fbc1ff5
16 files changed
+334
-84
tests/entrypoints/openai/test_metrics.py
+31
@@ -85,6 +85,10 @@ EXPECTED_VALUES = {
85
"vllm:time_per_output_token_seconds":
86
[("_count", _NUM_REQUESTS * (_NUM_GENERATION_TOKENS_PER_REQUEST - 1))],
87
"vllm:e2e_request_latency_seconds": [("_count", _NUM_REQUESTS)],
88
+ "vllm:request_queue_time_seconds": [("_count", _NUM_REQUESTS)],
89
+ "vllm:request_inference_time_seconds": [("_count", _NUM_REQUESTS)],
90
+ "vllm:request_prefill_time_seconds": [("_count", _NUM_REQUESTS)],
91
+ "vllm:request_decode_time_seconds": [("_count", _NUM_REQUESTS)],
92
"vllm:request_prompt_tokens":
93
[("_sum", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST),
94
("_count", _NUM_REQUESTS)],
@@ -169,6 +173,18 @@ EXPECTED_METRICS = [
173
"vllm:e2e_request_latency_seconds_sum",
174
"vllm:e2e_request_latency_seconds_bucket",
175
"vllm:e2e_request_latency_seconds_count",
176
+ "vllm:request_queue_time_seconds_sum",
177
+ "vllm:request_queue_time_seconds_bucket",
178
+ "vllm:request_queue_time_seconds_count",
179
+ "vllm:request_inference_time_seconds_sum",
180
+ "vllm:request_inference_time_seconds_bucket",
181
+ "vllm:request_inference_time_seconds_count",
182
+ "vllm:request_prefill_time_seconds_sum",
183
+ "vllm:request_prefill_time_seconds_bucket",
184
+ "vllm:request_prefill_time_seconds_count",
185
+ "vllm:request_decode_time_seconds_sum",
186
+ "vllm:request_decode_time_seconds_bucket",
187
+ "vllm:request_decode_time_seconds_count",
188
"vllm:request_prompt_tokens_sum",
189
"vllm:request_prompt_tokens_bucket",
190
"vllm:request_prompt_tokens_count",
@@ -220,6 +236,21 @@ EXPECTED_METRICS_V1 = [
236
"vllm:time_per_output_token_seconds_sum",
237
"vllm:time_per_output_token_seconds_bucket",
238
"vllm:time_per_output_token_seconds_count",
239
+ "vllm:e2e_request_latency_seconds_sum",
240
+ "vllm:e2e_request_latency_seconds_bucket",
241
+ "vllm:e2e_request_latency_seconds_count",
242
+ "vllm:request_queue_time_seconds_sum",
243
+ "vllm:request_queue_time_seconds_bucket",
244
+ "vllm:request_queue_time_seconds_count",
245
+ "vllm:request_inference_time_seconds_sum",
246
+ "vllm:request_inference_time_seconds_bucket",
247
+ "vllm:request_inference_time_seconds_count",
248
+ "vllm:request_prefill_time_seconds_sum",
249
+ "vllm:request_prefill_time_seconds_bucket",
250
+ "vllm:request_prefill_time_seconds_count",
251
+ "vllm:request_decode_time_seconds_sum",
252
+ "vllm:request_decode_time_seconds_bucket",
253
+ "vllm:request_decode_time_seconds_count",
254
]
255
256
tests/v1/core/test_scheduler.py
+2
-1
@@ -38,7 +38,8 @@ def create_scheduler(
38
return Scheduler(scheduler_config,
39
model_config,
40
cache_config,
41
- lora_config=None)
41
+ lora_config=None,
42
+ log_stats=True)
43
44
45
def create_requests(
tests/v1/engine/test_engine_core.py
+4
-2
@@ -50,7 +50,8 @@ def test_engine_core(monkeypatch):
50
executor_class = Executor.get_class(vllm_config)
51
52
engine_core = EngineCore(vllm_config=vllm_config,
53
- executor_class=executor_class)
53
+ executor_class=executor_class,
54
+ log_stats=True)
55
"""Test basic request lifecycle."""
56
57
# First request.
@@ -157,7 +158,8 @@ def test_engine_core_advanced_sampling(monkeypatch):
158
executor_class = Executor.get_class(vllm_config)
159
160
engine_core = EngineCore(vllm_config=vllm_config,
160
- executor_class=executor_class)
161
+ executor_class=executor_class,
162
+ log_stats=True)
163
"""Test basic request lifecycle."""
164
# First request.
165
request: EngineCoreRequest = make_request()
tests/v1/engine/test_engine_core_client.py
+2
@@ -94,6 +94,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
94
asyncio_mode=False,
95
vllm_config=vllm_config,
96
executor_class=executor_class,
97
+ log_stats=False,
98
)
99
100
MAX_TOKENS = 20
@@ -163,6 +164,7 @@ async def test_engine_core_client_asyncio(monkeypatch):
164
asyncio_mode=True,
165
vllm_config=vllm_config,
166
executor_class=executor_class,
167
+ log_stats=True,
168
)
169
170
MAX_TOKENS = 20
tests/v1/engine/test_output_processor.py
+15
-8
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import math
4
+import time
5
from typing import Dict, List, Optional
6
7
import pytest
@@ -15,6 +16,7 @@ from vllm.sequence import PromptLogprobs, SampleLogprobs
16
from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.v1.engine import EngineCoreRequest
18
from vllm.v1.engine.output_processor import OutputProcessor
19
+from vllm.v1.metrics.stats import IterationStats
20
21
22
def _ref_convert_id_to_token(
@@ -603,6 +605,7 @@ def test_iteration_stats(dummy_test_vectors):
605
output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
606
log_stats=True)
607
engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
608
+ engine_core_timestamp = time.monotonic()
609
610
# Make N requests.
611
requests = [
@@ -630,8 +633,9 @@ def test_iteration_stats(dummy_test_vectors):
633
634
# First iteration has 2 prefills.
635
outputs = engine_core.get_outputs()[:num_active]
633
- processed_outputs = output_processor.process_outputs(outputs)
634
- iteration_stats = processed_outputs.iteration_stats
636
+ iteration_stats = IterationStats()
637
+ output_processor.process_outputs(outputs, engine_core_timestamp,
638
+ iteration_stats)
639
total_prompt_tokens = sum([
640
len(prompt_tokens)
641
for prompt_tokens in dummy_test_vectors.prompt_tokens[:num_active]
@@ -642,8 +646,9 @@ def test_iteration_stats(dummy_test_vectors):
646
647
# Just decodes in this step.
648
outputs = engine_core.get_outputs()[:num_active]
645
- processed_outputs = output_processor.process_outputs(outputs)
646
- iteration_stats = processed_outputs.iteration_stats
649
+ iteration_stats = IterationStats()
650
+ output_processor.process_outputs(outputs, engine_core_timestamp,
651
+ iteration_stats)
652
653
assert iteration_stats.num_prompt_tokens == 0
654
assert iteration_stats.num_generation_tokens == num_active
@@ -652,8 +657,9 @@ def test_iteration_stats(dummy_test_vectors):
657
output_processor.add_request(inactive_request)
658
num_active += 1
659
outputs = engine_core.get_outputs()[:num_active]
655
- processed_outputs = output_processor.process_outputs(outputs)
656
- iteration_stats = processed_outputs.iteration_stats
660
+ iteration_stats = IterationStats()
661
+ output_processor.process_outputs(outputs, engine_core_timestamp,
662
+ iteration_stats)
663
total_prompt_tokens = len(dummy_test_vectors.prompt_tokens[num_active - 1])
664
665
assert iteration_stats.num_prompt_tokens == total_prompt_tokens
@@ -661,8 +667,9 @@ def test_iteration_stats(dummy_test_vectors):
667
668
# Just decodes in this step.
669
outputs = engine_core.get_outputs()[:num_active]
664
- processed_outputs = output_processor.process_outputs(outputs)
665
- iteration_stats = processed_outputs.iteration_stats
670
+ iteration_stats = IterationStats()
671
+ output_processor.process_outputs(outputs, engine_core_timestamp,
672
+ iteration_stats)
673
674
assert iteration_stats.num_prompt_tokens == 0
675
assert iteration_stats.num_generation_tokens == num_active
vllm/v1/core/kv_cache_manager.py
+3
@@ -26,6 +26,7 @@ class KVCacheManager:
26
sliding_window: Optional[int] = None,
27
enable_caching: bool = True,
28
num_preallocate_tokens: int = 64,
29
+ log_stats: bool = False,
30
) -> None:
31
self.block_size = block_size
32
self.num_gpu_blocks = num_gpu_blocks
@@ -33,6 +34,8 @@ class KVCacheManager:
34
self.max_num_blocks_per_req = cdiv(max_model_len, block_size)
35
self.sliding_window = sliding_window
36
self.enable_caching = enable_caching
37
+ # FIXME: make prefix cache stats conditional on log_stats
38
+ self.log_stats = log_stats
39
# NOTE(woosuk): To avoid frequent block allocation, we preallocate some
40
# blocks for each request. For example, when a request reaches the end
41
# of its block table, we preallocate N blocks in advance. This way, we
vllm/v1/core/scheduler.py
+29
-4
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import time
4
from collections import deque
5
from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
@@ -10,7 +11,8 @@ from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
11
from vllm.v1.core.kv_cache_manager import KVCacheManager
12
from vllm.v1.core.scheduler_output import (CachedRequestData, NewRequestData,
13
SchedulerOutput)
13
-from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
14
+from vllm.v1.engine import (EngineCoreEvent, EngineCoreEventType,
15
+ EngineCoreOutput, EngineCoreOutputs)
16
from vllm.v1.metrics.stats import SchedulerStats
17
from vllm.v1.outputs import ModelRunnerOutput
18
from vllm.v1.request import Request, RequestStatus
@@ -26,10 +28,12 @@ class Scheduler:
28
model_config: ModelConfig,
29
cache_config: CacheConfig,
30
lora_config: Optional[LoRAConfig],
31
+ log_stats: bool,
32
) -> None:
33
self.scheduler_config = scheduler_config
34
self.cache_config = cache_config
35
self.lora_config = lora_config
36
+ self.log_stats = log_stats
37
38
# Scheduling constraints.
39
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
@@ -45,7 +49,8 @@ class Scheduler:
49
num_gpu_blocks=num_gpu_blocks,
50
max_model_len=self.max_model_len,
51
sliding_window=self.cache_config.sliding_window,
48
- enable_caching=self.cache_config.enable_prefix_caching)
52
+ enable_caching=self.cache_config.enable_prefix_caching,
53
+ log_stats=self.log_stats)
54
self.block_size = self.cache_config.block_size
55
56
# req_id -> Request
@@ -107,6 +112,8 @@ class Scheduler:
112
scheduled_encoder_inputs: Dict[str, List[int]] = {}
113
encoder_budget = self.max_num_encoder_input_tokens
114
115
+ scheduled_timestamp = time.monotonic()
116
+
117
# First, schedule the RUNNING requests.
118
req_index = 0
119
while req_index < len(self.running) and token_budget > 0:
@@ -246,6 +253,7 @@ class Scheduler:
253
self.running.append(request)
254
if request.status == RequestStatus.WAITING:
255
scheduled_new_reqs.append(request)
256
+ self.request_scheduled(request, scheduled_timestamp)
257
elif request.status == RequestStatus.PREEMPTED:
258
scheduled_resumed_reqs.append(request)
259
else:
@@ -508,7 +516,8 @@ class Scheduler:
516
finish_reason=request.get_finished_reason(),
517
new_logprobs=new_logprobs,
518
new_prompt_logprobs_tensors=prompt_logprobs_tensors,
511
- stop_reason=request.stop_reason))
519
+ stop_reason=request.stop_reason,
520
+ events=request.take_events()))
521
522
if not stopped:
523
new_running.append(request)
@@ -541,6 +550,7 @@ class Scheduler:
550
def add_request(self, request: Request) -> None:
551
self.waiting.append(request)
552
self.requests[request.request_id] = request
553
+ self.request_queued(request)
554
555
def finish_requests(
556
self,
@@ -588,7 +598,22 @@ class Scheduler:
598
def reset_prefix_cache(self) -> bool:
599
return self.kv_cache_manager.reset_prefix_cache()
600
591
- def make_stats(self) -> SchedulerStats:
601
+ def request_queued(self, request: Request):
602
+ if not self.log_stats:
603
+ return
604
+ request.events.append(
605
+ EngineCoreEvent.new_event(EngineCoreEventType.QUEUED))
606
+
607
+ def request_scheduled(self, request: Request, timestamp: float):
608
+ if not self.log_stats:
609
+ return
610
+ request.events.append(
611
+ EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
612
+ timestamp))
613
+
614
+ def make_stats(self) -> Optional[SchedulerStats]:
615
+ if not self.log_stats:
616
+ return None
617
return SchedulerStats(
618
num_running_reqs=len(self.running),
619
num_waiting_reqs=len(self.waiting),
vllm/v1/engine/__init__.py
+32
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import enum
4
+import time
5
from typing import List, Optional, Union
6
7
import msgspec
@@ -60,6 +61,30 @@ class EngineCoreRequest(
61
lora_request: Optional[LoRARequest]
62
63
64
+class EngineCoreEventType(enum.IntEnum):
65
+ """The type of engine core request event."""
66
+ QUEUED = 1
67
+ SCHEDULED = 2
68
+
69
+
70
+class EngineCoreEvent(msgspec.Struct):
71
+ """A timestamped engine core event associated with a request.
72
+
73
+ The timestamp is a monotonic timestamps and is used for by the engine
74
+ frontend to calculate intervals between engine core events. These
75
+ timestamps should not be compared with timestamps from other processes.
76
+ """
77
+ type: EngineCoreEventType
78
+ timestamp: float
79
+
80
+ @classmethod
81
+ def new_event(cls,
82
+ event_type: EngineCoreEventType,
83
+ timestamp: Optional[float] = None) -> "EngineCoreEvent":
84
+ timestamp = time.monotonic() if timestamp is None else timestamp
85
+ return cls(event_type, timestamp)
86
+
87
+
88
class EngineCoreOutput(
89
msgspec.Struct,
90
array_like=True, # type: ignore[call-arg]
@@ -74,6 +99,7 @@ class EngineCoreOutput(
99
100
finish_reason: Optional[FinishReason] = None
101
stop_reason: Union[int, str, None] = None
102
+ events: Optional[List[EngineCoreEvent]] = None
103
104
@property
105
def finished(self) -> bool:
@@ -91,7 +117,12 @@ class EngineCoreOutputs(
117
118
# [num_reqs]
119
outputs: List[EngineCoreOutput]
94
- scheduler_stats: SchedulerStats
120
+ scheduler_stats: Optional[SchedulerStats]
121
+ timestamp: float = 0.0
122
+
123
+ def __post_init__(self):
124
+ if self.timestamp == 0.0:
125
+ self.timestamp = time.monotonic()
126
127
128
class EngineCoreRequestType(enum.Enum):
vllm/v1/engine/async_llm.py
+14
-10
@@ -53,10 +53,12 @@ class AsyncLLM(EngineClient):
53
54
self.log_requests = log_requests
55
self.log_stats = log_stats
56
- self.stat_loggers: List[StatLoggerBase] = [
57
- LoggingStatLogger(),
58
- PrometheusStatLogger(vllm_config.model_config),
59
- ]
56
+ self.stat_loggers: List[StatLoggerBase] = []
57
+ if self.log_stats:
58
+ self.stat_loggers.extend([
59
+ LoggingStatLogger(),
60
+ PrometheusStatLogger(vllm_config.model_config),
61
+ ])
62
63
# Tokenizer (+ ensure liveness if running in another process).
64
self.tokenizer = init_tokenizer_from_configs(
@@ -85,6 +87,7 @@ class AsyncLLM(EngineClient):
87
asyncio_mode=True,
88
vllm_config=vllm_config,
89
executor_class=executor_class,
90
+ log_stats=self.log_stats,
91
)
92
93
self.output_handler: Optional[asyncio.Task] = None
@@ -246,6 +249,8 @@ class AsyncLLM(EngineClient):
249
# 1) Pull EngineCoreOutputs from the EngineCore.
250
outputs = await self.engine_core.get_output_async()
251
252
+ iteration_stats = IterationStats() if self.log_stats else None
253
+
254
# Split outputs into chunks of at most
255
# VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
256
# event loop for too long.
@@ -257,14 +262,12 @@ class AsyncLLM(EngineClient):
262
outputs.outputs,
263
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
264
260
- iteration_stats = None
265
for i, outputs_slice in enumerate(slices):
266
# 2) Process EngineCoreOutputs.
267
processed_outputs = self.output_processor.process_outputs(
264
- outputs_slice, iteration_stats)
268
+ outputs_slice, outputs.timestamp, iteration_stats)
269
# NOTE: RequestOutputs are pushed to their queues.
270
assert not processed_outputs.request_outputs
267
- iteration_stats = processed_outputs.iteration_stats
271
272
# Allow other asyncio tasks to run between chunks
273
if i + 1 < len(slices):
@@ -277,7 +280,6 @@ class AsyncLLM(EngineClient):
280
# 4) Logging.
281
# TODO(rob): make into a coroutine and launch it in
282
# background thread once Prometheus overhead is non-trivial.
280
- assert iteration_stats is not None
283
self._log_stats(
284
scheduler_stats=outputs.scheduler_stats,
285
iteration_stats=iteration_stats,
@@ -299,12 +301,14 @@ class AsyncLLM(EngineClient):
301
302
def _log_stats(
303
self,
302
- scheduler_stats: SchedulerStats,
303
- iteration_stats: IterationStats,
304
+ scheduler_stats: Optional[SchedulerStats],
305
+ iteration_stats: Optional[IterationStats],
306
):
307
if not self.log_stats:
308
return
309
310
+ assert scheduler_stats is not None
311
+ assert iteration_stats is not None
312
for logger in self.stat_loggers:
313
logger.log(scheduler_stats=scheduler_stats,
314
iteration_stats=iteration_stats)
vllm/v1/engine/core.py
+6
-4
@@ -38,12 +38,15 @@ class EngineCore:
38
self,
39
vllm_config: VllmConfig,
40
executor_class: Type[Executor],
41
+ log_stats: bool,
42
):
43
assert vllm_config.model_config.runner_type != "pooling"
44
45
logger.info("Initializing a V1 LLM engine (v%s) with config: %s",
46
VLLM_VERSION, vllm_config)
47
48
+ self.log_stats = log_stats
49
+
50
# Setup Model.
51
self.model_executor = executor_class(vllm_config)
52
@@ -59,6 +62,7 @@ class EngineCore:
62
model_config=vllm_config.model_config,
63
cache_config=vllm_config.cache_config,
64
lora_config=vllm_config.lora_config,
65
+ log_stats=self.log_stats,
66
)
67
68
self.mm_input_mapper_server = MMInputMapperServer(
@@ -148,11 +152,9 @@ class EngineCoreProc(EngineCore):
152
ready_pipe: Connection,
153
vllm_config: VllmConfig,
154
executor_class: Type[Executor],
151
- log_stats: bool = False,
155
+ log_stats: bool,
156
):
153
- super().__init__(vllm_config, executor_class)
154
-
155
- self.log_stats = log_stats
157
+ super().__init__(vllm_config, executor_class, log_stats)
158
159
# Background Threads and Queues for IO. These enable us to
160
# overlap ZMQ socket IO with GPU since they release the GIL,
vllm/v1/engine/core_client.py
+10
-9
@@ -41,6 +41,7 @@ class EngineCoreClient(ABC):
41
asyncio_mode: bool,
42
vllm_config: VllmConfig,
43
executor_class: Type[Executor],
44
+ log_stats: bool,
45
) -> "EngineCoreClient":
46
47
# TODO: support this for debugging purposes.
@@ -50,12 +51,12 @@ class EngineCoreClient(ABC):
51
"is not currently supported.")
52
53
if multiprocess_mode and asyncio_mode:
53
- return AsyncMPClient(vllm_config, executor_class)
54
+ return AsyncMPClient(vllm_config, executor_class, log_stats)
55
56
if multiprocess_mode and not asyncio_mode:
56
- return SyncMPClient(vllm_config, executor_class)
57
+ return SyncMPClient(vllm_config, executor_class, log_stats)
58
58
- return InprocClient(vllm_config, executor_class)
59
+ return InprocClient(vllm_config, executor_class, log_stats)
60
61
@abstractmethod
62
def shutdown(self):
@@ -204,13 +205,13 @@ class MPClient(EngineCoreClient):
205
class SyncMPClient(MPClient):
206
"""Synchronous client for multi-proc EngineCore."""
207
207
- def __init__(self, vllm_config: VllmConfig,
208
- executor_class: Type[Executor]):
208
+ def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],
209
+ log_stats: bool):
210
super().__init__(
211
asyncio_mode=False,
212
vllm_config=vllm_config,
213
executor_class=executor_class,
213
- log_stats=False,
214
+ log_stats=log_stats,
215
)
216
217
def get_output(self) -> EngineCoreOutputs:
@@ -245,13 +246,13 @@ class SyncMPClient(MPClient):
246
class AsyncMPClient(MPClient):
247
"""Asyncio-compatible client for multi-proc EngineCore."""
248
248
- def __init__(self, vllm_config: VllmConfig,
249
- executor_class: Type[Executor]):
249
+ def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],
250
+ log_stats: bool):
251
super().__init__(
252
asyncio_mode=True,
253
vllm_config=vllm_config,
254
executor_class=executor_class,
254
- log_stats=True,
255
+ log_stats=log_stats,
256
)
257
258
self.outputs_queue: Optional[asyncio.Queue[bytes]] = None
vllm/v1/engine/llm_engine.py
+1
@@ -73,6 +73,7 @@ class LLMEngine:
73
asyncio_mode=False,
74
vllm_config=vllm_config,
75
executor_class=executor_class,
76
+ log_stats=False, # FIXME: implement
77
)
78
79
@classmethod
vllm/v1/engine/output_processor.py
+44
-15
@@ -19,7 +19,6 @@ class OutputProcessorOutput:
19
20
request_outputs: List[RequestOutput]
21
reqs_to_abort: List[str]
22
- iteration_stats: IterationStats
22
23
24
class RequestState:
@@ -34,6 +33,7 @@ class RequestState:
33
detokenizer: IncrementalDetokenizer,
34
arrival_time: float,
35
queue: Optional[asyncio.Queue[RequestOutput]],
36
+ log_stats: bool,
37
):
38
self.request_id = request_id
39
self.output_kind = output_kind
@@ -45,14 +45,16 @@ class RequestState:
45
self.is_prefilling = True
46
self.queue = queue
47
48
- self.stats = RequestStateStats(last_token_time=arrival_time)
48
+ self.stats = RequestStateStats(
49
+ arrival_time=arrival_time) if log_stats else None
50
51
@classmethod
52
def from_new_request(
53
cls,
54
tokenizer: AnyTokenizer,
55
request: EngineCoreRequest,
55
- queue: Optional[asyncio.Queue[RequestOutput]] = None,
56
+ queue: Optional[asyncio.Queue[RequestOutput]],
57
+ log_stats: bool,
58
) -> "RequestState":
59
return cls(
60
request_id=request.request_id,
@@ -69,6 +71,7 @@ class RequestState:
71
),
72
arrival_time=request.arrival_time,
73
queue=queue,
74
+ log_stats=log_stats,
75
)
76
77
@@ -112,11 +115,13 @@ class OutputProcessor:
115
self.request_states[request_id] = RequestState.from_new_request(
116
tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
117
request=request,
115
- queue=queue)
118
+ queue=queue,
119
+ log_stats=self.log_stats)
120
121
def process_outputs(
122
self,
123
engine_core_outputs: List[EngineCoreOutput],
124
+ engine_core_timestamp: Optional[float] = None,
125
iteration_stats: Optional[IterationStats] = None,
126
) -> OutputProcessorOutput:
127
"""
@@ -145,8 +150,6 @@ class OutputProcessor:
150
151
request_outputs: List[RequestOutput] = []
152
reqs_to_abort: List[str] = []
148
- if not iteration_stats:
149
- iteration_stats = IterationStats(self.log_stats)
153
for engine_core_output in engine_core_outputs:
154
req_id = engine_core_output.request_id
155
req_state = self.request_states.get(req_id)
@@ -155,10 +158,9 @@ class OutputProcessor:
158
continue
159
160
# 1) Compute stats for this iteration.
158
- iteration_stats.update_from_output(engine_core_output,
159
- req_state.is_prefilling,
160
- req_state.prompt_len,
161
- req_state.stats)
161
+ self._update_stats_from_output(req_state, engine_core_output,
162
+ engine_core_timestamp,
163
+ iteration_stats)
164
165
new_token_ids = engine_core_output.new_token_ids
166
finish_reason = engine_core_output.finish_reason
@@ -205,17 +207,44 @@ class OutputProcessor:
207
# detected stop string, abort needed in EngineCore.
208
reqs_to_abort.append(req_id)
209
208
- # Track per-request stats.
209
- assert finish_reason is not None
210
- iteration_stats.update_from_finished_request(
211
- finish_reason, request_output, req_state.stats)
210
+ # Track per-request stats
211
+ self._update_stats_from_finished(req_state, request_output,
212
+ finish_reason,
213
+ iteration_stats)
214
215
return OutputProcessorOutput(
216
request_outputs=request_outputs,
217
reqs_to_abort=reqs_to_abort,
216
- iteration_stats=iteration_stats,
218
)
219
220
+ def _update_stats_from_output(self, req_state: RequestState,
221
+ engine_core_output: EngineCoreOutput,
222
+ engine_core_timestamp: Optional[float],
223
+ iteration_stats: Optional[IterationStats]):
224
+ if iteration_stats is None:
225
+ return
226
+
227
+ assert engine_core_timestamp is not None
228
+ assert req_state.stats is not None
229
+ iteration_stats.update_from_output(engine_core_output,
230
+ engine_core_timestamp,
231
+ req_state.is_prefilling,
232
+ req_state.prompt_len,
233
+ req_state.stats)
234
+
235
+ def _update_stats_from_finished(self, req_state: RequestState,
236
+ request_output: RequestOutput,
237
+ finish_reason: Optional[FinishReason],
238
+ iteration_stats: Optional[IterationStats]):
239
+ if iteration_stats is None:
240
+ return
241
+
242
+ assert finish_reason is not None
243
+ assert req_state.stats is not None
244
+ iteration_stats.update_from_finished_request(finish_reason,
245
+ request_output,
246
+ req_state.stats)
247
+
248
@staticmethod
249
def _make_request_output(
250
request_state: RequestState,
vllm/v1/metrics/loggers.py
+49
@@ -182,6 +182,45 @@ class PrometheusStatLogger(StatLoggerBase):
182
],
183
labelnames=labelnames).labels(*labelvalues)
184
185
+ request_latency_buckets = [
186
+ 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
187
+ 40.0, 50.0, 60.0
188
+ ]
189
+ self.histogram_e2e_time_request = \
190
+ prometheus_client.Histogram(
191
+ name="vllm:e2e_request_latency_seconds",
192
+ documentation="Histogram of e2e request latency in seconds.",
193
+ buckets=request_latency_buckets,
194
+ labelnames=labelnames).labels(*labelvalues)
195
+ self.histogram_queue_time_request = \
196
+ prometheus_client.Histogram(
197
+ name="vllm:request_queue_time_seconds",
198
+ documentation=
199
+ "Histogram of time spent in WAITING phase for request.",
200
+ buckets=request_latency_buckets,
201
+ labelnames=labelnames).labels(*labelvalues)
202
+ self.histogram_inference_time_request = \
203
+ prometheus_client.Histogram(
204
+ name="vllm:request_inference_time_seconds",
205
+ documentation=
206
+ "Histogram of time spent in RUNNING phase for request.",
207
+ buckets=request_latency_buckets,
208
+ labelnames=labelnames).labels(*labelvalues)
209
+ self.histogram_prefill_time_request = \
210
+ prometheus_client.Histogram(
211
+ name="vllm:request_prefill_time_seconds",
212
+ documentation=
213
+ "Histogram of time spent in PREFILL phase for request.",
214
+ buckets=request_latency_buckets,
215
+ labelnames=labelnames).labels(*labelvalues)
216
+ self.histogram_decode_time_request = \
217
+ prometheus_client.Histogram(
218
+ name="vllm:request_decode_time_seconds",
219
+ documentation=
220
+ "Histogram of time spent in DECODE phase for request.",
221
+ buckets=request_latency_buckets,
222
+ labelnames=labelnames).labels(*labelvalues)
223
+
224
def log(self, scheduler_stats: SchedulerStats,
225
iteration_stats: IterationStats):
226
"""Log to prometheus."""
@@ -201,6 +240,12 @@ class PrometheusStatLogger(StatLoggerBase):
240
241
for finished_request in iteration_stats.finished_requests:
242
self.counter_request_success[finished_request.finish_reason].inc()
243
+ self.histogram_e2e_time_request.observe(
244
+ finished_request.e2e_latency)
245
+ self.histogram_inference_time_request.observe(
246
+ finished_request.inference_time)
247
+ self.histogram_decode_time_request.observe(
248
+ finished_request.decode_time)
249
self.histogram_num_prompt_tokens_request.observe(
250
finished_request.num_prompt_tokens)
251
self.histogram_num_generation_tokens_request.observe(
@@ -210,6 +255,10 @@ class PrometheusStatLogger(StatLoggerBase):
255
self.histogram_time_to_first_token.observe(ttft)
256
for tpot in iteration_stats.time_per_output_tokens_iter:
257
self.histogram_time_per_output_token.observe(tpot)
258
+ for queue_time in iteration_stats.queue_times_iter:
259
+ self.histogram_queue_time_request.observe(queue_time)
260
+ for prefill_time in iteration_stats.prefill_times_iter:
261
+ self.histogram_prefill_time_request.observe(prefill_time)
262
263
@staticmethod
264
def _unregister_vllm_metrics():
vllm/v1/metrics/stats.py
+74
-23
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List
6
7
if TYPE_CHECKING:
8
from vllm.outputs import RequestOutput
9
- from vllm.v1.engine import EngineCoreOutput, FinishReason
9
+ from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason
10
11
12
@dataclass
@@ -41,7 +41,15 @@ class RequestStateStats:
41
"""Stats that need to be tracked across delta updates."""
42
43
num_generation_tokens: int = 0
44
- last_token_time: float = 0.0
44
+
45
+ # This is a engine frontend timestamp (wall-clock)
46
+ arrival_time: float = 0.0
47
+
48
+ # These are engine core timestamps (monotonic)
49
+ queued_ts: float = 0.0
50
+ scheduled_ts: float = 0.0
51
+ first_token_ts: float = 0.0
52
+ last_token_ts: float = 0.0
53
54
55
@dataclass
@@ -49,33 +57,37 @@ class FinishedRequestStats:
57
"""Stats associated with a finished request."""
58
59
finish_reason: "FinishReason"
60
+ e2e_latency: float = 0.0
61
num_prompt_tokens: int = 0
62
num_generation_tokens: int = 0
63
+ inference_time: float = 0.0
64
+ decode_time: float = 0.0
65
66
67
class IterationStats:
68
"""Stats associated with a single set of EngineCoreOutputs."""
69
59
- def __init__(self, log_stats: bool):
60
- self.log_stats = log_stats
70
+ def __init__(self):
71
+ self.iteration_timestamp = time.time()
72
self.num_generation_tokens = 0
73
self.num_prompt_tokens = 0
74
self.finished_requests: List[FinishedRequestStats] = []
75
self.time_to_first_tokens_iter: List[float] = []
76
self.time_per_output_tokens_iter: List[float] = []
77
+ self.queue_times_iter: List[float] = []
78
+ self.prefill_times_iter: List[float] = []
79
67
- def update_from_output(self, output: "EngineCoreOutput",
68
- is_prefilling: bool, prompt_len: int,
69
- request_state_stats: RequestStateStats):
70
- if not self.log_stats:
71
- return
80
+ def _time_since(self, start: float) -> float:
81
+ """Calculate an interval relative to this iteration's timestamp."""
82
+ return self.iteration_timestamp - start
83
84
+ def update_from_output(self, output: "EngineCoreOutput",
85
+ engine_core_timestamp: float, is_prefilling: bool,
86
+ prompt_len: int, req_stats: RequestStateStats):
87
num_new_generation_tokens = len(output.new_token_ids)
74
- now = time.time()
75
- last_token_latency = now - request_state_stats.last_token_time
88
89
self.num_generation_tokens += num_new_generation_tokens
78
- if is_prefilling:
90
+ if is_prefilling and num_new_generation_tokens > 0:
91
# TODO(andy): we used to assert that num_new_generation_tokens
92
# > 0 with an invariant that EngineCore does not stream outputs
93
# for partially completed prefills (scheduler.update_from_output
@@ -84,19 +96,58 @@ class IterationStats:
96
# partially completed prompt.
97
# This will be reverted in a follow up PR and we should re-enable
98
# this assertion / invariant.
99
+ self.num_prompt_tokens += prompt_len
100
+
101
+ first_token_latency = self._time_since(req_stats.arrival_time)
102
+ self.time_to_first_tokens_iter.append(first_token_latency)
103
+
104
+ req_stats.num_generation_tokens += num_new_generation_tokens
105
+
106
+ # Process request-level engine core events
107
+ if output.events is not None:
108
+ self.update_from_events(output.events, is_prefilling, req_stats)
109
+
110
+ # Process the batch-level "new tokens" engine core event
111
+ if is_prefilling:
112
+ # TODO: re-enable no-output-for-partial-prefills invariant as above
113
if num_new_generation_tokens > 0:
88
- self.num_prompt_tokens += prompt_len
89
- self.time_to_first_tokens_iter.append(last_token_latency)
114
+ prefill_interval = \
115
+ engine_core_timestamp - req_stats.scheduled_ts
116
+ self.prefill_times_iter.append(prefill_interval)
117
+ req_stats.first_token_ts = engine_core_timestamp
118
else:
91
- self.time_per_output_tokens_iter.append(last_token_latency)
92
-
93
- request_state_stats.num_generation_tokens += num_new_generation_tokens
94
- request_state_stats.last_token_time = now
119
+ tpot = engine_core_timestamp - req_stats.last_token_ts
120
+ self.time_per_output_tokens_iter.append(tpot)
121
+
122
+ # TODO: re-enable no-output-for-partial-prefills invariant as above
123
+ if num_new_generation_tokens > 0:
124
+ req_stats.last_token_ts = engine_core_timestamp
125
+
126
+ def update_from_events(self, events: List["EngineCoreEvent"],
127
+ is_prefilling: bool, req_stats: RequestStateStats):
128
+ # Avoid circular dependency
129
+ from vllm.v1.engine import EngineCoreEventType
130
+ for event in events:
131
+ if event.type == EngineCoreEventType.QUEUED:
132
+ req_stats.queued_ts = event.timestamp
133
+ elif event.type == EngineCoreEventType.SCHEDULED:
134
+ queued_interval = event.timestamp - req_stats.queued_ts
135
+ self.queue_times_iter.append(queued_interval)
136
+ req_stats.scheduled_ts = event.timestamp
137
138
def update_from_finished_request(self, finish_reason: "FinishReason",
139
request_output: "RequestOutput",
98
- request_state_stats: RequestStateStats):
99
- self.finished_requests.append(
100
- FinishedRequestStats(finish_reason,
101
- len(request_output.prompt_token_ids),
102
- request_state_stats.num_generation_tokens))
140
+ req_stats: RequestStateStats):
141
+ e2e_latency = self._time_since(req_stats.arrival_time)
142
+
143
+ inference_time = req_stats.last_token_ts - req_stats.scheduled_ts
144
+ decode_time = req_stats.last_token_ts - req_stats.first_token_ts
145
+
146
+ finished_req = \
147
+ FinishedRequestStats(finish_reason=finish_reason,
148
+ e2e_latency=e2e_latency,
149
+ num_prompt_tokens=len(request_output.prompt_token_ids),
150
+ num_generation_tokens=req_stats.num_generation_tokens,
151
+ inference_time=inference_time,
152
+ decode_time=decode_time)
153
+ self.finished_requests.append(finished_req)
vllm/v1/request.py
+18
-7
@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, List, Optional, Union
5
6
from vllm.lora.request import LoRARequest
7
from vllm.sampling_params import SamplingParams
8
-from vllm.sequence import RequestMetrics
9
-from vllm.v1.engine import EngineCoreRequest, FinishReason
8
+from vllm.v1.engine import (EngineCoreEvent, EngineCoreEventType,
9
+ EngineCoreRequest, FinishReason)
10
from vllm.v1.utils import ConstantList
11
12
if TYPE_CHECKING:
@@ -33,14 +33,10 @@ class Request:
33
self.sampling_params = sampling_params
34
# Because of LoRA, the eos token id can be different for each request.
35
self.eos_token_id = eos_token_id
36
- self.metrics = RequestMetrics(arrival_time=arrival_time,
37
- last_token_time=arrival_time,
38
- first_scheduled_time=None,
39
- first_token_time=None,
40
- time_in_queue=None)
36
self.lora_request = lora_request
37
38
self.status = RequestStatus.WAITING
39
+ self.events: List[EngineCoreEvent] = []
40
self.stop_reason: Union[int, str, None] = None
41
assert sampling_params.max_tokens is not None
42
self.max_tokens = sampling_params.max_tokens
@@ -83,6 +79,21 @@ class Request:
79
lora_request=request.lora_request,
80
)
81
82
+ def queued(self, timestamp: Optional[float] = None) -> None:
83
+ self.events.append(
84
+ EngineCoreEvent.new_event(EngineCoreEventType.QUEUED, timestamp))
85
+
86
+ def scheduled(self, timestamp: Optional[float] = None) -> None:
87
+ self.events.append(
88
+ EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
89
+ timestamp))
90
+
91
+ def take_events(self) -> Optional[List[EngineCoreEvent]]:
92
+ if not self.events:
93
+ return None
94
+ events, self.events = self.events, []
95
+ return events
96
+
97
def append_output_token_ids(
98
self,
99
token_ids: Union[int, List[int]],