@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75e6e14516
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][Metrics] Add several request timing histograms (#12644)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Feb 11, 2025 at 15:14 UTC 75e6e145164c8e47a97b6e29654fe81b2fbc1ff5
16 files changed +334 -84
tests/entrypoints/openai/test_metrics.py
+31
@@ -85,6 +85,10 @@ EXPECTED_VALUES = {
85 "vllm:time_per_output_token_seconds":
86 [("_count", _NUM_REQUESTS * (_NUM_GENERATION_TOKENS_PER_REQUEST - 1))],
87 "vllm:e2e_request_latency_seconds": [("_count", _NUM_REQUESTS)],
88 + "vllm:request_queue_time_seconds": [("_count", _NUM_REQUESTS)],
89 + "vllm:request_inference_time_seconds": [("_count", _NUM_REQUESTS)],
90 + "vllm:request_prefill_time_seconds": [("_count", _NUM_REQUESTS)],
91 + "vllm:request_decode_time_seconds": [("_count", _NUM_REQUESTS)],
92 "vllm:request_prompt_tokens":
93 [("_sum", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST),
94 ("_count", _NUM_REQUESTS)],
@@ -169,6 +173,18 @@ EXPECTED_METRICS = [
173 "vllm:e2e_request_latency_seconds_sum",
174 "vllm:e2e_request_latency_seconds_bucket",
175 "vllm:e2e_request_latency_seconds_count",
176 + "vllm:request_queue_time_seconds_sum",
177 + "vllm:request_queue_time_seconds_bucket",
178 + "vllm:request_queue_time_seconds_count",
179 + "vllm:request_inference_time_seconds_sum",
180 + "vllm:request_inference_time_seconds_bucket",
181 + "vllm:request_inference_time_seconds_count",
182 + "vllm:request_prefill_time_seconds_sum",
183 + "vllm:request_prefill_time_seconds_bucket",
184 + "vllm:request_prefill_time_seconds_count",
185 + "vllm:request_decode_time_seconds_sum",
186 + "vllm:request_decode_time_seconds_bucket",
187 + "vllm:request_decode_time_seconds_count",
188 "vllm:request_prompt_tokens_sum",
189 "vllm:request_prompt_tokens_bucket",
190 "vllm:request_prompt_tokens_count",
@@ -220,6 +236,21 @@ EXPECTED_METRICS_V1 = [
236 "vllm:time_per_output_token_seconds_sum",
237 "vllm:time_per_output_token_seconds_bucket",
238 "vllm:time_per_output_token_seconds_count",
239 + "vllm:e2e_request_latency_seconds_sum",
240 + "vllm:e2e_request_latency_seconds_bucket",
241 + "vllm:e2e_request_latency_seconds_count",
242 + "vllm:request_queue_time_seconds_sum",
243 + "vllm:request_queue_time_seconds_bucket",
244 + "vllm:request_queue_time_seconds_count",
245 + "vllm:request_inference_time_seconds_sum",
246 + "vllm:request_inference_time_seconds_bucket",
247 + "vllm:request_inference_time_seconds_count",
248 + "vllm:request_prefill_time_seconds_sum",
249 + "vllm:request_prefill_time_seconds_bucket",
250 + "vllm:request_prefill_time_seconds_count",
251 + "vllm:request_decode_time_seconds_sum",
252 + "vllm:request_decode_time_seconds_bucket",
253 + "vllm:request_decode_time_seconds_count",
254 ]
255
256
tests/v1/core/test_scheduler.py
+2 -1
@@ -38,7 +38,8 @@ def create_scheduler(
38 return Scheduler(scheduler_config,
39 model_config,
40 cache_config,
41 - lora_config=None)
41 + lora_config=None,
42 + log_stats=True)
43
44
45 def create_requests(
tests/v1/engine/test_engine_core.py
+4 -2
@@ -50,7 +50,8 @@ def test_engine_core(monkeypatch):
50 executor_class = Executor.get_class(vllm_config)
51
52 engine_core = EngineCore(vllm_config=vllm_config,
53 - executor_class=executor_class)
53 + executor_class=executor_class,
54 + log_stats=True)
55 """Test basic request lifecycle."""
56
57 # First request.
@@ -157,7 +158,8 @@ def test_engine_core_advanced_sampling(monkeypatch):
158 executor_class = Executor.get_class(vllm_config)
159
160 engine_core = EngineCore(vllm_config=vllm_config,
160 - executor_class=executor_class)
161 + executor_class=executor_class,
162 + log_stats=True)
163 """Test basic request lifecycle."""
164 # First request.
165 request: EngineCoreRequest = make_request()
tests/v1/engine/test_engine_core_client.py
+2
@@ -94,6 +94,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
94 asyncio_mode=False,
95 vllm_config=vllm_config,
96 executor_class=executor_class,
97 + log_stats=False,
98 )
99
100 MAX_TOKENS = 20
@@ -163,6 +164,7 @@ async def test_engine_core_client_asyncio(monkeypatch):
164 asyncio_mode=True,
165 vllm_config=vllm_config,
166 executor_class=executor_class,
167 + log_stats=True,
168 )
169
170 MAX_TOKENS = 20
tests/v1/engine/test_output_processor.py
+15 -8
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 +import time
5 from typing import Dict, List, Optional
6
7 import pytest
@@ -15,6 +16,7 @@ from vllm.sequence import PromptLogprobs, SampleLogprobs
16 from vllm.transformers_utils.tokenizer import AnyTokenizer
17 from vllm.v1.engine import EngineCoreRequest
18 from vllm.v1.engine.output_processor import OutputProcessor
19 +from vllm.v1.metrics.stats import IterationStats
20
21
22 def _ref_convert_id_to_token(
@@ -603,6 +605,7 @@ def test_iteration_stats(dummy_test_vectors):
605 output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
606 log_stats=True)
607 engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
608 + engine_core_timestamp = time.monotonic()
609
610 # Make N requests.
611 requests = [
@@ -630,8 +633,9 @@ def test_iteration_stats(dummy_test_vectors):
633
634 # First iteration has 2 prefills.
635 outputs = engine_core.get_outputs()[:num_active]
633 - processed_outputs = output_processor.process_outputs(outputs)
634 - iteration_stats = processed_outputs.iteration_stats
636 + iteration_stats = IterationStats()
637 + output_processor.process_outputs(outputs, engine_core_timestamp,
638 + iteration_stats)
639 total_prompt_tokens = sum([
640 len(prompt_tokens)
641 for prompt_tokens in dummy_test_vectors.prompt_tokens[:num_active]
@@ -642,8 +646,9 @@ def test_iteration_stats(dummy_test_vectors):
646
647 # Just decodes in this step.
648 outputs = engine_core.get_outputs()[:num_active]
645 - processed_outputs = output_processor.process_outputs(outputs)
646 - iteration_stats = processed_outputs.iteration_stats
649 + iteration_stats = IterationStats()
650 + output_processor.process_outputs(outputs, engine_core_timestamp,
651 + iteration_stats)
652
653 assert iteration_stats.num_prompt_tokens == 0
654 assert iteration_stats.num_generation_tokens == num_active
@@ -652,8 +657,9 @@ def test_iteration_stats(dummy_test_vectors):
657 output_processor.add_request(inactive_request)
658 num_active += 1
659 outputs = engine_core.get_outputs()[:num_active]
655 - processed_outputs = output_processor.process_outputs(outputs)
656 - iteration_stats = processed_outputs.iteration_stats
660 + iteration_stats = IterationStats()
661 + output_processor.process_outputs(outputs, engine_core_timestamp,
662 + iteration_stats)
663 total_prompt_tokens = len(dummy_test_vectors.prompt_tokens[num_active - 1])
664
665 assert iteration_stats.num_prompt_tokens == total_prompt_tokens
@@ -661,8 +667,9 @@ def test_iteration_stats(dummy_test_vectors):
667
668 # Just decodes in this step.
669 outputs = engine_core.get_outputs()[:num_active]
664 - processed_outputs = output_processor.process_outputs(outputs)
665 - iteration_stats = processed_outputs.iteration_stats
670 + iteration_stats = IterationStats()
671 + output_processor.process_outputs(outputs, engine_core_timestamp,
672 + iteration_stats)
673
674 assert iteration_stats.num_prompt_tokens == 0
675 assert iteration_stats.num_generation_tokens == num_active
vllm/v1/core/kv_cache_manager.py
+3
@@ -26,6 +26,7 @@ class KVCacheManager:
26 sliding_window: Optional[int] = None,
27 enable_caching: bool = True,
28 num_preallocate_tokens: int = 64,
29 + log_stats: bool = False,
30 ) -> None:
31 self.block_size = block_size
32 self.num_gpu_blocks = num_gpu_blocks
@@ -33,6 +34,8 @@ class KVCacheManager:
34 self.max_num_blocks_per_req = cdiv(max_model_len, block_size)
35 self.sliding_window = sliding_window
36 self.enable_caching = enable_caching
37 + # FIXME: make prefix cache stats conditional on log_stats
38 + self.log_stats = log_stats
39 # NOTE(woosuk): To avoid frequent block allocation, we preallocate some
40 # blocks for each request. For example, when a request reaches the end
41 # of its block table, we preallocate N blocks in advance. This way, we
vllm/v1/core/scheduler.py
+29 -4
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +import time
4 from collections import deque
5 from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
@@ -10,7 +11,8 @@ from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
11 from vllm.v1.core.kv_cache_manager import KVCacheManager
12 from vllm.v1.core.scheduler_output import (CachedRequestData, NewRequestData,
13 SchedulerOutput)
13 -from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
14 +from vllm.v1.engine import (EngineCoreEvent, EngineCoreEventType,
15 + EngineCoreOutput, EngineCoreOutputs)
16 from vllm.v1.metrics.stats import SchedulerStats
17 from vllm.v1.outputs import ModelRunnerOutput
18 from vllm.v1.request import Request, RequestStatus
@@ -26,10 +28,12 @@ class Scheduler:
28 model_config: ModelConfig,
29 cache_config: CacheConfig,
30 lora_config: Optional[LoRAConfig],
31 + log_stats: bool,
32 ) -> None:
33 self.scheduler_config = scheduler_config
34 self.cache_config = cache_config
35 self.lora_config = lora_config
36 + self.log_stats = log_stats
37
38 # Scheduling constraints.
39 self.max_num_running_reqs = self.scheduler_config.max_num_seqs
@@ -45,7 +49,8 @@ class Scheduler:
49 num_gpu_blocks=num_gpu_blocks,
50 max_model_len=self.max_model_len,
51 sliding_window=self.cache_config.sliding_window,
48 - enable_caching=self.cache_config.enable_prefix_caching)
52 + enable_caching=self.cache_config.enable_prefix_caching,
53 + log_stats=self.log_stats)
54 self.block_size = self.cache_config.block_size
55
56 # req_id -> Request
@@ -107,6 +112,8 @@ class Scheduler:
112 scheduled_encoder_inputs: Dict[str, List[int]] = {}
113 encoder_budget = self.max_num_encoder_input_tokens
114
115 + scheduled_timestamp = time.monotonic()
116 +
117 # First, schedule the RUNNING requests.
118 req_index = 0
119 while req_index < len(self.running) and token_budget > 0:
@@ -246,6 +253,7 @@ class Scheduler:
253 self.running.append(request)
254 if request.status == RequestStatus.WAITING:
255 scheduled_new_reqs.append(request)
256 + self.request_scheduled(request, scheduled_timestamp)
257 elif request.status == RequestStatus.PREEMPTED:
258 scheduled_resumed_reqs.append(request)
259 else:
@@ -508,7 +516,8 @@ class Scheduler:
516 finish_reason=request.get_finished_reason(),
517 new_logprobs=new_logprobs,
518 new_prompt_logprobs_tensors=prompt_logprobs_tensors,
511 - stop_reason=request.stop_reason))
519 + stop_reason=request.stop_reason,
520 + events=request.take_events()))
521
522 if not stopped:
523 new_running.append(request)
@@ -541,6 +550,7 @@ class Scheduler:
550 def add_request(self, request: Request) -> None:
551 self.waiting.append(request)
552 self.requests[request.request_id] = request
553 + self.request_queued(request)
554
555 def finish_requests(
556 self,
@@ -588,7 +598,22 @@ class Scheduler:
598 def reset_prefix_cache(self) -> bool:
599 return self.kv_cache_manager.reset_prefix_cache()
600
591 - def make_stats(self) -> SchedulerStats:
601 + def request_queued(self, request: Request):
602 + if not self.log_stats:
603 + return
604 + request.events.append(
605 + EngineCoreEvent.new_event(EngineCoreEventType.QUEUED))
606 +
607 + def request_scheduled(self, request: Request, timestamp: float):
608 + if not self.log_stats:
609 + return
610 + request.events.append(
611 + EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
612 + timestamp))
613 +
614 + def make_stats(self) -> Optional[SchedulerStats]:
615 + if not self.log_stats:
616 + return None
617 return SchedulerStats(
618 num_running_reqs=len(self.running),
619 num_waiting_reqs=len(self.waiting),
vllm/v1/engine/__init__.py
+32 -1
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import enum
4 +import time
5 from typing import List, Optional, Union
6
7 import msgspec
@@ -60,6 +61,30 @@ class EngineCoreRequest(
61 lora_request: Optional[LoRARequest]
62
63
64 +class EngineCoreEventType(enum.IntEnum):
65 + """The type of engine core request event."""
66 + QUEUED = 1
67 + SCHEDULED = 2
68 +
69 +
70 +class EngineCoreEvent(msgspec.Struct):
71 + """A timestamped engine core event associated with a request.
72 +
73 + The timestamp is a monotonic timestamps and is used for by the engine
74 + frontend to calculate intervals between engine core events. These
75 + timestamps should not be compared with timestamps from other processes.
76 + """
77 + type: EngineCoreEventType
78 + timestamp: float
79 +
80 + @classmethod
81 + def new_event(cls,
82 + event_type: EngineCoreEventType,
83 + timestamp: Optional[float] = None) -> "EngineCoreEvent":
84 + timestamp = time.monotonic() if timestamp is None else timestamp
85 + return cls(event_type, timestamp)
86 +
87 +
88 class EngineCoreOutput(
89 msgspec.Struct,
90 array_like=True, # type: ignore[call-arg]
@@ -74,6 +99,7 @@ class EngineCoreOutput(
99
100 finish_reason: Optional[FinishReason] = None
101 stop_reason: Union[int, str, None] = None
102 + events: Optional[List[EngineCoreEvent]] = None
103
104 @property
105 def finished(self) -> bool:
@@ -91,7 +117,12 @@ class EngineCoreOutputs(
117
118 # [num_reqs]
119 outputs: List[EngineCoreOutput]
94 - scheduler_stats: SchedulerStats
120 + scheduler_stats: Optional[SchedulerStats]
121 + timestamp: float = 0.0
122 +
123 + def __post_init__(self):
124 + if self.timestamp == 0.0:
125 + self.timestamp = time.monotonic()
126
127
128 class EngineCoreRequestType(enum.Enum):
vllm/v1/engine/async_llm.py
+14 -10
@@ -53,10 +53,12 @@ class AsyncLLM(EngineClient):
53
54 self.log_requests = log_requests
55 self.log_stats = log_stats
56 - self.stat_loggers: List[StatLoggerBase] = [
57 - LoggingStatLogger(),
58 - PrometheusStatLogger(vllm_config.model_config),
59 - ]
56 + self.stat_loggers: List[StatLoggerBase] = []
57 + if self.log_stats:
58 + self.stat_loggers.extend([
59 + LoggingStatLogger(),
60 + PrometheusStatLogger(vllm_config.model_config),
61 + ])
62
63 # Tokenizer (+ ensure liveness if running in another process).
64 self.tokenizer = init_tokenizer_from_configs(
@@ -85,6 +87,7 @@ class AsyncLLM(EngineClient):
87 asyncio_mode=True,
88 vllm_config=vllm_config,
89 executor_class=executor_class,
90 + log_stats=self.log_stats,
91 )
92
93 self.output_handler: Optional[asyncio.Task] = None
@@ -246,6 +249,8 @@ class AsyncLLM(EngineClient):
249 # 1) Pull EngineCoreOutputs from the EngineCore.
250 outputs = await self.engine_core.get_output_async()
251
252 + iteration_stats = IterationStats() if self.log_stats else None
253 +
254 # Split outputs into chunks of at most
255 # VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
256 # event loop for too long.
@@ -257,14 +262,12 @@ class AsyncLLM(EngineClient):
262 outputs.outputs,
263 cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
264
260 - iteration_stats = None
265 for i, outputs_slice in enumerate(slices):
266 # 2) Process EngineCoreOutputs.
267 processed_outputs = self.output_processor.process_outputs(
264 - outputs_slice, iteration_stats)
268 + outputs_slice, outputs.timestamp, iteration_stats)
269 # NOTE: RequestOutputs are pushed to their queues.
270 assert not processed_outputs.request_outputs
267 - iteration_stats = processed_outputs.iteration_stats
271
272 # Allow other asyncio tasks to run between chunks
273 if i + 1 < len(slices):
@@ -277,7 +280,6 @@ class AsyncLLM(EngineClient):
280 # 4) Logging.
281 # TODO(rob): make into a coroutine and launch it in
282 # background thread once Prometheus overhead is non-trivial.
280 - assert iteration_stats is not None
283 self._log_stats(
284 scheduler_stats=outputs.scheduler_stats,
285 iteration_stats=iteration_stats,
@@ -299,12 +301,14 @@ class AsyncLLM(EngineClient):
301
302 def _log_stats(
303 self,
302 - scheduler_stats: SchedulerStats,
303 - iteration_stats: IterationStats,
304 + scheduler_stats: Optional[SchedulerStats],
305 + iteration_stats: Optional[IterationStats],
306 ):
307 if not self.log_stats:
308 return
309
310 + assert scheduler_stats is not None
311 + assert iteration_stats is not None
312 for logger in self.stat_loggers:
313 logger.log(scheduler_stats=scheduler_stats,
314 iteration_stats=iteration_stats)
vllm/v1/engine/core.py
+6 -4
@@ -38,12 +38,15 @@ class EngineCore:
38 self,
39 vllm_config: VllmConfig,
40 executor_class: Type[Executor],
41 + log_stats: bool,
42 ):
43 assert vllm_config.model_config.runner_type != "pooling"
44
45 logger.info("Initializing a V1 LLM engine (v%s) with config: %s",
46 VLLM_VERSION, vllm_config)
47
48 + self.log_stats = log_stats
49 +
50 # Setup Model.
51 self.model_executor = executor_class(vllm_config)
52
@@ -59,6 +62,7 @@ class EngineCore:
62 model_config=vllm_config.model_config,
63 cache_config=vllm_config.cache_config,
64 lora_config=vllm_config.lora_config,
65 + log_stats=self.log_stats,
66 )
67
68 self.mm_input_mapper_server = MMInputMapperServer(
@@ -148,11 +152,9 @@ class EngineCoreProc(EngineCore):
152 ready_pipe: Connection,
153 vllm_config: VllmConfig,
154 executor_class: Type[Executor],
151 - log_stats: bool = False,
155 + log_stats: bool,
156 ):
153 - super().__init__(vllm_config, executor_class)
154 -
155 - self.log_stats = log_stats
157 + super().__init__(vllm_config, executor_class, log_stats)
158
159 # Background Threads and Queues for IO. These enable us to
160 # overlap ZMQ socket IO with GPU since they release the GIL,
vllm/v1/engine/core_client.py
+10 -9
@@ -41,6 +41,7 @@ class EngineCoreClient(ABC):
41 asyncio_mode: bool,
42 vllm_config: VllmConfig,
43 executor_class: Type[Executor],
44 + log_stats: bool,
45 ) -> "EngineCoreClient":
46
47 # TODO: support this for debugging purposes.
@@ -50,12 +51,12 @@ class EngineCoreClient(ABC):
51 "is not currently supported.")
52
53 if multiprocess_mode and asyncio_mode:
53 - return AsyncMPClient(vllm_config, executor_class)
54 + return AsyncMPClient(vllm_config, executor_class, log_stats)
55
56 if multiprocess_mode and not asyncio_mode:
56 - return SyncMPClient(vllm_config, executor_class)
57 + return SyncMPClient(vllm_config, executor_class, log_stats)
58
58 - return InprocClient(vllm_config, executor_class)
59 + return InprocClient(vllm_config, executor_class, log_stats)
60
61 @abstractmethod
62 def shutdown(self):
@@ -204,13 +205,13 @@ class MPClient(EngineCoreClient):
205 class SyncMPClient(MPClient):
206 """Synchronous client for multi-proc EngineCore."""
207
207 - def __init__(self, vllm_config: VllmConfig,
208 - executor_class: Type[Executor]):
208 + def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],
209 + log_stats: bool):
210 super().__init__(
211 asyncio_mode=False,
212 vllm_config=vllm_config,
213 executor_class=executor_class,
213 - log_stats=False,
214 + log_stats=log_stats,
215 )
216
217 def get_output(self) -> EngineCoreOutputs:
@@ -245,13 +246,13 @@ class SyncMPClient(MPClient):
246 class AsyncMPClient(MPClient):
247 """Asyncio-compatible client for multi-proc EngineCore."""
248
248 - def __init__(self, vllm_config: VllmConfig,
249 - executor_class: Type[Executor]):
249 + def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],
250 + log_stats: bool):
251 super().__init__(
252 asyncio_mode=True,
253 vllm_config=vllm_config,
254 executor_class=executor_class,
254 - log_stats=True,
255 + log_stats=log_stats,
256 )
257
258 self.outputs_queue: Optional[asyncio.Queue[bytes]] = None
vllm/v1/engine/llm_engine.py
+1
@@ -73,6 +73,7 @@ class LLMEngine:
73 asyncio_mode=False,
74 vllm_config=vllm_config,
75 executor_class=executor_class,
76 + log_stats=False, # FIXME: implement
77 )
78
79 @classmethod
vllm/v1/engine/output_processor.py
+44 -15
@@ -19,7 +19,6 @@ class OutputProcessorOutput:
19
20 request_outputs: List[RequestOutput]
21 reqs_to_abort: List[str]
22 - iteration_stats: IterationStats
22
23
24 class RequestState:
@@ -34,6 +33,7 @@ class RequestState:
33 detokenizer: IncrementalDetokenizer,
34 arrival_time: float,
35 queue: Optional[asyncio.Queue[RequestOutput]],
36 + log_stats: bool,
37 ):
38 self.request_id = request_id
39 self.output_kind = output_kind
@@ -45,14 +45,16 @@ class RequestState:
45 self.is_prefilling = True
46 self.queue = queue
47
48 - self.stats = RequestStateStats(last_token_time=arrival_time)
48 + self.stats = RequestStateStats(
49 + arrival_time=arrival_time) if log_stats else None
50
51 @classmethod
52 def from_new_request(
53 cls,
54 tokenizer: AnyTokenizer,
55 request: EngineCoreRequest,
55 - queue: Optional[asyncio.Queue[RequestOutput]] = None,
56 + queue: Optional[asyncio.Queue[RequestOutput]],
57 + log_stats: bool,
58 ) -> "RequestState":
59 return cls(
60 request_id=request.request_id,
@@ -69,6 +71,7 @@ class RequestState:
71 ),
72 arrival_time=request.arrival_time,
73 queue=queue,
74 + log_stats=log_stats,
75 )
76
77
@@ -112,11 +115,13 @@ class OutputProcessor:
115 self.request_states[request_id] = RequestState.from_new_request(
116 tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
117 request=request,
115 - queue=queue)
118 + queue=queue,
119 + log_stats=self.log_stats)
120
121 def process_outputs(
122 self,
123 engine_core_outputs: List[EngineCoreOutput],
124 + engine_core_timestamp: Optional[float] = None,
125 iteration_stats: Optional[IterationStats] = None,
126 ) -> OutputProcessorOutput:
127 """
@@ -145,8 +150,6 @@ class OutputProcessor:
150
151 request_outputs: List[RequestOutput] = []
152 reqs_to_abort: List[str] = []
148 - if not iteration_stats:
149 - iteration_stats = IterationStats(self.log_stats)
153 for engine_core_output in engine_core_outputs:
154 req_id = engine_core_output.request_id
155 req_state = self.request_states.get(req_id)
@@ -155,10 +158,9 @@ class OutputProcessor:
158 continue
159
160 # 1) Compute stats for this iteration.
158 - iteration_stats.update_from_output(engine_core_output,
159 - req_state.is_prefilling,
160 - req_state.prompt_len,
161 - req_state.stats)
161 + self._update_stats_from_output(req_state, engine_core_output,
162 + engine_core_timestamp,
163 + iteration_stats)
164
165 new_token_ids = engine_core_output.new_token_ids
166 finish_reason = engine_core_output.finish_reason
@@ -205,17 +207,44 @@ class OutputProcessor:
207 # detected stop string, abort needed in EngineCore.
208 reqs_to_abort.append(req_id)
209
208 - # Track per-request stats.
209 - assert finish_reason is not None
210 - iteration_stats.update_from_finished_request(
211 - finish_reason, request_output, req_state.stats)
210 + # Track per-request stats
211 + self._update_stats_from_finished(req_state, request_output,
212 + finish_reason,
213 + iteration_stats)
214
215 return OutputProcessorOutput(
216 request_outputs=request_outputs,
217 reqs_to_abort=reqs_to_abort,
216 - iteration_stats=iteration_stats,
218 )
219
220 + def _update_stats_from_output(self, req_state: RequestState,
221 + engine_core_output: EngineCoreOutput,
222 + engine_core_timestamp: Optional[float],
223 + iteration_stats: Optional[IterationStats]):
224 + if iteration_stats is None:
225 + return
226 +
227 + assert engine_core_timestamp is not None
228 + assert req_state.stats is not None
229 + iteration_stats.update_from_output(engine_core_output,
230 + engine_core_timestamp,
231 + req_state.is_prefilling,
232 + req_state.prompt_len,
233 + req_state.stats)
234 +
235 + def _update_stats_from_finished(self, req_state: RequestState,
236 + request_output: RequestOutput,
237 + finish_reason: Optional[FinishReason],
238 + iteration_stats: Optional[IterationStats]):
239 + if iteration_stats is None:
240 + return
241 +
242 + assert finish_reason is not None
243 + assert req_state.stats is not None
244 + iteration_stats.update_from_finished_request(finish_reason,
245 + request_output,
246 + req_state.stats)
247 +
248 @staticmethod
249 def _make_request_output(
250 request_state: RequestState,
vllm/v1/metrics/loggers.py
+49
@@ -182,6 +182,45 @@ class PrometheusStatLogger(StatLoggerBase):
182 ],
183 labelnames=labelnames).labels(*labelvalues)
184
185 + request_latency_buckets = [
186 + 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
187 + 40.0, 50.0, 60.0
188 + ]
189 + self.histogram_e2e_time_request = \
190 + prometheus_client.Histogram(
191 + name="vllm:e2e_request_latency_seconds",
192 + documentation="Histogram of e2e request latency in seconds.",
193 + buckets=request_latency_buckets,
194 + labelnames=labelnames).labels(*labelvalues)
195 + self.histogram_queue_time_request = \
196 + prometheus_client.Histogram(
197 + name="vllm:request_queue_time_seconds",
198 + documentation=
199 + "Histogram of time spent in WAITING phase for request.",
200 + buckets=request_latency_buckets,
201 + labelnames=labelnames).labels(*labelvalues)
202 + self.histogram_inference_time_request = \
203 + prometheus_client.Histogram(
204 + name="vllm:request_inference_time_seconds",
205 + documentation=
206 + "Histogram of time spent in RUNNING phase for request.",
207 + buckets=request_latency_buckets,
208 + labelnames=labelnames).labels(*labelvalues)
209 + self.histogram_prefill_time_request = \
210 + prometheus_client.Histogram(
211 + name="vllm:request_prefill_time_seconds",
212 + documentation=
213 + "Histogram of time spent in PREFILL phase for request.",
214 + buckets=request_latency_buckets,
215 + labelnames=labelnames).labels(*labelvalues)
216 + self.histogram_decode_time_request = \
217 + prometheus_client.Histogram(
218 + name="vllm:request_decode_time_seconds",
219 + documentation=
220 + "Histogram of time spent in DECODE phase for request.",
221 + buckets=request_latency_buckets,
222 + labelnames=labelnames).labels(*labelvalues)
223 +
224 def log(self, scheduler_stats: SchedulerStats,
225 iteration_stats: IterationStats):
226 """Log to prometheus."""
@@ -201,6 +240,12 @@ class PrometheusStatLogger(StatLoggerBase):
240
241 for finished_request in iteration_stats.finished_requests:
242 self.counter_request_success[finished_request.finish_reason].inc()
243 + self.histogram_e2e_time_request.observe(
244 + finished_request.e2e_latency)
245 + self.histogram_inference_time_request.observe(
246 + finished_request.inference_time)
247 + self.histogram_decode_time_request.observe(
248 + finished_request.decode_time)
249 self.histogram_num_prompt_tokens_request.observe(
250 finished_request.num_prompt_tokens)
251 self.histogram_num_generation_tokens_request.observe(
@@ -210,6 +255,10 @@ class PrometheusStatLogger(StatLoggerBase):
255 self.histogram_time_to_first_token.observe(ttft)
256 for tpot in iteration_stats.time_per_output_tokens_iter:
257 self.histogram_time_per_output_token.observe(tpot)
258 + for queue_time in iteration_stats.queue_times_iter:
259 + self.histogram_queue_time_request.observe(queue_time)
260 + for prefill_time in iteration_stats.prefill_times_iter:
261 + self.histogram_prefill_time_request.observe(prefill_time)
262
263 @staticmethod
264 def _unregister_vllm_metrics():
vllm/v1/metrics/stats.py
+74 -23
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List
6
7 if TYPE_CHECKING:
8 from vllm.outputs import RequestOutput
9 - from vllm.v1.engine import EngineCoreOutput, FinishReason
9 + from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason
10
11
12 @dataclass
@@ -41,7 +41,15 @@ class RequestStateStats:
41 """Stats that need to be tracked across delta updates."""
42
43 num_generation_tokens: int = 0
44 - last_token_time: float = 0.0
44 +
45 + # This is a engine frontend timestamp (wall-clock)
46 + arrival_time: float = 0.0
47 +
48 + # These are engine core timestamps (monotonic)
49 + queued_ts: float = 0.0
50 + scheduled_ts: float = 0.0
51 + first_token_ts: float = 0.0
52 + last_token_ts: float = 0.0
53
54
55 @dataclass
@@ -49,33 +57,37 @@ class FinishedRequestStats:
57 """Stats associated with a finished request."""
58
59 finish_reason: "FinishReason"
60 + e2e_latency: float = 0.0
61 num_prompt_tokens: int = 0
62 num_generation_tokens: int = 0
63 + inference_time: float = 0.0
64 + decode_time: float = 0.0
65
66
67 class IterationStats:
68 """Stats associated with a single set of EngineCoreOutputs."""
69
59 - def __init__(self, log_stats: bool):
60 - self.log_stats = log_stats
70 + def __init__(self):
71 + self.iteration_timestamp = time.time()
72 self.num_generation_tokens = 0
73 self.num_prompt_tokens = 0
74 self.finished_requests: List[FinishedRequestStats] = []
75 self.time_to_first_tokens_iter: List[float] = []
76 self.time_per_output_tokens_iter: List[float] = []
77 + self.queue_times_iter: List[float] = []
78 + self.prefill_times_iter: List[float] = []
79
67 - def update_from_output(self, output: "EngineCoreOutput",
68 - is_prefilling: bool, prompt_len: int,
69 - request_state_stats: RequestStateStats):
70 - if not self.log_stats:
71 - return
80 + def _time_since(self, start: float) -> float:
81 + """Calculate an interval relative to this iteration's timestamp."""
82 + return self.iteration_timestamp - start
83
84 + def update_from_output(self, output: "EngineCoreOutput",
85 + engine_core_timestamp: float, is_prefilling: bool,
86 + prompt_len: int, req_stats: RequestStateStats):
87 num_new_generation_tokens = len(output.new_token_ids)
74 - now = time.time()
75 - last_token_latency = now - request_state_stats.last_token_time
88
89 self.num_generation_tokens += num_new_generation_tokens
78 - if is_prefilling:
90 + if is_prefilling and num_new_generation_tokens > 0:
91 # TODO(andy): we used to assert that num_new_generation_tokens
92 # > 0 with an invariant that EngineCore does not stream outputs
93 # for partially completed prefills (scheduler.update_from_output
@@ -84,19 +96,58 @@ class IterationStats:
96 # partially completed prompt.
97 # This will be reverted in a follow up PR and we should re-enable
98 # this assertion / invariant.
99 + self.num_prompt_tokens += prompt_len
100 +
101 + first_token_latency = self._time_since(req_stats.arrival_time)
102 + self.time_to_first_tokens_iter.append(first_token_latency)
103 +
104 + req_stats.num_generation_tokens += num_new_generation_tokens
105 +
106 + # Process request-level engine core events
107 + if output.events is not None:
108 + self.update_from_events(output.events, is_prefilling, req_stats)
109 +
110 + # Process the batch-level "new tokens" engine core event
111 + if is_prefilling:
112 + # TODO: re-enable no-output-for-partial-prefills invariant as above
113 if num_new_generation_tokens > 0:
88 - self.num_prompt_tokens += prompt_len
89 - self.time_to_first_tokens_iter.append(last_token_latency)
114 + prefill_interval = \
115 + engine_core_timestamp - req_stats.scheduled_ts
116 + self.prefill_times_iter.append(prefill_interval)
117 + req_stats.first_token_ts = engine_core_timestamp
118 else:
91 - self.time_per_output_tokens_iter.append(last_token_latency)
92 -
93 - request_state_stats.num_generation_tokens += num_new_generation_tokens
94 - request_state_stats.last_token_time = now
119 + tpot = engine_core_timestamp - req_stats.last_token_ts
120 + self.time_per_output_tokens_iter.append(tpot)
121 +
122 + # TODO: re-enable no-output-for-partial-prefills invariant as above
123 + if num_new_generation_tokens > 0:
124 + req_stats.last_token_ts = engine_core_timestamp
125 +
126 + def update_from_events(self, events: List["EngineCoreEvent"],
127 + is_prefilling: bool, req_stats: RequestStateStats):
128 + # Avoid circular dependency
129 + from vllm.v1.engine import EngineCoreEventType
130 + for event in events:
131 + if event.type == EngineCoreEventType.QUEUED:
132 + req_stats.queued_ts = event.timestamp
133 + elif event.type == EngineCoreEventType.SCHEDULED:
134 + queued_interval = event.timestamp - req_stats.queued_ts
135 + self.queue_times_iter.append(queued_interval)
136 + req_stats.scheduled_ts = event.timestamp
137
138 def update_from_finished_request(self, finish_reason: "FinishReason",
139 request_output: "RequestOutput",
98 - request_state_stats: RequestStateStats):
99 - self.finished_requests.append(
100 - FinishedRequestStats(finish_reason,
101 - len(request_output.prompt_token_ids),
102 - request_state_stats.num_generation_tokens))
140 + req_stats: RequestStateStats):
141 + e2e_latency = self._time_since(req_stats.arrival_time)
142 +
143 + inference_time = req_stats.last_token_ts - req_stats.scheduled_ts
144 + decode_time = req_stats.last_token_ts - req_stats.first_token_ts
145 +
146 + finished_req = \
147 + FinishedRequestStats(finish_reason=finish_reason,
148 + e2e_latency=e2e_latency,
149 + num_prompt_tokens=len(request_output.prompt_token_ids),
150 + num_generation_tokens=req_stats.num_generation_tokens,
151 + inference_time=inference_time,
152 + decode_time=decode_time)
153 + self.finished_requests.append(finished_req)
vllm/v1/request.py
+18 -7
@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, List, Optional, Union
5
6 from vllm.lora.request import LoRARequest
7 from vllm.sampling_params import SamplingParams
8 -from vllm.sequence import RequestMetrics
9 -from vllm.v1.engine import EngineCoreRequest, FinishReason
8 +from vllm.v1.engine import (EngineCoreEvent, EngineCoreEventType,
9 + EngineCoreRequest, FinishReason)
10 from vllm.v1.utils import ConstantList
11
12 if TYPE_CHECKING:
@@ -33,14 +33,10 @@ class Request:
33 self.sampling_params = sampling_params
34 # Because of LoRA, the eos token id can be different for each request.
35 self.eos_token_id = eos_token_id
36 - self.metrics = RequestMetrics(arrival_time=arrival_time,
37 - last_token_time=arrival_time,
38 - first_scheduled_time=None,
39 - first_token_time=None,
40 - time_in_queue=None)
36 self.lora_request = lora_request
37
38 self.status = RequestStatus.WAITING
39 + self.events: List[EngineCoreEvent] = []
40 self.stop_reason: Union[int, str, None] = None
41 assert sampling_params.max_tokens is not None
42 self.max_tokens = sampling_params.max_tokens
@@ -83,6 +79,21 @@ class Request:
79 lora_request=request.lora_request,
80 )
81
82 + def queued(self, timestamp: Optional[float] = None) -> None:
83 + self.events.append(
84 + EngineCoreEvent.new_event(EngineCoreEventType.QUEUED, timestamp))
85 +
86 + def scheduled(self, timestamp: Optional[float] = None) -> None:
87 + self.events.append(
88 + EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
89 + timestamp))
90 +
91 + def take_events(self) -> Optional[List[EngineCoreEvent]]:
92 + if not self.events:
93 + return None
94 + events, self.events = self.events, []
95 + return events
96 +
97 def append_output_token_ids(
98 self,
99 token_ids: Union[int, List[int]],