@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9597a095f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1][Core][1/n] Logging and Metrics (#11962)

Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>

Robert Shaw committed Jan 12, 2025 at 16:02 UTC 9597a095f2c02670b44f5973635ce4b9852e8eab
11 files changed +129 -84
tests/v1/engine/test_engine_core.py
+2 -2
@@ -80,7 +80,7 @@ def test_engine_core(monkeypatch):
80 assert len(engine_core.scheduler.running) == 4
81
82 # Loop through until they are all done.
83 - while len(engine_core.step()) > 0:
83 + while len(engine_core.step().outputs) > 0:
84 pass
85
86 assert len(engine_core.scheduler.waiting) == 0
@@ -170,7 +170,7 @@ def test_engine_core_advanced_sampling(monkeypatch):
170 assert len(engine_core.scheduler.waiting) == 1
171 assert len(engine_core.scheduler.running) == 0
172 # Loop through until they are all done.
173 - while len(engine_core.step()) > 0:
173 + while len(engine_core.step().outputs) > 0:
174 pass
175
176 assert len(engine_core.scheduler.waiting) == 0
tests/v1/engine/test_engine_core_client.py
+2 -2
@@ -43,7 +43,7 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
43 def loop_until_done(client: EngineCoreClient, outputs: Dict):
44
45 while True:
46 - engine_core_outputs = client.get_output()
46 + engine_core_outputs = client.get_output().outputs
47
48 if len(engine_core_outputs) == 0:
49 break
@@ -61,7 +61,7 @@ def loop_until_done(client: EngineCoreClient, outputs: Dict):
61 async def loop_until_done_async(client: EngineCoreClient, outputs: Dict):
62
63 while True:
64 - engine_core_outputs = await client.get_output_async()
64 + engine_core_outputs = await client.get_output_async().outputs
65
66 if len(engine_core_outputs) == 0:
67 break
vllm/v1/core/scheduler.py
+15 -5
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
8 from vllm.sampling_params import SamplingParams
9 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
10 from vllm.v1.core.kv_cache_manager import KVCacheManager
11 -from vllm.v1.engine import EngineCoreOutput
11 +from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
12 +from vllm.v1.metrics.stats import SchedulerStats
13 from vllm.v1.outputs import ModelRunnerOutput
14 from vllm.v1.request import Request, RequestStatus
15
@@ -394,12 +395,12 @@ class Scheduler:
395 self,
396 scheduler_output: "SchedulerOutput",
397 model_runner_output: "ModelRunnerOutput",
397 - ) -> List[EngineCoreOutput]:
398 + ) -> EngineCoreOutputs:
399 # NOTE(woosuk): This method doesn't consider speculative decoding.
400 sampled_token_ids = model_runner_output.sampled_token_ids
401 num_scheduled_tokens = scheduler_output.num_scheduled_tokens
402 new_running: List[Request] = []
402 - engine_core_outputs: List[EngineCoreOutput] = []
403 + outputs: List[EngineCoreOutput] = []
404 for request in self.running:
405 req_id = request.request_id
406 request.num_computed_tokens += num_scheduled_tokens[req_id]
@@ -438,7 +439,7 @@ class Scheduler:
439 finished=request.is_finished(),
440 finish_reason=request.get_finished_reason(),
441 stop_reason=request.stop_reason)
441 - engine_core_outputs.append(output)
442 + outputs.append(output)
443
444 # Breakout of the loop.
445 if stopped:
@@ -446,7 +447,10 @@ class Scheduler:
447
448 new_running.append(request)
449 self.running = new_running
449 - return engine_core_outputs
450 + return EngineCoreOutputs(
451 + outputs=outputs,
452 + scheduler_stats=self.make_stats(),
453 + )
454
455 def _check_stop(self, request: Request) -> bool:
456 if (request.num_tokens >= self.max_model_len
@@ -515,6 +519,12 @@ class Scheduler:
519 def has_unfinished_requests(self) -> bool:
520 return self.get_num_unfinished_requests() > 0
521
522 + def make_stats(self) -> SchedulerStats:
523 + return SchedulerStats(
524 + num_running_reqs=len(self.running),
525 + num_waiting_reqs=len(self.waiting),
526 + )
527 +
528
529 @dataclass
530 class NewRequestData:
vllm/v1/engine/__init__.py
+3
@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, List, Optional, Union
4
5 import msgspec
6
7 +from vllm.v1.metrics.stats import SchedulerStats
8 +
9 if TYPE_CHECKING:
10 from vllm.lora.request import LoRARequest
11 from vllm.multimodal import MultiModalKwargs
@@ -56,6 +58,7 @@ class EngineCoreOutputs(
58
59 # [num_reqs]
60 outputs: List[EngineCoreOutput]
61 + scheduler_stats: SchedulerStats
62
63
64 @dataclass
vllm/v1/engine/async_llm.py
+19 -7
@@ -4,7 +4,6 @@ from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
4
5 from vllm.config import ModelConfig, VllmConfig
6 from vllm.engine.arg_utils import AsyncEngineArgs
7 -from vllm.engine.metrics_types import StatLoggerBase
7 from vllm.engine.protocol import EngineClient
8 from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9 from vllm.inputs.preprocess import InputPreprocessor
@@ -22,6 +21,8 @@ from vllm.v1.engine.core_client import EngineCoreClient
21 from vllm.v1.engine.detokenizer import Detokenizer
22 from vllm.v1.engine.processor import Processor
23 from vllm.v1.executor.abstract import Executor
24 +from vllm.v1.metrics.loggers import LoggingStatLogger, StatLoggerBase
25 +from vllm.v1.metrics.stats import SchedulerStats
26
27 logger = init_logger(__name__)
28
@@ -34,7 +35,6 @@ class AsyncLLM(EngineClient):
35 executor_class: Type[Executor],
36 log_stats: bool,
37 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
37 - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
38 input_registry: InputRegistry = INPUT_REGISTRY,
39 use_cached_outputs: bool = False,
40 log_requests: bool = True,
@@ -45,7 +45,10 @@ class AsyncLLM(EngineClient):
45
46 self.log_requests = log_requests
47 self.log_stats = log_stats
48 - self.stat_loggers = stat_loggers
48 + self.stat_loggers: List[StatLoggerBase] = [
49 + LoggingStatLogger(),
50 + # TODO(rob): PrometheusStatLogger(),
51 + ]
52 self.model_config = vllm_config.model_config
53
54 # Tokenizer (+ ensure liveness if running in another process).
@@ -82,7 +85,6 @@ class AsyncLLM(EngineClient):
85 asyncio_mode=True,
86 vllm_config=vllm_config,
87 executor_class=executor_class,
85 - log_stats=self.log_stats,
88 )
89
90 self.output_handler: Optional[asyncio.Task] = None
@@ -94,7 +96,6 @@ class AsyncLLM(EngineClient):
96 engine_config: Optional[VllmConfig] = None,
97 start_engine_loop: bool = True,
98 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
97 - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
99 ) -> "AsyncLLM":
100 """Create an AsyncLLM from the EngineArgs."""
101
@@ -114,7 +115,6 @@ class AsyncLLM(EngineClient):
115 log_stats=not engine_args.disable_log_stats,
116 start_engine_loop=start_engine_loop,
117 usage_context=usage_context,
117 - stat_loggers=stat_loggers,
118 )
119
120 def shutdown(self):
@@ -254,7 +254,8 @@ class AsyncLLM(EngineClient):
254 outputs = await self.engine_core.get_output_async()
255
256 # 2) Detokenize based on the output.
257 - request_outputs, reqs_to_abort = self.detokenizer.step(outputs)
257 + request_outputs, reqs_to_abort = self.detokenizer.step(
258 + outputs.outputs)
259
260 # 3) Put the RequestOutputs into the per-request queues.
261 self._process_request_outputs(request_outputs)
@@ -262,6 +263,9 @@ class AsyncLLM(EngineClient):
263 # 4) Abort any requests that finished due to stop strings.
264 await self.engine_core.abort_requests_async(reqs_to_abort)
265
266 + # 5) Log any stats.
267 + await self._log_stats(scheduler_stats=outputs.scheduler_stats)
268 +
269 except Exception as e:
270 logger.exception("EngineCore output handler hit an error: %s", e)
271 kill_process_tree(os.getpid())
@@ -278,6 +282,14 @@ class AsyncLLM(EngineClient):
282 if request_id in self.rid_to_queue:
283 del self.rid_to_queue[request_id]
284
285 + async def _log_stats(self, scheduler_stats: SchedulerStats):
286 + """Log stats to the stat loggers."""
287 + if not self.log_stats:
288 + return
289 +
290 + for logger in self.stat_loggers:
291 + logger.log(scheduler_stats=scheduler_stats)
292 +
293 def encode(
294 self,
295 prompt: PromptType,
vllm/v1/engine/core.py
+16 -37
@@ -17,9 +17,9 @@ from vllm.transformers_utils.config import (
17 maybe_register_config_serialize_by_value)
18 from vllm.utils import get_exception_traceback, zmq_socket_ctx
19 from vllm.v1.core.scheduler import Scheduler
20 -from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
21 - EngineCoreProfile, EngineCoreRequest,
22 - EngineCoreRequestType, EngineCoreRequestUnion)
20 +from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
21 + EngineCoreRequest, EngineCoreRequestType,
22 + EngineCoreRequestUnion)
23 from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
24 from vllm.v1.executor.abstract import Executor
25 from vllm.v1.request import Request, RequestStatus
@@ -28,9 +28,7 @@ from vllm.version import __version__ as VLLM_VERSION
28
29 logger = init_logger(__name__)
30
31 -POLLING_TIMEOUT_MS = 5000
32 -POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
33 -LOGGING_TIME_S = 5
31 +POLLING_TIMEOUT_S = 2.5
32
33
34 class EngineCore:
@@ -40,10 +38,8 @@ class EngineCore:
38 self,
39 vllm_config: VllmConfig,
40 executor_class: Type[Executor],
43 - log_stats: bool = False,
41 ):
42 assert vllm_config.model_config.runner_type != "pooling"
46 - self.log_stats = log_stats
43
44 logger.info("Initializing an LLM engine (v%s) with config: %s",
45 VLLM_VERSION, vllm_config)
@@ -62,8 +58,6 @@ class EngineCore:
58 vllm_config.cache_config,
59 vllm_config.lora_config)
60
65 - self._last_logging_time = time.time()
66 -
61 self.mm_input_mapper_server = MMInputMapperServer(
62 vllm_config.model_config)
63
@@ -114,11 +108,12 @@ class EngineCore:
108 self.scheduler.finish_requests(request_ids,
109 RequestStatus.FINISHED_ABORTED)
110
117 - def step(self) -> List[EngineCoreOutput]:
111 + def step(self) -> EngineCoreOutputs:
112 """Schedule, execute, and make output."""
113
114 if not self.scheduler.has_unfinished_requests():
121 - return []
115 + return EngineCoreOutputs(
116 + outputs=[], scheduler_stats=self.scheduler.make_stats())
117
118 scheduler_output = self.scheduler.schedule()
119 output = self.model_executor.execute_model(scheduler_output)
@@ -145,7 +140,9 @@ class EngineCoreProc(EngineCore):
140 executor_class: Type[Executor],
141 log_stats: bool = False,
142 ):
148 - super().__init__(vllm_config, executor_class, log_stats)
143 + super().__init__(vllm_config, executor_class)
144 +
145 + self.log_stats = log_stats
146
147 # Background Threads and Queues for IO. These enable us to
148 # overlap ZMQ socket IO with GPU since they release the GIL,
@@ -153,7 +150,7 @@ class EngineCoreProc(EngineCore):
150 # model forward pass.
151 # Threads handle Socket <-> Queues and core_busy_loop uses Queue.
152 self.input_queue: queue.Queue[EngineCoreRequestUnion] = queue.Queue()
156 - self.output_queue: queue.Queue[List[EngineCoreOutput]] = queue.Queue()
153 + self.output_queue: queue.Queue[EngineCoreOutputs] = queue.Queue()
154 threading.Thread(target=self.process_input_socket,
155 args=(input_path, ),
156 daemon=True).start()
@@ -217,8 +214,10 @@ class EngineCoreProc(EngineCore):
214 self._handle_client_request(req)
215 break
216 except queue.Empty:
220 - self._log_stats()
217 logger.debug("EngineCore busy loop waiting.")
218 + # Break out the loop so we can log_stats in step().
219 + if self.log_stats:
220 + break
221 except BaseException:
222 raise
223
@@ -230,28 +229,9 @@ class EngineCoreProc(EngineCore):
229 # 3) Step the engine core.
230 outputs = self.step()
231
233 - # 4) Put EngineCoreOutputs into the output queue.
232 + # 5) Put EngineCoreOutputs into the output queue.
233 self.output_queue.put_nowait(outputs)
234
236 - self._log_stats()
237 -
238 - def _log_stats(self):
239 - """Log basic stats every LOGGING_TIME_S"""
240 -
241 - if not self.log_stats:
242 - return
243 -
244 - now = time.time()
245 -
246 - if now - self._last_logging_time > LOGGING_TIME_S:
247 - logger.info(
248 - "RUNNING: %s | WAITING: %s",
249 - len(self.scheduler.running),
250 - len(self.scheduler.waiting),
251 - )
252 -
253 - self._last_logging_time = now
254 -
235 def _handle_client_request(self, request: EngineCoreRequestUnion) -> None:
236 """Handle EngineCoreRequest or EngineCoreABORT from Client."""
237
@@ -301,7 +281,6 @@ class EngineCoreProc(EngineCore):
281
282 with zmq_socket_ctx(output_path, zmq.constants.PUSH) as socket:
283 while True:
304 - engine_core_outputs = self.output_queue.get()
305 - outputs = EngineCoreOutputs(outputs=engine_core_outputs)
284 + outputs = self.output_queue.get()
285 encoder.encode_into(outputs, buffer)
286 socket.send_multipart((buffer, ), copy=False)
vllm/v1/engine/core_client.py
+20 -28
@@ -12,9 +12,9 @@ from vllm.config import VllmConfig
12 from vllm.logger import init_logger
13 from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
14 make_zmq_socket)
15 -from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
16 - EngineCoreProfile, EngineCoreRequest,
17 - EngineCoreRequestType, EngineCoreRequestUnion)
15 +from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
16 + EngineCoreRequest, EngineCoreRequestType,
17 + EngineCoreRequestUnion)
18 from vllm.v1.engine.core import EngineCore, EngineCoreProc
19 from vllm.v1.executor.abstract import Executor
20 from vllm.v1.serial_utils import PickleEncoder
@@ -40,7 +40,6 @@ class EngineCoreClient(ABC):
40 asyncio_mode: bool,
41 vllm_config: VllmConfig,
42 executor_class: Type[Executor],
43 - log_stats: bool = False,
43 ) -> "EngineCoreClient":
44
45 # TODO: support this for debugging purposes.
@@ -50,18 +49,18 @@ class EngineCoreClient(ABC):
49 "is not currently supported.")
50
51 if multiprocess_mode and asyncio_mode:
53 - return AsyncMPClient(vllm_config, executor_class, log_stats)
52 + return AsyncMPClient(vllm_config, executor_class)
53
54 if multiprocess_mode and not asyncio_mode:
56 - return SyncMPClient(vllm_config, executor_class, log_stats)
55 + return SyncMPClient(vllm_config, executor_class)
56
58 - return InprocClient(vllm_config, executor_class, log_stats)
57 + return InprocClient(vllm_config, executor_class)
58
59 @abstractmethod
60 def shutdown(self):
61 ...
62
64 - def get_output(self) -> List[EngineCoreOutput]:
63 + def get_output(self) -> EngineCoreOutputs:
64 raise NotImplementedError
65
66 def add_request(self, request: EngineCoreRequest) -> None:
@@ -73,7 +72,7 @@ class EngineCoreClient(ABC):
72 def abort_requests(self, request_ids: List[str]) -> None:
73 raise NotImplementedError
74
76 - async def get_output_async(self) -> List[EngineCoreOutput]:
75 + async def get_output_async(self) -> EngineCoreOutputs:
76 raise NotImplementedError
77
78 async def add_request_async(self, request: EngineCoreRequest) -> None:
@@ -99,7 +98,7 @@ class InprocClient(EngineCoreClient):
98 def __init__(self, *args, **kwargs):
99 self.engine_core = EngineCore(*args, **kwargs)
100
102 - def get_output(self) -> List[EngineCoreOutput]:
101 + def get_output(self) -> EngineCoreOutputs:
102 return self.engine_core.step()
103
104 def add_request(self, request: EngineCoreRequest) -> None:
@@ -133,7 +132,7 @@ class MPClient(EngineCoreClient):
132 asyncio_mode: bool,
133 vllm_config: VllmConfig,
134 executor_class: Type[Executor],
136 - log_stats: bool = False,
135 + log_stats: bool,
136 ):
137 # The child processes will send SIGUSR1 when unrecoverable
138 # errors happen. We kill the process tree here so that the
@@ -194,22 +193,19 @@ class MPClient(EngineCoreClient):
193 class SyncMPClient(MPClient):
194 """Synchronous client for multi-proc EngineCore."""
195
197 - def __init__(self,
198 - vllm_config: VllmConfig,
199 - executor_class: Type[Executor],
200 - log_stats: bool = False):
196 + def __init__(self, vllm_config: VllmConfig,
197 + executor_class: Type[Executor]):
198 super().__init__(
199 asyncio_mode=False,
200 vllm_config=vllm_config,
201 executor_class=executor_class,
205 - log_stats=log_stats,
202 + log_stats=False,
203 )
204
208 - def get_output(self) -> List[EngineCoreOutput]:
205 + def get_output(self) -> EngineCoreOutputs:
206
207 (frame, ) = self.output_socket.recv_multipart(copy=False)
211 - engine_core_outputs = self.decoder.decode(frame.buffer).outputs
212 - return engine_core_outputs
208 + return self.decoder.decode(frame.buffer)
209
210 def _send_input(self, request_type: EngineCoreRequestType,
211 request: EngineCoreRequestUnion) -> None:
@@ -235,23 +231,19 @@ class SyncMPClient(MPClient):
231 class AsyncMPClient(MPClient):
232 """Asyncio-compatible client for multi-proc EngineCore."""
233
238 - def __init__(self,
239 - vllm_config: VllmConfig,
240 - executor_class: Type[Executor],
241 - log_stats: bool = False):
234 + def __init__(self, vllm_config: VllmConfig,
235 + executor_class: Type[Executor]):
236 super().__init__(
237 asyncio_mode=True,
238 vllm_config=vllm_config,
239 executor_class=executor_class,
246 - log_stats=log_stats,
240 + log_stats=True,
241 )
242
249 - async def get_output_async(self) -> List[EngineCoreOutput]:
243 + async def get_output_async(self) -> EngineCoreOutputs:
244
245 frames = await self.output_socket.recv_multipart(copy=False)
252 - engine_core_outputs = self.decoder.decode(frames[0].buffer).outputs
253 -
254 - return engine_core_outputs
246 + return self.decoder.decode(frames[0].buffer)
247
248 async def _send_input(self, request_type: EngineCoreRequestType,
249 request: EngineCoreRequestUnion) -> None:
vllm/v1/engine/llm_engine.py
+2 -3
@@ -74,7 +74,6 @@ class LLMEngine:
74 asyncio_mode=False,
75 vllm_config=vllm_config,
76 executor_class=executor_class,
77 - log_stats=False,
77 )
78
79 @classmethod
@@ -147,11 +146,11 @@ class LLMEngine:
146 def step(self) -> List[RequestOutput]:
147
148 # 1) Get EngineCoreOutput from the EngineCore.
150 - engine_core_outputs = self.engine_core.get_output()
149 + outputs = self.engine_core.get_output()
150
151 # 2) Detokenizer the EngineCoreOutput.
152 request_outputs, requests_to_abort = self.detokenizer.step(
154 - engine_core_outputs)
153 + outputs.outputs)
154
155 # 3) Abort requests that finished due to stopping criteria.
156 if requests_to_abort:
vllm/v1/metrics/__init__.py
vllm/v1/metrics/loggers.py new
+38
@@ -0,0 +1,38 @@
1 +import time
2 +from abc import ABC, abstractmethod
3 +
4 +from vllm.logger import init_logger
5 +from vllm.v1.metrics.stats import SchedulerStats
6 +
7 +logger = init_logger(__name__)
8 +
9 +_LOCAL_LOGGING_INTERVAL_SEC = 5.0
10 +
11 +
12 +class StatLoggerBase(ABC):
13 +
14 + @abstractmethod
15 + def log(self, scheduler_stats: SchedulerStats):
16 + ...
17 +
18 +
19 +class LoggingStatLogger(StatLoggerBase):
20 +
21 + def __init__(self):
22 + self.last_log_time = time.monotonic()
23 +
24 + def log(self, scheduler_stats: SchedulerStats):
25 + """Log Stats to standard output."""
26 +
27 + # Log every _LOCAL_LOGGING_INTERVAL_SEC.
28 + now = time.monotonic()
29 + if now - self.last_log_time < _LOCAL_LOGGING_INTERVAL_SEC:
30 + return
31 + self.last_log_time = now
32 +
33 + # Format and print output.
34 + logger.info(
35 + "Running: %d reqs, Waiting: %d reqs ",
36 + scheduler_stats.num_running_reqs,
37 + scheduler_stats.num_waiting_reqs,
38 + )
vllm/v1/metrics/stats.py new
+12
@@ -0,0 +1,12 @@
1 +from dataclasses import dataclass
2 +
3 +
4 +@dataclass
5 +class SchedulerStats:
6 + """Stats associated with the scheduler."""
7 +
8 + num_running_reqs: int = 0
9 + num_waiting_reqs: int = 0
10 +
11 + # gpu_cache_usage: float = 0.0
12 + # gpu_prefix_cache_hit_rate: float = 0.0