A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1][Core][1/n] Logging and Metrics (#11962)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Robert Shaw committed
Jan 12, 2025 at 16:02 UTC
9597a095f2c02670b44f5973635ce4b9852e8eab
11 files changed
+129
-84
tests/v1/engine/test_engine_core.py
+2
-2
@@ -80,7 +80,7 @@ def test_engine_core(monkeypatch):
80
assert len(engine_core.scheduler.running) == 4
81
82
# Loop through until they are all done.
83
- while len(engine_core.step()) > 0:
83
+ while len(engine_core.step().outputs) > 0:
84
pass
85
86
assert len(engine_core.scheduler.waiting) == 0
@@ -170,7 +170,7 @@ def test_engine_core_advanced_sampling(monkeypatch):
170
assert len(engine_core.scheduler.waiting) == 1
171
assert len(engine_core.scheduler.running) == 0
172
# Loop through until they are all done.
173
- while len(engine_core.step()) > 0:
173
+ while len(engine_core.step().outputs) > 0:
174
pass
175
176
assert len(engine_core.scheduler.waiting) == 0
tests/v1/engine/test_engine_core_client.py
+2
-2
@@ -43,7 +43,7 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
43
def loop_until_done(client: EngineCoreClient, outputs: Dict):
44
45
while True:
46
- engine_core_outputs = client.get_output()
46
+ engine_core_outputs = client.get_output().outputs
47
48
if len(engine_core_outputs) == 0:
49
break
@@ -61,7 +61,7 @@ def loop_until_done(client: EngineCoreClient, outputs: Dict):
61
async def loop_until_done_async(client: EngineCoreClient, outputs: Dict):
62
63
while True:
64
- engine_core_outputs = await client.get_output_async()
64
+ engine_core_outputs = await client.get_output_async().outputs
65
66
if len(engine_core_outputs) == 0:
67
break
vllm/v1/core/scheduler.py
+15
-5
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
8
from vllm.sampling_params import SamplingParams
9
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
10
from vllm.v1.core.kv_cache_manager import KVCacheManager
11
-from vllm.v1.engine import EngineCoreOutput
11
+from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
12
+from vllm.v1.metrics.stats import SchedulerStats
13
from vllm.v1.outputs import ModelRunnerOutput
14
from vllm.v1.request import Request, RequestStatus
15
@@ -394,12 +395,12 @@ class Scheduler:
395
self,
396
scheduler_output: "SchedulerOutput",
397
model_runner_output: "ModelRunnerOutput",
397
- ) -> List[EngineCoreOutput]:
398
+ ) -> EngineCoreOutputs:
399
# NOTE(woosuk): This method doesn't consider speculative decoding.
400
sampled_token_ids = model_runner_output.sampled_token_ids
401
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
402
new_running: List[Request] = []
402
- engine_core_outputs: List[EngineCoreOutput] = []
403
+ outputs: List[EngineCoreOutput] = []
404
for request in self.running:
405
req_id = request.request_id
406
request.num_computed_tokens += num_scheduled_tokens[req_id]
@@ -438,7 +439,7 @@ class Scheduler:
439
finished=request.is_finished(),
440
finish_reason=request.get_finished_reason(),
441
stop_reason=request.stop_reason)
441
- engine_core_outputs.append(output)
442
+ outputs.append(output)
443
444
# Breakout of the loop.
445
if stopped:
@@ -446,7 +447,10 @@ class Scheduler:
447
448
new_running.append(request)
449
self.running = new_running
449
- return engine_core_outputs
450
+ return EngineCoreOutputs(
451
+ outputs=outputs,
452
+ scheduler_stats=self.make_stats(),
453
+ )
454
455
def _check_stop(self, request: Request) -> bool:
456
if (request.num_tokens >= self.max_model_len
@@ -515,6 +519,12 @@ class Scheduler:
519
def has_unfinished_requests(self) -> bool:
520
return self.get_num_unfinished_requests() > 0
521
522
+ def make_stats(self) -> SchedulerStats:
523
+ return SchedulerStats(
524
+ num_running_reqs=len(self.running),
525
+ num_waiting_reqs=len(self.waiting),
526
+ )
527
+
528
529
@dataclass
530
class NewRequestData:
vllm/v1/engine/__init__.py
+3
@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, List, Optional, Union
4
5
import msgspec
6
7
+from vllm.v1.metrics.stats import SchedulerStats
8
+
9
if TYPE_CHECKING:
10
from vllm.lora.request import LoRARequest
11
from vllm.multimodal import MultiModalKwargs
@@ -56,6 +58,7 @@ class EngineCoreOutputs(
58
59
# [num_reqs]
60
outputs: List[EngineCoreOutput]
61
+ scheduler_stats: SchedulerStats
62
63
64
@dataclass
vllm/v1/engine/async_llm.py
+19
-7
@@ -4,7 +4,6 @@ from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
4
5
from vllm.config import ModelConfig, VllmConfig
6
from vllm.engine.arg_utils import AsyncEngineArgs
7
-from vllm.engine.metrics_types import StatLoggerBase
7
from vllm.engine.protocol import EngineClient
8
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9
from vllm.inputs.preprocess import InputPreprocessor
@@ -22,6 +21,8 @@ from vllm.v1.engine.core_client import EngineCoreClient
21
from vllm.v1.engine.detokenizer import Detokenizer
22
from vllm.v1.engine.processor import Processor
23
from vllm.v1.executor.abstract import Executor
24
+from vllm.v1.metrics.loggers import LoggingStatLogger, StatLoggerBase
25
+from vllm.v1.metrics.stats import SchedulerStats
26
27
logger = init_logger(__name__)
28
@@ -34,7 +35,6 @@ class AsyncLLM(EngineClient):
35
executor_class: Type[Executor],
36
log_stats: bool,
37
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
37
- stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
38
input_registry: InputRegistry = INPUT_REGISTRY,
39
use_cached_outputs: bool = False,
40
log_requests: bool = True,
@@ -45,7 +45,10 @@ class AsyncLLM(EngineClient):
45
46
self.log_requests = log_requests
47
self.log_stats = log_stats
48
- self.stat_loggers = stat_loggers
48
+ self.stat_loggers: List[StatLoggerBase] = [
49
+ LoggingStatLogger(),
50
+ # TODO(rob): PrometheusStatLogger(),
51
+ ]
52
self.model_config = vllm_config.model_config
53
54
# Tokenizer (+ ensure liveness if running in another process).
@@ -82,7 +85,6 @@ class AsyncLLM(EngineClient):
85
asyncio_mode=True,
86
vllm_config=vllm_config,
87
executor_class=executor_class,
85
- log_stats=self.log_stats,
88
)
89
90
self.output_handler: Optional[asyncio.Task] = None
@@ -94,7 +96,6 @@ class AsyncLLM(EngineClient):
96
engine_config: Optional[VllmConfig] = None,
97
start_engine_loop: bool = True,
98
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
97
- stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
99
) -> "AsyncLLM":
100
"""Create an AsyncLLM from the EngineArgs."""
101
@@ -114,7 +115,6 @@ class AsyncLLM(EngineClient):
115
log_stats=not engine_args.disable_log_stats,
116
start_engine_loop=start_engine_loop,
117
usage_context=usage_context,
117
- stat_loggers=stat_loggers,
118
)
119
120
def shutdown(self):
@@ -254,7 +254,8 @@ class AsyncLLM(EngineClient):
254
outputs = await self.engine_core.get_output_async()
255
256
# 2) Detokenize based on the output.
257
- request_outputs, reqs_to_abort = self.detokenizer.step(outputs)
257
+ request_outputs, reqs_to_abort = self.detokenizer.step(
258
+ outputs.outputs)
259
260
# 3) Put the RequestOutputs into the per-request queues.
261
self._process_request_outputs(request_outputs)
@@ -262,6 +263,9 @@ class AsyncLLM(EngineClient):
263
# 4) Abort any requests that finished due to stop strings.
264
await self.engine_core.abort_requests_async(reqs_to_abort)
265
266
+ # 5) Log any stats.
267
+ await self._log_stats(scheduler_stats=outputs.scheduler_stats)
268
+
269
except Exception as e:
270
logger.exception("EngineCore output handler hit an error: %s", e)
271
kill_process_tree(os.getpid())
@@ -278,6 +282,14 @@ class AsyncLLM(EngineClient):
282
if request_id in self.rid_to_queue:
283
del self.rid_to_queue[request_id]
284
285
+ async def _log_stats(self, scheduler_stats: SchedulerStats):
286
+ """Log stats to the stat loggers."""
287
+ if not self.log_stats:
288
+ return
289
+
290
+ for logger in self.stat_loggers:
291
+ logger.log(scheduler_stats=scheduler_stats)
292
+
293
def encode(
294
self,
295
prompt: PromptType,
vllm/v1/engine/core.py
+16
-37
@@ -17,9 +17,9 @@ from vllm.transformers_utils.config import (
17
maybe_register_config_serialize_by_value)
18
from vllm.utils import get_exception_traceback, zmq_socket_ctx
19
from vllm.v1.core.scheduler import Scheduler
20
-from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
21
- EngineCoreProfile, EngineCoreRequest,
22
- EngineCoreRequestType, EngineCoreRequestUnion)
20
+from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
21
+ EngineCoreRequest, EngineCoreRequestType,
22
+ EngineCoreRequestUnion)
23
from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
24
from vllm.v1.executor.abstract import Executor
25
from vllm.v1.request import Request, RequestStatus
@@ -28,9 +28,7 @@ from vllm.version import __version__ as VLLM_VERSION
28
29
logger = init_logger(__name__)
30
31
-POLLING_TIMEOUT_MS = 5000
32
-POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
33
-LOGGING_TIME_S = 5
31
+POLLING_TIMEOUT_S = 2.5
32
33
34
class EngineCore:
@@ -40,10 +38,8 @@ class EngineCore:
38
self,
39
vllm_config: VllmConfig,
40
executor_class: Type[Executor],
43
- log_stats: bool = False,
41
):
42
assert vllm_config.model_config.runner_type != "pooling"
46
- self.log_stats = log_stats
43
44
logger.info("Initializing an LLM engine (v%s) with config: %s",
45
VLLM_VERSION, vllm_config)
@@ -62,8 +58,6 @@ class EngineCore:
58
vllm_config.cache_config,
59
vllm_config.lora_config)
60
65
- self._last_logging_time = time.time()
66
-
61
self.mm_input_mapper_server = MMInputMapperServer(
62
vllm_config.model_config)
63
@@ -114,11 +108,12 @@ class EngineCore:
108
self.scheduler.finish_requests(request_ids,
109
RequestStatus.FINISHED_ABORTED)
110
117
- def step(self) -> List[EngineCoreOutput]:
111
+ def step(self) -> EngineCoreOutputs:
112
"""Schedule, execute, and make output."""
113
114
if not self.scheduler.has_unfinished_requests():
121
- return []
115
+ return EngineCoreOutputs(
116
+ outputs=[], scheduler_stats=self.scheduler.make_stats())
117
118
scheduler_output = self.scheduler.schedule()
119
output = self.model_executor.execute_model(scheduler_output)
@@ -145,7 +140,9 @@ class EngineCoreProc(EngineCore):
140
executor_class: Type[Executor],
141
log_stats: bool = False,
142
):
148
- super().__init__(vllm_config, executor_class, log_stats)
143
+ super().__init__(vllm_config, executor_class)
144
+
145
+ self.log_stats = log_stats
146
147
# Background Threads and Queues for IO. These enable us to
148
# overlap ZMQ socket IO with GPU since they release the GIL,
@@ -153,7 +150,7 @@ class EngineCoreProc(EngineCore):
150
# model forward pass.
151
# Threads handle Socket <-> Queues and core_busy_loop uses Queue.
152
self.input_queue: queue.Queue[EngineCoreRequestUnion] = queue.Queue()
156
- self.output_queue: queue.Queue[List[EngineCoreOutput]] = queue.Queue()
153
+ self.output_queue: queue.Queue[EngineCoreOutputs] = queue.Queue()
154
threading.Thread(target=self.process_input_socket,
155
args=(input_path, ),
156
daemon=True).start()
@@ -217,8 +214,10 @@ class EngineCoreProc(EngineCore):
214
self._handle_client_request(req)
215
break
216
except queue.Empty:
220
- self._log_stats()
217
logger.debug("EngineCore busy loop waiting.")
218
+ # Break out the loop so we can log_stats in step().
219
+ if self.log_stats:
220
+ break
221
except BaseException:
222
raise
223
@@ -230,28 +229,9 @@ class EngineCoreProc(EngineCore):
229
# 3) Step the engine core.
230
outputs = self.step()
231
233
- # 4) Put EngineCoreOutputs into the output queue.
232
+ # 5) Put EngineCoreOutputs into the output queue.
233
self.output_queue.put_nowait(outputs)
234
236
- self._log_stats()
237
-
238
- def _log_stats(self):
239
- """Log basic stats every LOGGING_TIME_S"""
240
-
241
- if not self.log_stats:
242
- return
243
-
244
- now = time.time()
245
-
246
- if now - self._last_logging_time > LOGGING_TIME_S:
247
- logger.info(
248
- "RUNNING: %s | WAITING: %s",
249
- len(self.scheduler.running),
250
- len(self.scheduler.waiting),
251
- )
252
-
253
- self._last_logging_time = now
254
-
235
def _handle_client_request(self, request: EngineCoreRequestUnion) -> None:
236
"""Handle EngineCoreRequest or EngineCoreABORT from Client."""
237
@@ -301,7 +281,6 @@ class EngineCoreProc(EngineCore):
281
282
with zmq_socket_ctx(output_path, zmq.constants.PUSH) as socket:
283
while True:
304
- engine_core_outputs = self.output_queue.get()
305
- outputs = EngineCoreOutputs(outputs=engine_core_outputs)
284
+ outputs = self.output_queue.get()
285
encoder.encode_into(outputs, buffer)
286
socket.send_multipart((buffer, ), copy=False)
vllm/v1/engine/core_client.py
+20
-28
@@ -12,9 +12,9 @@ from vllm.config import VllmConfig
12
from vllm.logger import init_logger
13
from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
14
make_zmq_socket)
15
-from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
16
- EngineCoreProfile, EngineCoreRequest,
17
- EngineCoreRequestType, EngineCoreRequestUnion)
15
+from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
16
+ EngineCoreRequest, EngineCoreRequestType,
17
+ EngineCoreRequestUnion)
18
from vllm.v1.engine.core import EngineCore, EngineCoreProc
19
from vllm.v1.executor.abstract import Executor
20
from vllm.v1.serial_utils import PickleEncoder
@@ -40,7 +40,6 @@ class EngineCoreClient(ABC):
40
asyncio_mode: bool,
41
vllm_config: VllmConfig,
42
executor_class: Type[Executor],
43
- log_stats: bool = False,
43
) -> "EngineCoreClient":
44
45
# TODO: support this for debugging purposes.
@@ -50,18 +49,18 @@ class EngineCoreClient(ABC):
49
"is not currently supported.")
50
51
if multiprocess_mode and asyncio_mode:
53
- return AsyncMPClient(vllm_config, executor_class, log_stats)
52
+ return AsyncMPClient(vllm_config, executor_class)
53
54
if multiprocess_mode and not asyncio_mode:
56
- return SyncMPClient(vllm_config, executor_class, log_stats)
55
+ return SyncMPClient(vllm_config, executor_class)
56
58
- return InprocClient(vllm_config, executor_class, log_stats)
57
+ return InprocClient(vllm_config, executor_class)
58
59
@abstractmethod
60
def shutdown(self):
61
...
62
64
- def get_output(self) -> List[EngineCoreOutput]:
63
+ def get_output(self) -> EngineCoreOutputs:
64
raise NotImplementedError
65
66
def add_request(self, request: EngineCoreRequest) -> None:
@@ -73,7 +72,7 @@ class EngineCoreClient(ABC):
72
def abort_requests(self, request_ids: List[str]) -> None:
73
raise NotImplementedError
74
76
- async def get_output_async(self) -> List[EngineCoreOutput]:
75
+ async def get_output_async(self) -> EngineCoreOutputs:
76
raise NotImplementedError
77
78
async def add_request_async(self, request: EngineCoreRequest) -> None:
@@ -99,7 +98,7 @@ class InprocClient(EngineCoreClient):
98
def __init__(self, *args, **kwargs):
99
self.engine_core = EngineCore(*args, **kwargs)
100
102
- def get_output(self) -> List[EngineCoreOutput]:
101
+ def get_output(self) -> EngineCoreOutputs:
102
return self.engine_core.step()
103
104
def add_request(self, request: EngineCoreRequest) -> None:
@@ -133,7 +132,7 @@ class MPClient(EngineCoreClient):
132
asyncio_mode: bool,
133
vllm_config: VllmConfig,
134
executor_class: Type[Executor],
136
- log_stats: bool = False,
135
+ log_stats: bool,
136
):
137
# The child processes will send SIGUSR1 when unrecoverable
138
# errors happen. We kill the process tree here so that the
@@ -194,22 +193,19 @@ class MPClient(EngineCoreClient):
193
class SyncMPClient(MPClient):
194
"""Synchronous client for multi-proc EngineCore."""
195
197
- def __init__(self,
198
- vllm_config: VllmConfig,
199
- executor_class: Type[Executor],
200
- log_stats: bool = False):
196
+ def __init__(self, vllm_config: VllmConfig,
197
+ executor_class: Type[Executor]):
198
super().__init__(
199
asyncio_mode=False,
200
vllm_config=vllm_config,
201
executor_class=executor_class,
205
- log_stats=log_stats,
202
+ log_stats=False,
203
)
204
208
- def get_output(self) -> List[EngineCoreOutput]:
205
+ def get_output(self) -> EngineCoreOutputs:
206
207
(frame, ) = self.output_socket.recv_multipart(copy=False)
211
- engine_core_outputs = self.decoder.decode(frame.buffer).outputs
212
- return engine_core_outputs
208
+ return self.decoder.decode(frame.buffer)
209
210
def _send_input(self, request_type: EngineCoreRequestType,
211
request: EngineCoreRequestUnion) -> None:
@@ -235,23 +231,19 @@ class SyncMPClient(MPClient):
231
class AsyncMPClient(MPClient):
232
"""Asyncio-compatible client for multi-proc EngineCore."""
233
238
- def __init__(self,
239
- vllm_config: VllmConfig,
240
- executor_class: Type[Executor],
241
- log_stats: bool = False):
234
+ def __init__(self, vllm_config: VllmConfig,
235
+ executor_class: Type[Executor]):
236
super().__init__(
237
asyncio_mode=True,
238
vllm_config=vllm_config,
239
executor_class=executor_class,
246
- log_stats=log_stats,
240
+ log_stats=True,
241
)
242
249
- async def get_output_async(self) -> List[EngineCoreOutput]:
243
+ async def get_output_async(self) -> EngineCoreOutputs:
244
245
frames = await self.output_socket.recv_multipart(copy=False)
252
- engine_core_outputs = self.decoder.decode(frames[0].buffer).outputs
253
-
254
- return engine_core_outputs
246
+ return self.decoder.decode(frames[0].buffer)
247
248
async def _send_input(self, request_type: EngineCoreRequestType,
249
request: EngineCoreRequestUnion) -> None:
vllm/v1/engine/llm_engine.py
+2
-3
@@ -74,7 +74,6 @@ class LLMEngine:
74
asyncio_mode=False,
75
vllm_config=vllm_config,
76
executor_class=executor_class,
77
- log_stats=False,
77
)
78
79
@classmethod
@@ -147,11 +146,11 @@ class LLMEngine:
146
def step(self) -> List[RequestOutput]:
147
148
# 1) Get EngineCoreOutput from the EngineCore.
150
- engine_core_outputs = self.engine_core.get_output()
149
+ outputs = self.engine_core.get_output()
150
151
# 2) Detokenizer the EngineCoreOutput.
152
request_outputs, requests_to_abort = self.detokenizer.step(
154
- engine_core_outputs)
153
+ outputs.outputs)
154
155
# 3) Abort requests that finished due to stopping criteria.
156
if requests_to_abort:
vllm/v1/metrics/__init__.py
vllm/v1/metrics/loggers.py
new
+38
@@ -0,0 +1,38 @@
1
+import time
2
+from abc import ABC, abstractmethod
3
+
4
+from vllm.logger import init_logger
5
+from vllm.v1.metrics.stats import SchedulerStats
6
+
7
+logger = init_logger(__name__)
8
+
9
+_LOCAL_LOGGING_INTERVAL_SEC = 5.0
10
+
11
+
12
+class StatLoggerBase(ABC):
13
+
14
+ @abstractmethod
15
+ def log(self, scheduler_stats: SchedulerStats):
16
+ ...
17
+
18
+
19
+class LoggingStatLogger(StatLoggerBase):
20
+
21
+ def __init__(self):
22
+ self.last_log_time = time.monotonic()
23
+
24
+ def log(self, scheduler_stats: SchedulerStats):
25
+ """Log Stats to standard output."""
26
+
27
+ # Log every _LOCAL_LOGGING_INTERVAL_SEC.
28
+ now = time.monotonic()
29
+ if now - self.last_log_time < _LOCAL_LOGGING_INTERVAL_SEC:
30
+ return
31
+ self.last_log_time = now
32
+
33
+ # Format and print output.
34
+ logger.info(
35
+ "Running: %d reqs, Waiting: %d reqs ",
36
+ scheduler_stats.num_running_reqs,
37
+ scheduler_stats.num_waiting_reqs,
38
+ )
vllm/v1/metrics/stats.py
new
+12
@@ -0,0 +1,12 @@
1
+from dataclasses import dataclass
2
+
3
+
4
+@dataclass
5
+class SchedulerStats:
6
+ """Stats associated with the scheduler."""
7
+
8
+ num_running_reqs: int = 0
9
+ num_waiting_reqs: int = 0
10
+
11
+ # gpu_cache_usage: float = 0.0
12
+ # gpu_prefix_cache_hit_rate: float = 0.0