A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] EngineCore supports profiling (#10564)
Signed-off-by: Abatom <abzhonghua@gmail.com>
Zhonghua Deng committed
Nov 23, 2024 at 09:16 UTC
d345f409b7478c0e547b238916ec9e90b6156bbc
5 files changed
+68
-9
vllm/v1/engine/__init__.py
+6
@@ -68,6 +68,11 @@ class EngineCoreOutputs(msgspec.Struct,
68
outputs: List[EngineCoreOutput]
69
70
71
+@dataclass
72
+class EngineCoreProfile:
73
+ is_start: bool
74
+
75
+
76
class EngineCoreRequestType(enum.Enum):
77
"""
78
Request types defined as hex byte strings, so it can be sent over sockets
@@ -75,3 +80,4 @@ class EngineCoreRequestType(enum.Enum):
80
"""
81
ADD = b'\x00'
82
ABORT = b'\x01'
83
+ PROFILE = b'\x02'
vllm/v1/engine/async_llm.py
+2
-2
@@ -346,10 +346,10 @@ class AsyncLLM(EngineClient):
346
logger.debug("Called check_health.")
347
348
async def start_profile(self) -> None:
349
- raise ValueError("Not supported on V1 yet.")
349
+ await self.engine_core.profile(True)
350
351
async def stop_profile(self) -> None:
352
- raise ValueError("Not supported on V1 yet.")
352
+ await self.engine_core.profile(False)
353
354
@property
355
def is_running(self) -> bool:
vllm/v1/engine/core.py
+12
-2
@@ -1,4 +1,5 @@
1
import multiprocessing
2
+import pickle
3
import queue
4
import threading
5
import time
@@ -16,7 +17,8 @@ from vllm.logger import init_logger
17
from vllm.usage.usage_lib import UsageContext
18
from vllm.v1.core.scheduler import Scheduler
19
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19
- EngineCoreRequest, EngineCoreRequestType)
20
+ EngineCoreProfile, EngineCoreRequest,
21
+ EngineCoreRequestType)
22
from vllm.v1.engine.mm_input_mapper import MMInputMapper
23
from vllm.v1.executor.gpu_executor import GPUExecutor
24
from vllm.v1.request import Request, RequestStatus
@@ -126,6 +128,9 @@ class EngineCore:
128
scheduler_output, output)
129
return engine_core_outputs
130
131
+ def profile(self, is_start=True):
132
+ self.model_executor.worker.profile(is_start)
133
+
134
135
class EngineCoreProc(EngineCore):
136
"""ZMQ-wrapper for running EngineCore in background process."""
@@ -312,11 +317,14 @@ class EngineCoreProc(EngineCore):
317
self._last_logging_time = now
318
319
def _handle_client_request(
315
- self, request: Union[EngineCoreRequest, List[str]]) -> None:
320
+ self, request: Union[EngineCoreRequest, EngineCoreProfile,
321
+ List[str]]) -> None:
322
"""Handle EngineCoreRequest or EngineCoreABORT from Client."""
323
324
if isinstance(request, EngineCoreRequest):
325
self.add_request(request)
326
+ elif isinstance(request, EngineCoreProfile):
327
+ self.model_executor.worker.profile(request.is_start)
328
else:
329
# TODO: make an EngineCoreAbort wrapper
330
assert isinstance(request, list)
@@ -341,6 +349,8 @@ class EngineCoreProc(EngineCore):
349
request = decoder_add_req.decode(request_data)
350
elif request_type == EngineCoreRequestType.ABORT.value:
351
request = decoder_abort_req.decode(request_data)
352
+ elif request_type == EngineCoreRequestType.PROFILE.value:
353
+ request = pickle.loads(request_data)
354
else:
355
raise ValueError(f"Unknown RequestType: {request_type}")
356
vllm/v1/engine/core_client.py
+23
-5
@@ -9,7 +9,8 @@ import zmq.asyncio
9
from vllm.logger import init_logger
10
from vllm.utils import get_open_zmq_ipc_path
11
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
12
- EngineCoreRequest, EngineCoreRequestType)
12
+ EngineCoreProfile, EngineCoreRequest,
13
+ EngineCoreRequestType)
14
from vllm.v1.engine.core import EngineCore, EngineCoreProc
15
from vllm.v1.serial_utils import PickleEncoder
16
@@ -58,6 +59,9 @@ class EngineCoreClient:
59
def add_request(self, request: EngineCoreRequest) -> None:
60
raise NotImplementedError
61
62
+ async def profile(self, is_start=True) -> None:
63
+ raise NotImplementedError
64
+
65
def abort_requests(self, request_ids: List[str]) -> None:
66
raise NotImplementedError
67
@@ -95,6 +99,9 @@ class InprocClient(EngineCoreClient):
99
def abort_requests(self, request_ids: List[str]) -> None:
100
self.engine_core.abort_requests(request_ids)
101
102
+ async def profile(self, is_start=True) -> None:
103
+ self.engine_core.profile(is_start)
104
+
105
106
class MPClient(EngineCoreClient):
107
"""
@@ -177,8 +184,10 @@ class SyncMPClient(MPClient):
184
engine_core_outputs = self.decoder.decode(frame.buffer).outputs
185
return engine_core_outputs
186
180
- def _send_input(self, request_type: EngineCoreRequestType,
181
- request: Union[EngineCoreRequest, List[str]]) -> None:
187
+ def _send_input(
188
+ self, request_type: EngineCoreRequestType,
189
+ request: Union[EngineCoreRequest, EngineCoreProfile,
190
+ List[str]]) -> None:
191
192
# (RequestType, SerializedRequest)
193
msg = (request_type.value, self.encoder.encode(request))
@@ -190,6 +199,10 @@ class SyncMPClient(MPClient):
199
def abort_requests(self, request_ids: List[str]) -> None:
200
self._send_input(EngineCoreRequestType.ABORT, request_ids)
201
202
+ async def profile(self, is_start=True) -> None:
203
+ self._send_input(EngineCoreRequestType.PROFILE,
204
+ EngineCoreProfile(is_start))
205
+
206
207
class AsyncMPClient(MPClient):
208
"""Asyncio-compatible client for multi-proc EngineCore."""
@@ -205,8 +218,9 @@ class AsyncMPClient(MPClient):
218
return engine_core_outputs
219
220
async def _send_input(
208
- self, request_type: EngineCoreRequestType,
209
- request: Union[EngineCoreRequest, List[str]]) -> None:
221
+ self, request_type: EngineCoreRequestType,
222
+ request: Union[EngineCoreRequest, EngineCoreProfile,
223
+ List[str]]) -> None:
224
225
msg = (request_type.value, self.encoder.encode(request))
226
await self.input_socket.send_multipart(msg, copy=False)
@@ -217,3 +231,7 @@ class AsyncMPClient(MPClient):
231
async def abort_requests_async(self, request_ids: List[str]) -> None:
232
if len(request_ids) > 0:
233
await self._send_input(EngineCoreRequestType.ABORT, request_ids)
234
+
235
+ async def profile(self, is_start=True) -> None:
236
+ await self._send_input(EngineCoreRequestType.PROFILE,
237
+ EngineCoreProfile(is_start))
vllm/v1/worker/gpu_worker.py
+25
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
6
import torch
7
import torch.distributed
8
9
+import vllm.envs as envs
10
from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment,
@@ -56,6 +57,22 @@ class Worker:
57
init_cached_hf_modules()
58
59
self.model_runner = GPUModelRunner(vllm_config)
60
+ # Torch profiler. Enabled and configured through env vars:
61
+ # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
62
+ if envs.VLLM_TORCH_PROFILER_DIR:
63
+ torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
64
+ logger.info("Profiling enabled. Traces will be saved to: %s",
65
+ torch_profiler_trace_dir)
66
+ self.profiler = torch.profiler.profile(
67
+ activities=[
68
+ torch.profiler.ProfilerActivity.CPU,
69
+ torch.profiler.ProfilerActivity.CUDA,
70
+ ],
71
+ with_stack=True,
72
+ on_trace_ready=torch.profiler.tensorboard_trace_handler(
73
+ torch_profiler_trace_dir, use_gzip=True))
74
+ else:
75
+ self.profiler = None
76
77
def initialize(self):
78
if self.device_config.device.type == "cuda":
@@ -184,6 +201,14 @@ class Worker:
201
# TODO(woosuk): Send the output to the engine process.
202
return output
203
204
+ def profile(self, is_start=True):
205
+ if self.profiler is None:
206
+ raise RuntimeError("Profiler is not enabled.")
207
+ if is_start:
208
+ self.profiler.start()
209
+ else:
210
+ self.profiler.stop()
211
+
212
213
def init_worker_distributed_environment(
214
parallel_config: ParallelConfig,