@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d345f409b7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] EngineCore supports profiling (#10564)

Signed-off-by: Abatom <abzhonghua@gmail.com>

Zhonghua Deng committed Nov 23, 2024 at 09:16 UTC d345f409b7478c0e547b238916ec9e90b6156bbc
5 files changed +68 -9
vllm/v1/engine/__init__.py
+6
@@ -68,6 +68,11 @@ class EngineCoreOutputs(msgspec.Struct,
68 outputs: List[EngineCoreOutput]
69
70
71 +@dataclass
72 +class EngineCoreProfile:
73 + is_start: bool
74 +
75 +
76 class EngineCoreRequestType(enum.Enum):
77 """
78 Request types defined as hex byte strings, so it can be sent over sockets
@@ -75,3 +80,4 @@ class EngineCoreRequestType(enum.Enum):
80 """
81 ADD = b'\x00'
82 ABORT = b'\x01'
83 + PROFILE = b'\x02'
vllm/v1/engine/async_llm.py
+2 -2
@@ -346,10 +346,10 @@ class AsyncLLM(EngineClient):
346 logger.debug("Called check_health.")
347
348 async def start_profile(self) -> None:
349 - raise ValueError("Not supported on V1 yet.")
349 + await self.engine_core.profile(True)
350
351 async def stop_profile(self) -> None:
352 - raise ValueError("Not supported on V1 yet.")
352 + await self.engine_core.profile(False)
353
354 @property
355 def is_running(self) -> bool:
vllm/v1/engine/core.py
+12 -2
@@ -1,4 +1,5 @@
1 import multiprocessing
2 +import pickle
3 import queue
4 import threading
5 import time
@@ -16,7 +17,8 @@ from vllm.logger import init_logger
17 from vllm.usage.usage_lib import UsageContext
18 from vllm.v1.core.scheduler import Scheduler
19 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19 - EngineCoreRequest, EngineCoreRequestType)
20 + EngineCoreProfile, EngineCoreRequest,
21 + EngineCoreRequestType)
22 from vllm.v1.engine.mm_input_mapper import MMInputMapper
23 from vllm.v1.executor.gpu_executor import GPUExecutor
24 from vllm.v1.request import Request, RequestStatus
@@ -126,6 +128,9 @@ class EngineCore:
128 scheduler_output, output)
129 return engine_core_outputs
130
131 + def profile(self, is_start=True):
132 + self.model_executor.worker.profile(is_start)
133 +
134
135 class EngineCoreProc(EngineCore):
136 """ZMQ-wrapper for running EngineCore in background process."""
@@ -312,11 +317,14 @@ class EngineCoreProc(EngineCore):
317 self._last_logging_time = now
318
319 def _handle_client_request(
315 - self, request: Union[EngineCoreRequest, List[str]]) -> None:
320 + self, request: Union[EngineCoreRequest, EngineCoreProfile,
321 + List[str]]) -> None:
322 """Handle EngineCoreRequest or EngineCoreABORT from Client."""
323
324 if isinstance(request, EngineCoreRequest):
325 self.add_request(request)
326 + elif isinstance(request, EngineCoreProfile):
327 + self.model_executor.worker.profile(request.is_start)
328 else:
329 # TODO: make an EngineCoreAbort wrapper
330 assert isinstance(request, list)
@@ -341,6 +349,8 @@ class EngineCoreProc(EngineCore):
349 request = decoder_add_req.decode(request_data)
350 elif request_type == EngineCoreRequestType.ABORT.value:
351 request = decoder_abort_req.decode(request_data)
352 + elif request_type == EngineCoreRequestType.PROFILE.value:
353 + request = pickle.loads(request_data)
354 else:
355 raise ValueError(f"Unknown RequestType: {request_type}")
356
vllm/v1/engine/core_client.py
+23 -5
@@ -9,7 +9,8 @@ import zmq.asyncio
9 from vllm.logger import init_logger
10 from vllm.utils import get_open_zmq_ipc_path
11 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
12 - EngineCoreRequest, EngineCoreRequestType)
12 + EngineCoreProfile, EngineCoreRequest,
13 + EngineCoreRequestType)
14 from vllm.v1.engine.core import EngineCore, EngineCoreProc
15 from vllm.v1.serial_utils import PickleEncoder
16
@@ -58,6 +59,9 @@ class EngineCoreClient:
59 def add_request(self, request: EngineCoreRequest) -> None:
60 raise NotImplementedError
61
62 + async def profile(self, is_start=True) -> None:
63 + raise NotImplementedError
64 +
65 def abort_requests(self, request_ids: List[str]) -> None:
66 raise NotImplementedError
67
@@ -95,6 +99,9 @@ class InprocClient(EngineCoreClient):
99 def abort_requests(self, request_ids: List[str]) -> None:
100 self.engine_core.abort_requests(request_ids)
101
102 + async def profile(self, is_start=True) -> None:
103 + self.engine_core.profile(is_start)
104 +
105
106 class MPClient(EngineCoreClient):
107 """
@@ -177,8 +184,10 @@ class SyncMPClient(MPClient):
184 engine_core_outputs = self.decoder.decode(frame.buffer).outputs
185 return engine_core_outputs
186
180 - def _send_input(self, request_type: EngineCoreRequestType,
181 - request: Union[EngineCoreRequest, List[str]]) -> None:
187 + def _send_input(
188 + self, request_type: EngineCoreRequestType,
189 + request: Union[EngineCoreRequest, EngineCoreProfile,
190 + List[str]]) -> None:
191
192 # (RequestType, SerializedRequest)
193 msg = (request_type.value, self.encoder.encode(request))
@@ -190,6 +199,10 @@ class SyncMPClient(MPClient):
199 def abort_requests(self, request_ids: List[str]) -> None:
200 self._send_input(EngineCoreRequestType.ABORT, request_ids)
201
202 + async def profile(self, is_start=True) -> None:
203 + self._send_input(EngineCoreRequestType.PROFILE,
204 + EngineCoreProfile(is_start))
205 +
206
207 class AsyncMPClient(MPClient):
208 """Asyncio-compatible client for multi-proc EngineCore."""
@@ -205,8 +218,9 @@ class AsyncMPClient(MPClient):
218 return engine_core_outputs
219
220 async def _send_input(
208 - self, request_type: EngineCoreRequestType,
209 - request: Union[EngineCoreRequest, List[str]]) -> None:
221 + self, request_type: EngineCoreRequestType,
222 + request: Union[EngineCoreRequest, EngineCoreProfile,
223 + List[str]]) -> None:
224
225 msg = (request_type.value, self.encoder.encode(request))
226 await self.input_socket.send_multipart(msg, copy=False)
@@ -217,3 +231,7 @@ class AsyncMPClient(MPClient):
231 async def abort_requests_async(self, request_ids: List[str]) -> None:
232 if len(request_ids) > 0:
233 await self._send_input(EngineCoreRequestType.ABORT, request_ids)
234 +
235 + async def profile(self, is_start=True) -> None:
236 + await self._send_input(EngineCoreRequestType.PROFILE,
237 + EngineCoreProfile(is_start))
vllm/v1/worker/gpu_worker.py
+25
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
6 import torch
7 import torch.distributed
8
9 +import vllm.envs as envs
10 from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment,
@@ -56,6 +57,22 @@ class Worker:
57 init_cached_hf_modules()
58
59 self.model_runner = GPUModelRunner(vllm_config)
60 + # Torch profiler. Enabled and configured through env vars:
61 + # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
62 + if envs.VLLM_TORCH_PROFILER_DIR:
63 + torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
64 + logger.info("Profiling enabled. Traces will be saved to: %s",
65 + torch_profiler_trace_dir)
66 + self.profiler = torch.profiler.profile(
67 + activities=[
68 + torch.profiler.ProfilerActivity.CPU,
69 + torch.profiler.ProfilerActivity.CUDA,
70 + ],
71 + with_stack=True,
72 + on_trace_ready=torch.profiler.tensorboard_trace_handler(
73 + torch_profiler_trace_dir, use_gzip=True))
74 + else:
75 + self.profiler = None
76
77 def initialize(self):
78 if self.device_config.device.type == "cuda":
@@ -184,6 +201,14 @@ class Worker:
201 # TODO(woosuk): Send the output to the engine process.
202 return output
203
204 + def profile(self, is_start=True):
205 + if self.profiler is None:
206 + raise RuntimeError("Profiler is not enabled.")
207 + if is_start:
208 + self.profiler.start()
209 + else:
210 + self.profiler.stop()
211 +
212
213 def init_worker_distributed_environment(
214 parallel_config: ParallelConfig,