A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Frontend] MQLLMEngine supports profiling. (#8761)
科英 committed
Sep 26, 2024 at 00:37 UTC
64840dfae48621c5c2004eb8f1cb7fba49f9b24e
3 files changed
+45
-7
vllm/engine/multiprocessing/__init__.py
+7
-1
@@ -107,7 +107,13 @@ class RPCStartupResponse:
107
tracing_enabled: bool
108
109
110
-RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest]
110
+class RPCUProfileRequest(Enum):
111
+ START_PROFILE = 1
112
+ STOP_PROFILE = 2
113
+
114
+
115
+RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
116
+ RPCUProfileRequest]
117
118
REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
119
vllm/engine/multiprocessing/client.py
+18
-5
@@ -21,7 +21,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
21
IPC_OUTPUT_EXT, RPC_REQUEST_T,
22
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
23
RPCError, RPCProcessRequest,
24
- RPCStartupRequest, RPCStartupResponse)
24
+ RPCStartupRequest, RPCStartupResponse,
25
+ RPCUProfileRequest)
26
# yapf: enable
27
from vllm.envs import VLLM_RPC_TIMEOUT
28
from vllm.inputs import PromptType
@@ -38,10 +39,10 @@ logger = init_logger(__name__)
39
40
class MQClientClosedError(Exception):
41
"""Exception class raised when the client is used post-close.
41
-
42
+
43
The client can be closed, which closes the ZMQ context. This normally
43
- happens on server shutdown. In some cases, methods like abort and
44
- do_log_stats will still be called and then try to open a socket, which
44
+ happens on server shutdown. In some cases, methods like abort and
45
+ do_log_stats will still be called and then try to open a socket, which
46
causes a ZMQError and creates a huge stack trace.
47
So, we throw this error such that we can suppress it.
48
"""
@@ -345,7 +346,7 @@ class MQLLMEngineClient:
346
async def check_health(self):
347
"""
348
The check health loop probes the health status of the
348
- Engine's health every N seconds and sets _errored_with
349
+ Engine's health every N seconds and sets _errored_with
350
if the engine is unhealthy.
351
"""
352
if self._errored_with is not None:
@@ -561,3 +562,15 @@ class MQLLMEngineClient:
562
await self.abort(request_id)
563
finally:
564
self.output_queues.pop(request_id)
565
+
566
+ async def start_profile(self) -> None:
567
+ """Start profiling the engine"""
568
+
569
+ await self._send_one_way_rpc_request(
570
+ request=RPCUProfileRequest.START_PROFILE, socket=self.input_socket)
571
+
572
+ async def stop_profile(self) -> None:
573
+ """Stop profiling the engine"""
574
+
575
+ await self._send_one_way_rpc_request(
576
+ request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
vllm/engine/multiprocessing/engine.py
+20
-1
@@ -18,9 +18,11 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
18
IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
19
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
20
RPCError, RPCProcessRequest,
21
- RPCStartupRequest, RPCStartupResponse)
21
+ RPCStartupRequest, RPCStartupResponse,
22
+ RPCUProfileRequest)
23
# yapf: enable
24
from vllm.envs import VLLM_RPC_TIMEOUT
25
+from vllm.executor.gpu_executor import GPUExecutor
26
from vllm.logger import init_logger
27
from vllm.outputs import RequestOutput
28
from vllm.usage.usage_lib import UsageContext
@@ -249,6 +251,11 @@ class MQLLMEngine:
251
self._handle_process_request(request)
252
elif isinstance(request, RPCAbortRequest):
253
self._handle_abort_request(request)
254
+ elif isinstance(request, RPCUProfileRequest):
255
+ if request == RPCUProfileRequest.START_PROFILE:
256
+ self.start_profile()
257
+ else:
258
+ self.stop_profile()
259
else:
260
raise ValueError("Unknown RPCRequest Type: "
261
f"{type(request)}")
@@ -356,6 +363,18 @@ class MQLLMEngine:
363
def _alive(self):
364
self._last_alive_time = time.time()
365
366
+ def start_profile(self) -> None:
367
+ if type(self.engine.model_executor) is GPUExecutor:
368
+ self.engine.model_executor.start_profile()
369
+ else:
370
+ self.engine.model_executor._run_workers("start_profile")
371
+
372
+ def stop_profile(self) -> None:
373
+ if type(self.engine.model_executor) is GPUExecutor:
374
+ self.engine.model_executor.stop_profile()
375
+ else:
376
+ self.engine.model_executor._run_workers("stop_profile")
377
+
378
379
def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
380
ipc_path: str):