@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64840dfae4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Frontend] MQLLMEngine supports profiling. (#8761)

科英 committed Sep 26, 2024 at 00:37 UTC 64840dfae48621c5c2004eb8f1cb7fba49f9b24e
3 files changed +45 -7
vllm/engine/multiprocessing/__init__.py
+7 -1
@@ -107,7 +107,13 @@ class RPCStartupResponse:
107 tracing_enabled: bool
108
109
110 -RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest]
110 +class RPCUProfileRequest(Enum):
111 + START_PROFILE = 1
112 + STOP_PROFILE = 2
113 +
114 +
115 +RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
116 + RPCUProfileRequest]
117
118 REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
119
vllm/engine/multiprocessing/client.py
+18 -5
@@ -21,7 +21,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
21 IPC_OUTPUT_EXT, RPC_REQUEST_T,
22 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
23 RPCError, RPCProcessRequest,
24 - RPCStartupRequest, RPCStartupResponse)
24 + RPCStartupRequest, RPCStartupResponse,
25 + RPCUProfileRequest)
26 # yapf: enable
27 from vllm.envs import VLLM_RPC_TIMEOUT
28 from vllm.inputs import PromptType
@@ -38,10 +39,10 @@ logger = init_logger(__name__)
39
40 class MQClientClosedError(Exception):
41 """Exception class raised when the client is used post-close.
41 -
42 +
43 The client can be closed, which closes the ZMQ context. This normally
43 - happens on server shutdown. In some cases, methods like abort and
44 - do_log_stats will still be called and then try to open a socket, which
44 + happens on server shutdown. In some cases, methods like abort and
45 + do_log_stats will still be called and then try to open a socket, which
46 causes a ZMQError and creates a huge stack trace.
47 So, we throw this error such that we can suppress it.
48 """
@@ -345,7 +346,7 @@ class MQLLMEngineClient:
346 async def check_health(self):
347 """
348 The check health loop probes the health status of the
348 - Engine's health every N seconds and sets _errored_with
349 + Engine's health every N seconds and sets _errored_with
350 if the engine is unhealthy.
351 """
352 if self._errored_with is not None:
@@ -561,3 +562,15 @@ class MQLLMEngineClient:
562 await self.abort(request_id)
563 finally:
564 self.output_queues.pop(request_id)
565 +
566 + async def start_profile(self) -> None:
567 + """Start profiling the engine"""
568 +
569 + await self._send_one_way_rpc_request(
570 + request=RPCUProfileRequest.START_PROFILE, socket=self.input_socket)
571 +
572 + async def stop_profile(self) -> None:
573 + """Stop profiling the engine"""
574 +
575 + await self._send_one_way_rpc_request(
576 + request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
vllm/engine/multiprocessing/engine.py
+20 -1
@@ -18,9 +18,11 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
18 IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
19 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
20 RPCError, RPCProcessRequest,
21 - RPCStartupRequest, RPCStartupResponse)
21 + RPCStartupRequest, RPCStartupResponse,
22 + RPCUProfileRequest)
23 # yapf: enable
24 from vllm.envs import VLLM_RPC_TIMEOUT
25 +from vllm.executor.gpu_executor import GPUExecutor
26 from vllm.logger import init_logger
27 from vllm.outputs import RequestOutput
28 from vllm.usage.usage_lib import UsageContext
@@ -249,6 +251,11 @@ class MQLLMEngine:
251 self._handle_process_request(request)
252 elif isinstance(request, RPCAbortRequest):
253 self._handle_abort_request(request)
254 + elif isinstance(request, RPCUProfileRequest):
255 + if request == RPCUProfileRequest.START_PROFILE:
256 + self.start_profile()
257 + else:
258 + self.stop_profile()
259 else:
260 raise ValueError("Unknown RPCRequest Type: "
261 f"{type(request)}")
@@ -356,6 +363,18 @@ class MQLLMEngine:
363 def _alive(self):
364 self._last_alive_time = time.time()
365
366 + def start_profile(self) -> None:
367 + if type(self.engine.model_executor) is GPUExecutor:
368 + self.engine.model_executor.start_profile()
369 + else:
370 + self.engine.model_executor._run_workers("start_profile")
371 +
372 + def stop_profile(self) -> None:
373 + if type(self.engine.model_executor) is GPUExecutor:
374 + self.engine.model_executor.stop_profile()
375 + else:
376 + self.engine.model_executor._run_workers("stop_profile")
377 +
378
379 def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
380 ipc_path: str):