@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df04dffade
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] [4/N] API Server: ZMQ/MP Utilities (#11541)

Robert Shaw committed Dec 27, 2024 at 20:45 UTC df04dffade84c87cafd74de4c39e6fd7cb95c24f
12 files changed +247 -215
docs/requirements-docs.txt
+1
@@ -19,3 +19,4 @@ openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entr
19 fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
20 partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
21 requests
22 +zmq
tests/v1/engine/test_engine_core.py
+4 -9
@@ -7,7 +7,6 @@ from transformers import AutoTokenizer
7 from vllm import SamplingParams
8 from vllm.engine.arg_utils import EngineArgs
9 from vllm.platforms import current_platform
10 -from vllm.usage.usage_lib import UsageContext
10 from vllm.v1.engine import EngineCoreRequest
11 from vllm.v1.engine.async_llm import AsyncLLM
12 from vllm.v1.engine.core import EngineCore
@@ -43,13 +42,11 @@ def test_engine_core(monkeypatch):
42 m.setenv("VLLM_USE_V1", "1")
43 """Setup the EngineCore."""
44 engine_args = EngineArgs(model=MODEL_NAME)
46 - vllm_config = engine_args.create_engine_config(
47 - usage_context=UsageContext.UNKNOWN_CONTEXT)
45 + vllm_config = engine_args.create_engine_config()
46 executor_class = AsyncLLM._get_executor_cls(vllm_config)
47
48 engine_core = EngineCore(vllm_config=vllm_config,
51 - executor_class=executor_class,
52 - usage_context=UsageContext.UNKNOWN_CONTEXT)
49 + executor_class=executor_class)
50 """Test basic request lifecycle."""
51
52 # First request.
@@ -151,13 +148,11 @@ def test_engine_core_advanced_sampling(monkeypatch):
148 m.setenv("VLLM_USE_V1", "1")
149 """Setup the EngineCore."""
150 engine_args = EngineArgs(model=MODEL_NAME)
154 - vllm_config = engine_args.create_engine_config(
155 - usage_context=UsageContext.UNKNOWN_CONTEXT)
151 + vllm_config = engine_args.create_engine_config()
152 executor_class = AsyncLLM._get_executor_cls(vllm_config)
153
154 engine_core = EngineCore(vllm_config=vllm_config,
159 - executor_class=executor_class,
160 - usage_context=UsageContext.UNKNOWN_CONTEXT)
155 + executor_class=executor_class)
156 """Test basic request lifecycle."""
157 # First request.
158 request: EngineCoreRequest = make_request()
tests/v1/engine/test_engine_core_client.py
+4 -6
@@ -86,11 +86,10 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
86 UsageContext.UNKNOWN_CONTEXT)
87 executor_class = AsyncLLM._get_executor_cls(vllm_config)
88 client = EngineCoreClient.make_client(
89 - vllm_config,
90 - executor_class,
91 - UsageContext.UNKNOWN_CONTEXT,
89 multiprocess_mode=multiprocessing_mode,
90 asyncio_mode=False,
91 + vllm_config=vllm_config,
92 + executor_class=executor_class,
93 )
94
95 MAX_TOKENS = 20
@@ -158,11 +157,10 @@ async def test_engine_core_client_asyncio(monkeypatch):
157 usage_context=UsageContext.UNKNOWN_CONTEXT)
158 executor_class = AsyncLLM._get_executor_cls(vllm_config)
159 client = EngineCoreClient.make_client(
161 - vllm_config,
162 - executor_class,
163 - UsageContext.UNKNOWN_CONTEXT,
160 multiprocess_mode=True,
161 asyncio_mode=True,
162 + vllm_config=vllm_config,
163 + executor_class=executor_class,
164 )
165
166 MAX_TOKENS = 20
vllm/entrypoints/openai/api_server.py
+10 -1
@@ -68,7 +68,7 @@ from vllm.entrypoints.utils import with_cancellation
68 from vllm.logger import init_logger
69 from vllm.usage.usage_lib import UsageContext
70 from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
71 - is_valid_ipv6_address, set_ulimit)
71 + is_valid_ipv6_address, kill_process_tree, set_ulimit)
72 from vllm.version import __version__ as VLLM_VERSION
73
74 TIMEOUT_KEEP_ALIVE = 5 # seconds
@@ -737,6 +737,15 @@ async def run_server(args, **uvicorn_kwargs) -> None:
737
738 signal.signal(signal.SIGTERM, signal_handler)
739
740 + # The child processes will send SIGQUIT to this process when
741 + # any error happens. This process then clean up the whole tree.
742 + # TODO(rob): move this into AsyncLLM.__init__ once we remove
743 + # the context manager below.
744 + def sigquit_handler(signum, frame):
745 + kill_process_tree(os.getpid())
746 +
747 + signal.signal(signal.SIGQUIT, sigquit_handler)
748 +
749 async with build_async_engine_client(args) as engine_client:
750 app = build_app(args)
751
vllm/executor/multiproc_worker_utils.py
+1 -21
@@ -1,5 +1,4 @@
1 import asyncio
2 -import multiprocessing
2 import os
3 import sys
4 import threading
@@ -13,10 +12,9 @@ from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
12
13 import torch
14
16 -import vllm.envs as envs
15 from vllm.logger import init_logger
16 from vllm.triton_utils.importing import HAS_TRITON
19 -from vllm.utils import cuda_is_initialized
17 +from vllm.utils import _check_multiproc_method, get_mp_context
18
19 if HAS_TRITON:
20 from vllm.triton_utils import maybe_set_triton_cache_manager
@@ -274,24 +272,6 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
272 file.write = write_with_prefix # type: ignore[method-assign]
273
274
277 -def _check_multiproc_method():
278 - if (cuda_is_initialized()
279 - and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
280 - logger.warning("CUDA was previously initialized. We must use "
281 - "the `spawn` multiprocessing start method. Setting "
282 - "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
283 - "See https://docs.vllm.ai/en/latest/getting_started/"
284 - "debugging.html#python-multiprocessing "
285 - "for more information.")
286 - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
287 -
288 -
289 -def get_mp_context():
290 - _check_multiproc_method()
291 - mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
292 - return multiprocessing.get_context(mp_method)
293 -
294 -
275 def set_multiprocessing_worker_envs(parallel_config):
276 """ Set up environment variables that should be used when there are workers
277 in a multiprocessing environment. This should be called by the parent
vllm/utils.py
+87 -3
@@ -10,6 +10,7 @@ import importlib.metadata
10 import importlib.util
11 import inspect
12 import ipaddress
13 +import multiprocessing
14 import os
15 import re
16 import resource
@@ -20,6 +21,7 @@ import sys
21 import tempfile
22 import threading
23 import time
24 +import traceback
25 import uuid
26 import warnings
27 import weakref
@@ -29,8 +31,9 @@ from collections.abc import Hashable, Iterable, Mapping
31 from dataclasses import dataclass, field
32 from functools import lru_cache, partial, wraps
33 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
32 - Dict, Generator, Generic, List, Literal, NamedTuple,
33 - Optional, Tuple, Type, TypeVar, Union, overload)
34 + Dict, Generator, Generic, Iterator, List, Literal,
35 + NamedTuple, Optional, Tuple, Type, TypeVar, Union,
36 + overload)
37 from uuid import uuid4
38
39 import numpy as np
@@ -39,6 +42,8 @@ import psutil
42 import torch
43 import torch.types
44 import yaml
45 +import zmq
46 +import zmq.asyncio
47 from packaging.version import Version
48 from torch.library import Library
49 from typing_extensions import ParamSpec, TypeIs, assert_never
@@ -1844,7 +1849,7 @@ def memory_profiling(
1849 result.non_kv_cache_memory_in_bytes = result.non_torch_increase_in_bytes + result.torch_peak_increase_in_bytes + result.weights_memory_in_bytes # noqa
1850
1851
1847 -# Adapted from: https://github.com/sgl-project/sglang/blob/f46f394f4d4dbe4aae85403dec006199b34d2840/python/sglang/srt/utils.py#L630 # noqa: E501Curre
1852 +# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L630 # noqa: E501
1853 def set_ulimit(target_soft_limit=65535):
1854 resource_type = resource.RLIMIT_NOFILE
1855 current_soft, current_hard = resource.getrlimit(resource_type)
@@ -1859,3 +1864,82 @@ def set_ulimit(target_soft_limit=65535):
1864 "with error %s. This can cause fd limit errors like"
1865 "`OSError: [Errno 24] Too many open files`. Consider "
1866 "increasing with ulimit -n", current_soft, e)
1867 +
1868 +
1869 +# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/utils.py#L28 # noqa: E501
1870 +def get_exception_traceback():
1871 + etype, value, tb = sys.exc_info()
1872 + err_str = "".join(traceback.format_exception(etype, value, tb))
1873 + return err_str
1874 +
1875 +
1876 +# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L783 # noqa: E501
1877 +def make_zmq_socket(
1878 + ctx: Union[zmq.asyncio.Context, zmq.Context], # type: ignore[name-defined]
1879 + path: str,
1880 + type: Any,
1881 +) -> Union[zmq.Socket, zmq.asyncio.Socket]: # type: ignore[name-defined]
1882 + """Make a ZMQ socket with the proper bind/connect semantics."""
1883 +
1884 + mem = psutil.virtual_memory()
1885 + socket = ctx.socket(type)
1886 +
1887 + # Calculate buffer size based on system memory
1888 + total_mem = mem.total / 1024**3
1889 + available_mem = mem.available / 1024**3
1890 + # For systems with substantial memory (>32GB total, >16GB available):
1891 + # - Set a large 0.5GB buffer to improve throughput
1892 + # For systems with less memory:
1893 + # - Use system default (-1) to avoid excessive memory consumption
1894 + if total_mem > 32 and available_mem > 16:
1895 + buf_size = int(0.5 * 1024**3) # 0.5GB in bytes
1896 + else:
1897 + buf_size = -1 # Use system default buffer size
1898 +
1899 + if type == zmq.constants.PULL:
1900 + socket.setsockopt(zmq.constants.RCVHWM, 0)
1901 + socket.setsockopt(zmq.constants.RCVBUF, buf_size)
1902 + socket.connect(path)
1903 + elif type == zmq.constants.PUSH:
1904 + socket.setsockopt(zmq.constants.SNDHWM, 0)
1905 + socket.setsockopt(zmq.constants.SNDBUF, buf_size)
1906 + socket.bind(path)
1907 + else:
1908 + raise ValueError(f"Unknown Socket Type: {type}")
1909 +
1910 + return socket
1911 +
1912 +
1913 +@contextlib.contextmanager
1914 +def zmq_socket_ctx(
1915 + path: str,
1916 + type: Any) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
1917 + """Context manager for a ZMQ socket"""
1918 +
1919 + ctx = zmq.Context(io_threads=2) # type: ignore[attr-defined]
1920 + try:
1921 + yield make_zmq_socket(ctx, path, type)
1922 +
1923 + except KeyboardInterrupt:
1924 + logger.debug("Got Keyboard Interrupt.")
1925 +
1926 + finally:
1927 + ctx.destroy(linger=0)
1928 +
1929 +
1930 +def _check_multiproc_method():
1931 + if (cuda_is_initialized()
1932 + and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
1933 + logger.warning("CUDA was previously initialized. We must use "
1934 + "the `spawn` multiprocessing start method. Setting "
1935 + "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
1936 + "See https://docs.vllm.ai/en/latest/getting_started/"
1937 + "debugging.html#python-multiprocessing "
1938 + "for more information.")
1939 + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
1940 +
1941 +
1942 +def get_mp_context():
1943 + _check_multiproc_method()
1944 + mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
1945 + return multiprocessing.get_context(mp_method)
vllm/v1/engine/async_llm.py
+3 -3
@@ -75,11 +75,11 @@ class AsyncLLM(EngineClient):
75
76 # EngineCore (starts the engine in background process).
77 self.engine_core = EngineCoreClient.make_client(
78 - vllm_config=vllm_config,
79 - executor_class=executor_class,
80 - usage_context=usage_context,
78 multiprocess_mode=True,
79 asyncio_mode=True,
80 + vllm_config=vllm_config,
81 + executor_class=executor_class,
82 + log_stats=self.log_stats,
83 )
84
85 self.output_handler: Optional[asyncio.Task] = None
vllm/v1/engine/core.py
+22 -89
@@ -3,20 +3,19 @@ import queue
3 import signal
4 import threading
5 import time
6 -from dataclasses import dataclass
7 -from multiprocessing.process import BaseProcess
6 +from multiprocessing.connection import Connection
7 from typing import List, Tuple, Type
8
9 +import psutil
10 import zmq
11 import zmq.asyncio
12 from msgspec import msgpack
13
14 from vllm.config import CacheConfig, VllmConfig
15 -from vllm.executor.multiproc_worker_utils import get_mp_context
15 from vllm.logger import init_logger
16 from vllm.transformers_utils.config import (
17 maybe_register_config_serialize_by_value)
19 -from vllm.usage.usage_lib import UsageContext
18 +from vllm.utils import get_exception_traceback, zmq_socket_ctx
19 from vllm.v1.core.scheduler import Scheduler
20 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
21 EngineCoreProfile, EngineCoreRequest,
@@ -25,14 +24,13 @@ from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
24 from vllm.v1.executor.abstract import Executor
25 from vllm.v1.request import Request, RequestStatus
26 from vllm.v1.serial_utils import PickleEncoder
28 -from vllm.v1.utils import make_zmq_socket
27 from vllm.version import __version__ as VLLM_VERSION
28
29 logger = init_logger(__name__)
30
31 POLLING_TIMEOUT_MS = 5000
32 POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
35 -LOGGING_TIME_S = POLLING_TIMEOUT_S
33 +LOGGING_TIME_S = 5
34
35
36 class EngineCore:
@@ -42,9 +40,10 @@ class EngineCore:
40 self,
41 vllm_config: VllmConfig,
42 executor_class: Type[Executor],
45 - usage_context: UsageContext,
43 + log_stats: bool = False,
44 ):
45 assert vllm_config.model_config.runner_type != "pooling"
46 + self.log_stats = log_stats
47
48 logger.info("Initializing an LLM engine (v%s) with config: %s",
49 VLLM_VERSION, vllm_config)
@@ -134,29 +133,19 @@ class EngineCore:
133 self.model_executor.profile(is_start)
134
135
137 -@dataclass
138 -class EngineCoreProcHandle:
139 - proc: BaseProcess
140 - ready_path: str
141 - input_path: str
142 - output_path: str
143 -
144 -
136 class EngineCoreProc(EngineCore):
137 """ZMQ-wrapper for running EngineCore in background process."""
138
148 - READY_STR = "READY"
149 -
139 def __init__(
140 self,
152 - vllm_config: VllmConfig,
153 - executor_class: Type[Executor],
154 - usage_context: UsageContext,
141 input_path: str,
142 output_path: str,
157 - ready_path: str,
143 + ready_pipe: Connection,
144 + vllm_config: VllmConfig,
145 + executor_class: Type[Executor],
146 + log_stats: bool = False,
147 ):
159 - super().__init__(vllm_config, executor_class, usage_context)
148 + super().__init__(vllm_config, executor_class, log_stats)
149
150 # Background Threads and Queues for IO. These enable us to
151 # overlap ZMQ socket IO with GPU since they release the GIL,
@@ -173,68 +162,7 @@ class EngineCoreProc(EngineCore):
162 daemon=True).start()
163
164 # Send Readiness signal to EngineClient.
176 - with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
177 - ready_socket.send_string(EngineCoreProc.READY_STR)
178 -
179 - @staticmethod
180 - def wait_for_startup(
181 - proc: BaseProcess,
182 - ready_path: str,
183 - ) -> None:
184 - """Wait until the EngineCore is ready."""
185 -
186 - try:
187 - sync_ctx = zmq.Context() # type: ignore[attr-defined]
188 - socket = sync_ctx.socket(zmq.constants.PULL)
189 - socket.connect(ready_path)
190 -
191 - # Wait for EngineCore to send EngineCoreProc.READY_STR.
192 - while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
193 - logger.debug("Waiting for EngineCoreProc to startup.")
194 -
195 - if not proc.is_alive():
196 - raise RuntimeError("EngineCoreProc failed to start.")
197 -
198 - message = socket.recv_string()
199 - assert message == EngineCoreProc.READY_STR
200 -
201 - except BaseException as e:
202 - logger.exception(e)
203 - raise e
204 -
205 - finally:
206 - sync_ctx.destroy(linger=0)
207 -
208 - @staticmethod
209 - def make_engine_core_process(
210 - vllm_config: VllmConfig,
211 - executor_class: Type[Executor],
212 - usage_context: UsageContext,
213 - input_path: str,
214 - output_path: str,
215 - ready_path: str,
216 - ) -> EngineCoreProcHandle:
217 - context = get_mp_context()
218 -
219 - process_kwargs = {
220 - "input_path": input_path,
221 - "output_path": output_path,
222 - "ready_path": ready_path,
223 - "vllm_config": vllm_config,
224 - "executor_class": executor_class,
225 - "usage_context": usage_context,
226 - }
227 - # Run EngineCore busy loop in background process.
228 - proc = context.Process(target=EngineCoreProc.run_engine_core,
229 - kwargs=process_kwargs)
230 - proc.start()
231 -
232 - # Wait for startup
233 - EngineCoreProc.wait_for_startup(proc, ready_path)
234 - return EngineCoreProcHandle(proc=proc,
235 - ready_path=ready_path,
236 - input_path=input_path,
237 - output_path=output_path)
165 + ready_pipe.send({"status": "READY"})
166
167 @staticmethod
168 def run_engine_core(*args, **kwargs):
@@ -258,6 +186,7 @@ class EngineCoreProc(EngineCore):
186 signal.signal(signal.SIGTERM, signal_handler)
187 signal.signal(signal.SIGINT, signal_handler)
188
189 + parent_process = psutil.Process().parent()
190 engine_core = None
191 try:
192 engine_core = EngineCoreProc(*args, **kwargs)
@@ -266,9 +195,10 @@ class EngineCoreProc(EngineCore):
195 except SystemExit:
196 logger.debug("EngineCore interrupted.")
197
269 - except BaseException as e:
270 - logger.exception(e)
271 - raise e
198 + except Exception:
199 + traceback = get_exception_traceback()
200 + logger.error("EngineCore hit an exception: %s", traceback)
201 + parent_process.send_signal(signal.SIGQUIT)
202
203 finally:
204 if engine_core is not None:
@@ -309,6 +239,9 @@ class EngineCoreProc(EngineCore):
239 def _log_stats(self):
240 """Log basic stats every LOGGING_TIME_S"""
241
242 + if not self.log_stats:
243 + return
244 +
245 now = time.time()
246
247 if now - self._last_logging_time > LOGGING_TIME_S:
@@ -339,7 +272,7 @@ class EngineCoreProc(EngineCore):
272 decoder_add_req = PickleEncoder()
273 decoder_abort_req = PickleEncoder()
274
342 - with make_zmq_socket(input_path, zmq.constants.PULL) as socket:
275 + with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
276 while True:
277 # (RequestType, RequestData)
278 type_frame, data_frame = socket.recv_multipart(copy=False)
@@ -367,7 +300,7 @@ class EngineCoreProc(EngineCore):
300 # Reuse send buffer.
301 buffer = bytearray()
302
370 - with make_zmq_socket(output_path, zmq.constants.PUSH) as socket:
303 + with zmq_socket_ctx(output_path, zmq.constants.PUSH) as socket:
304 while True:
305 engine_core_outputs = self.output_queue.get()
306 outputs = EngineCoreOutputs(outputs=engine_core_outputs)
vllm/v1/engine/core_client.py
+47 -45
@@ -1,19 +1,19 @@
1 -import os
2 -import weakref
3 -from typing import List, Optional
1 +from typing import List, Optional, Type
2
3 import msgspec
4 import zmq
5 import zmq.asyncio
6
7 +from vllm.config import VllmConfig
8 from vllm.logger import init_logger
10 -from vllm.utils import get_open_zmq_ipc_path, kill_process_tree
9 +from vllm.utils import get_open_zmq_ipc_path
10 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
11 EngineCoreProfile, EngineCoreRequest,
12 EngineCoreRequestType, EngineCoreRequestUnion)
14 -from vllm.v1.engine.core import (EngineCore, EngineCoreProc,
15 - EngineCoreProcHandle)
13 +from vllm.v1.engine.core import EngineCore, EngineCoreProc
14 +from vllm.v1.executor.abstract import Executor
15 from vllm.v1.serial_utils import PickleEncoder
16 +from vllm.v1.utils import BackgroundProcHandle
17
18 logger = init_logger(__name__)
19
@@ -31,10 +31,11 @@ class EngineCoreClient:
31
32 @staticmethod
33 def make_client(
34 - *args,
34 multiprocess_mode: bool,
35 asyncio_mode: bool,
37 - **kwargs,
36 + vllm_config: VllmConfig,
37 + executor_class: Type[Executor],
38 + log_stats: bool = False,
39 ) -> "EngineCoreClient":
40
41 # TODO: support this for debugging purposes.
@@ -44,12 +45,12 @@ class EngineCoreClient:
45 "is not currently supported.")
46
47 if multiprocess_mode and asyncio_mode:
47 - return AsyncMPClient(*args, **kwargs)
48 + return AsyncMPClient(vllm_config, executor_class, log_stats)
49
50 if multiprocess_mode and not asyncio_mode:
50 - return SyncMPClient(*args, **kwargs)
51 + return SyncMPClient(vllm_config, executor_class, log_stats)
52
52 - return InprocClient(*args, **kwargs)
53 + return InprocClient(vllm_config, executor_class, log_stats)
54
55 def shutdown(self):
56 pass
@@ -128,9 +129,10 @@ class MPClient(EngineCoreClient):
129
130 def __init__(
131 self,
131 - *args,
132 asyncio_mode: bool,
133 - **kwargs,
133 + vllm_config: VllmConfig,
134 + executor_class: Type[Executor],
135 + log_stats: bool = False,
136 ):
137 # Serialization setup.
138 self.encoder = PickleEncoder()
@@ -143,7 +145,6 @@ class MPClient(EngineCoreClient):
145 self.ctx = zmq.Context() # type: ignore[attr-defined]
146
147 # Path for IPC.
146 - ready_path = get_open_zmq_ipc_path()
148 output_path = get_open_zmq_ipc_path()
149 input_path = get_open_zmq_ipc_path()
150
@@ -156,47 +157,40 @@ class MPClient(EngineCoreClient):
157 self.input_socket.bind(input_path)
158
159 # Start EngineCore in background process.
159 - self.proc_handle: Optional[EngineCoreProcHandle]
160 - self.proc_handle = EngineCoreProc.make_engine_core_process(
161 - *args,
162 - input_path=
163 - input_path, # type: ignore[misc] # MyPy incorrectly flags duplicate keywords
164 - output_path=output_path, # type: ignore[misc]
165 - ready_path=ready_path, # type: ignore[misc]
166 - **kwargs,
167 - )
168 - self._finalizer = weakref.finalize(self, self.shutdown)
160 + self.proc_handle: Optional[BackgroundProcHandle]
161 + self.proc_handle = BackgroundProcHandle(
162 + input_path=input_path,
163 + output_path=output_path,
164 + process_name="EngineCore",
165 + target_fn=EngineCoreProc.run_engine_core,
166 + process_kwargs={
167 + "vllm_config": vllm_config,
168 + "executor_class": executor_class,
169 + "log_stats": log_stats,
170 + })
171
172 def shutdown(self):
173 # Shut down the zmq context.
174 self.ctx.destroy(linger=0)
175
176 if hasattr(self, "proc_handle") and self.proc_handle:
175 - # Shutdown the process if needed.
176 - if self.proc_handle.proc.is_alive():
177 - self.proc_handle.proc.terminate()
178 - self.proc_handle.proc.join(5)
179 -
180 - if self.proc_handle.proc.is_alive():
181 - kill_process_tree(self.proc_handle.proc.pid)
182 -
183 - # Remove zmq ipc socket files
184 - ipc_sockets = [
185 - self.proc_handle.ready_path, self.proc_handle.output_path,
186 - self.proc_handle.input_path
187 - ]
188 - for ipc_socket in ipc_sockets:
189 - socket_file = ipc_socket.replace("ipc://", "")
190 - if os and os.path.exists(socket_file):
191 - os.remove(socket_file)
177 + self.proc_handle.shutdown()
178 self.proc_handle = None
179
180
181 class SyncMPClient(MPClient):
182 """Synchronous client for multi-proc EngineCore."""
183
198 - def __init__(self, *args, **kwargs):
199 - super().__init__(*args, asyncio_mode=False, **kwargs)
184 + def __init__(self,
185 + vllm_config: VllmConfig,
186 + executor_class: Type[Executor],
187 + log_stats: bool = False):
188 + super().__init__(
189 + asyncio_mode=False,
190 + vllm_config=vllm_config,
191 + executor_class=executor_class,
192 + log_stats=log_stats,
193 + )
194
195 def get_output(self) -> List[EngineCoreOutput]:
196
@@ -225,8 +219,16 @@ class SyncMPClient(MPClient):
219 class AsyncMPClient(MPClient):
220 """Asyncio-compatible client for multi-proc EngineCore."""
221
228 - def __init__(self, *args, **kwargs):
229 - super().__init__(*args, asyncio_mode=True, **kwargs)
222 + def __init__(self,
223 + vllm_config: VllmConfig,
224 + executor_class: Type[Executor],
225 + log_stats: bool = False):
226 + super().__init__(
227 + asyncio_mode=True,
228 + vllm_config=vllm_config,
229 + executor_class=executor_class,
230 + log_stats=log_stats,
231 + )
232
233 async def get_output_async(self) -> List[EngineCoreOutput]:
234
vllm/v1/engine/llm_engine.py
+3 -3
@@ -72,11 +72,11 @@ class LLMEngine:
72
73 # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
74 self.engine_core = EngineCoreClient.make_client(
75 - vllm_config,
76 - executor_class,
77 - usage_context,
75 multiprocess_mode=multiprocess_mode,
76 asyncio_mode=False,
77 + vllm_config=vllm_config,
78 + executor_class=executor_class,
79 + log_stats=False,
80 )
81
82 @classmethod
vllm/v1/executor/multiproc_executor.py
+5 -6
@@ -17,13 +17,12 @@ from vllm.distributed import (destroy_distributed_environment,
17 from vllm.distributed.device_communicators.shm_broadcast import (Handle,
18 MessageQueue)
19 from vllm.executor.multiproc_worker_utils import (
20 - _add_prefix, get_mp_context, set_multiprocessing_worker_envs)
20 + _add_prefix, set_multiprocessing_worker_envs)
21 from vllm.logger import init_logger
22 -from vllm.utils import (get_distributed_init_method, get_open_port,
23 - get_open_zmq_ipc_path)
22 +from vllm.utils import (get_distributed_init_method, get_mp_context,
23 + get_open_port, get_open_zmq_ipc_path, zmq_socket_ctx)
24 from vllm.v1.executor.abstract import Executor
25 from vllm.v1.outputs import ModelRunnerOutput
26 -from vllm.v1.utils import make_zmq_socket
26 from vllm.worker.worker_base import WorkerWrapperBase
27
28 logger = init_logger(__name__)
@@ -250,7 +249,7 @@ class WorkerProc:
249 worker_response_mq_handle = self.worker_response_mq.export_handle()
250
251 # Send Readiness signal to EngineCore process.
253 - with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
252 + with zmq_socket_ctx(ready_path, zmq.constants.PUSH) as ready_socket:
253 payload = pickle.dumps(worker_response_mq_handle,
254 protocol=pickle.HIGHEST_PROTOCOL)
255 ready_socket.send_string(WorkerProc.READY_STR)
@@ -352,7 +351,7 @@ class WorkerProc:
351 ready_path: str,
352 ) -> Optional[Handle]:
353 """Wait until the Worker is ready."""
355 - with make_zmq_socket(ready_path, zmq.constants.PULL) as socket:
354 + with zmq_socket_ctx(ready_path, zmq.constants.PULL) as socket:
355
356 # Wait for Worker to send READY.
357 while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
vllm/v1/utils.py
+60 -29
@@ -1,11 +1,11 @@
1 +import os
2 +import weakref
3 from collections.abc import Sequence
2 -from contextlib import contextmanager
3 -from typing import (Any, Generic, Iterator, List, Optional, TypeVar, Union,
4 - overload)
5 -
6 -import zmq
4 +from typing import (Any, Callable, Dict, Generic, List, Optional, TypeVar,
5 + Union, overload)
6
7 from vllm.logger import init_logger
8 +from vllm.utils import get_mp_context, kill_process_tree
9
10 logger = init_logger(__name__)
11
@@ -77,27 +77,58 @@ class ConstantList(Generic[T], Sequence):
77 return len(self._x)
78
79
80 -@contextmanager
81 -def make_zmq_socket(
82 - path: str,
83 - type: Any) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
84 - """Context manager for a ZMQ socket"""
85 -
86 - ctx = zmq.Context() # type: ignore[attr-defined]
87 - try:
88 - socket = ctx.socket(type)
89 -
90 - if type == zmq.constants.PULL:
91 - socket.connect(path)
92 - elif type == zmq.constants.PUSH:
93 - socket.bind(path)
94 - else:
95 - raise ValueError(f"Unknown Socket Type: {type}")
96 -
97 - yield socket
98 -
99 - except KeyboardInterrupt:
100 - logger.debug("Worker had Keyboard Interrupt.")
101 -
102 - finally:
103 - ctx.destroy(linger=0)
80 +class BackgroundProcHandle:
81 + """
82 + Utility class to handle creation, readiness, and shutdown
83 + of background processes used by the AsyncLLM and LLMEngine.
84 + """
85 +
86 + def __init__(
87 + self,
88 + input_path: str,
89 + output_path: str,
90 + process_name: str,
91 + target_fn: Callable,
92 + process_kwargs: Dict[Any, Any],
93 + ):
94 + self._finalizer = weakref.finalize(self, self.shutdown)
95 +
96 + context = get_mp_context()
97 + reader, writer = context.Pipe(duplex=False)
98 +
99 + assert ("ready_pipe" not in process_kwargs
100 + and "input_path" not in process_kwargs
101 + and "output_path" not in process_kwargs)
102 + process_kwargs["ready_pipe"] = writer
103 + process_kwargs["input_path"] = input_path
104 + process_kwargs["output_path"] = output_path
105 + self.input_path = input_path
106 + self.output_path = output_path
107 +
108 + # Run Detokenizer busy loop in background process.
109 + self.proc = context.Process(target=target_fn, kwargs=process_kwargs)
110 + self.proc.start()
111 +
112 + # Wait for startup.
113 + if reader.recv()["status"] != "READY":
114 + raise RuntimeError(f"{process_name} initialization failed. "
115 + "See root cause above.")
116 +
117 + def __del__(self):
118 + self.shutdown()
119 +
120 + def shutdown(self):
121 + # Shutdown the process if needed.
122 + if hasattr(self, "proc") and self.proc.is_alive():
123 + self.proc.terminate()
124 + self.proc.join(5)
125 +
126 + if self.proc.is_alive():
127 + kill_process_tree(self.proc.pid)
128 +
129 + # Remove zmq ipc socket files
130 + ipc_sockets = [self.output_path, self.input_path]
131 + for ipc_socket in ipc_sockets:
132 + socket_file = ipc_socket.replace("ipc://", "")
133 + if os and os.path.exists(socket_file):
134 + os.remove(socket_file)