A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] [4/N] API Server: ZMQ/MP Utilities (#11541)
Robert Shaw committed
Dec 27, 2024 at 20:45 UTC
df04dffade84c87cafd74de4c39e6fd7cb95c24f
12 files changed
+247
-215
docs/requirements-docs.txt
+1
@@ -19,3 +19,4 @@ openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entr
19
fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
20
partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
21
requests
22
+zmq
tests/v1/engine/test_engine_core.py
+4
-9
@@ -7,7 +7,6 @@ from transformers import AutoTokenizer
7
from vllm import SamplingParams
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.platforms import current_platform
10
-from vllm.usage.usage_lib import UsageContext
10
from vllm.v1.engine import EngineCoreRequest
11
from vllm.v1.engine.async_llm import AsyncLLM
12
from vllm.v1.engine.core import EngineCore
@@ -43,13 +42,11 @@ def test_engine_core(monkeypatch):
42
m.setenv("VLLM_USE_V1", "1")
43
"""Setup the EngineCore."""
44
engine_args = EngineArgs(model=MODEL_NAME)
46
- vllm_config = engine_args.create_engine_config(
47
- usage_context=UsageContext.UNKNOWN_CONTEXT)
45
+ vllm_config = engine_args.create_engine_config()
46
executor_class = AsyncLLM._get_executor_cls(vllm_config)
47
48
engine_core = EngineCore(vllm_config=vllm_config,
51
- executor_class=executor_class,
52
- usage_context=UsageContext.UNKNOWN_CONTEXT)
49
+ executor_class=executor_class)
50
"""Test basic request lifecycle."""
51
52
# First request.
@@ -151,13 +148,11 @@ def test_engine_core_advanced_sampling(monkeypatch):
148
m.setenv("VLLM_USE_V1", "1")
149
"""Setup the EngineCore."""
150
engine_args = EngineArgs(model=MODEL_NAME)
154
- vllm_config = engine_args.create_engine_config(
155
- usage_context=UsageContext.UNKNOWN_CONTEXT)
151
+ vllm_config = engine_args.create_engine_config()
152
executor_class = AsyncLLM._get_executor_cls(vllm_config)
153
154
engine_core = EngineCore(vllm_config=vllm_config,
159
- executor_class=executor_class,
160
- usage_context=UsageContext.UNKNOWN_CONTEXT)
155
+ executor_class=executor_class)
156
"""Test basic request lifecycle."""
157
# First request.
158
request: EngineCoreRequest = make_request()
tests/v1/engine/test_engine_core_client.py
+4
-6
@@ -86,11 +86,10 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
86
UsageContext.UNKNOWN_CONTEXT)
87
executor_class = AsyncLLM._get_executor_cls(vllm_config)
88
client = EngineCoreClient.make_client(
89
- vllm_config,
90
- executor_class,
91
- UsageContext.UNKNOWN_CONTEXT,
89
multiprocess_mode=multiprocessing_mode,
90
asyncio_mode=False,
91
+ vllm_config=vllm_config,
92
+ executor_class=executor_class,
93
)
94
95
MAX_TOKENS = 20
@@ -158,11 +157,10 @@ async def test_engine_core_client_asyncio(monkeypatch):
157
usage_context=UsageContext.UNKNOWN_CONTEXT)
158
executor_class = AsyncLLM._get_executor_cls(vllm_config)
159
client = EngineCoreClient.make_client(
161
- vllm_config,
162
- executor_class,
163
- UsageContext.UNKNOWN_CONTEXT,
160
multiprocess_mode=True,
161
asyncio_mode=True,
162
+ vllm_config=vllm_config,
163
+ executor_class=executor_class,
164
)
165
166
MAX_TOKENS = 20
vllm/entrypoints/openai/api_server.py
+10
-1
@@ -68,7 +68,7 @@ from vllm.entrypoints.utils import with_cancellation
68
from vllm.logger import init_logger
69
from vllm.usage.usage_lib import UsageContext
70
from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
71
- is_valid_ipv6_address, set_ulimit)
71
+ is_valid_ipv6_address, kill_process_tree, set_ulimit)
72
from vllm.version import __version__ as VLLM_VERSION
73
74
TIMEOUT_KEEP_ALIVE = 5 # seconds
@@ -737,6 +737,15 @@ async def run_server(args, **uvicorn_kwargs) -> None:
737
738
signal.signal(signal.SIGTERM, signal_handler)
739
740
+ # The child processes will send SIGQUIT to this process when
741
+ # any error happens. This process then clean up the whole tree.
742
+ # TODO(rob): move this into AsyncLLM.__init__ once we remove
743
+ # the context manager below.
744
+ def sigquit_handler(signum, frame):
745
+ kill_process_tree(os.getpid())
746
+
747
+ signal.signal(signal.SIGQUIT, sigquit_handler)
748
+
749
async with build_async_engine_client(args) as engine_client:
750
app = build_app(args)
751
vllm/executor/multiproc_worker_utils.py
+1
-21
@@ -1,5 +1,4 @@
1
import asyncio
2
-import multiprocessing
2
import os
3
import sys
4
import threading
@@ -13,10 +12,9 @@ from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
12
13
import torch
14
16
-import vllm.envs as envs
15
from vllm.logger import init_logger
16
from vllm.triton_utils.importing import HAS_TRITON
19
-from vllm.utils import cuda_is_initialized
17
+from vllm.utils import _check_multiproc_method, get_mp_context
18
19
if HAS_TRITON:
20
from vllm.triton_utils import maybe_set_triton_cache_manager
@@ -274,24 +272,6 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
272
file.write = write_with_prefix # type: ignore[method-assign]
273
274
277
-def _check_multiproc_method():
278
- if (cuda_is_initialized()
279
- and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
280
- logger.warning("CUDA was previously initialized. We must use "
281
- "the `spawn` multiprocessing start method. Setting "
282
- "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
283
- "See https://docs.vllm.ai/en/latest/getting_started/"
284
- "debugging.html#python-multiprocessing "
285
- "for more information.")
286
- os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
287
-
288
-
289
-def get_mp_context():
290
- _check_multiproc_method()
291
- mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
292
- return multiprocessing.get_context(mp_method)
293
-
294
-
275
def set_multiprocessing_worker_envs(parallel_config):
276
""" Set up environment variables that should be used when there are workers
277
in a multiprocessing environment. This should be called by the parent
vllm/utils.py
+87
-3
@@ -10,6 +10,7 @@ import importlib.metadata
10
import importlib.util
11
import inspect
12
import ipaddress
13
+import multiprocessing
14
import os
15
import re
16
import resource
@@ -20,6 +21,7 @@ import sys
21
import tempfile
22
import threading
23
import time
24
+import traceback
25
import uuid
26
import warnings
27
import weakref
@@ -29,8 +31,9 @@ from collections.abc import Hashable, Iterable, Mapping
31
from dataclasses import dataclass, field
32
from functools import lru_cache, partial, wraps
33
from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
32
- Dict, Generator, Generic, List, Literal, NamedTuple,
33
- Optional, Tuple, Type, TypeVar, Union, overload)
34
+ Dict, Generator, Generic, Iterator, List, Literal,
35
+ NamedTuple, Optional, Tuple, Type, TypeVar, Union,
36
+ overload)
37
from uuid import uuid4
38
39
import numpy as np
@@ -39,6 +42,8 @@ import psutil
42
import torch
43
import torch.types
44
import yaml
45
+import zmq
46
+import zmq.asyncio
47
from packaging.version import Version
48
from torch.library import Library
49
from typing_extensions import ParamSpec, TypeIs, assert_never
@@ -1844,7 +1849,7 @@ def memory_profiling(
1849
result.non_kv_cache_memory_in_bytes = result.non_torch_increase_in_bytes + result.torch_peak_increase_in_bytes + result.weights_memory_in_bytes # noqa
1850
1851
1847
-# Adapted from: https://github.com/sgl-project/sglang/blob/f46f394f4d4dbe4aae85403dec006199b34d2840/python/sglang/srt/utils.py#L630 # noqa: E501Curre
1852
+# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L630 # noqa: E501
1853
def set_ulimit(target_soft_limit=65535):
1854
resource_type = resource.RLIMIT_NOFILE
1855
current_soft, current_hard = resource.getrlimit(resource_type)
@@ -1859,3 +1864,82 @@ def set_ulimit(target_soft_limit=65535):
1864
"with error %s. This can cause fd limit errors like"
1865
"`OSError: [Errno 24] Too many open files`. Consider "
1866
"increasing with ulimit -n", current_soft, e)
1867
+
1868
+
1869
+# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/utils.py#L28 # noqa: E501
1870
+def get_exception_traceback():
1871
+ etype, value, tb = sys.exc_info()
1872
+ err_str = "".join(traceback.format_exception(etype, value, tb))
1873
+ return err_str
1874
+
1875
+
1876
+# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L783 # noqa: E501
1877
+def make_zmq_socket(
1878
+ ctx: Union[zmq.asyncio.Context, zmq.Context], # type: ignore[name-defined]
1879
+ path: str,
1880
+ type: Any,
1881
+) -> Union[zmq.Socket, zmq.asyncio.Socket]: # type: ignore[name-defined]
1882
+ """Make a ZMQ socket with the proper bind/connect semantics."""
1883
+
1884
+ mem = psutil.virtual_memory()
1885
+ socket = ctx.socket(type)
1886
+
1887
+ # Calculate buffer size based on system memory
1888
+ total_mem = mem.total / 1024**3
1889
+ available_mem = mem.available / 1024**3
1890
+ # For systems with substantial memory (>32GB total, >16GB available):
1891
+ # - Set a large 0.5GB buffer to improve throughput
1892
+ # For systems with less memory:
1893
+ # - Use system default (-1) to avoid excessive memory consumption
1894
+ if total_mem > 32 and available_mem > 16:
1895
+ buf_size = int(0.5 * 1024**3) # 0.5GB in bytes
1896
+ else:
1897
+ buf_size = -1 # Use system default buffer size
1898
+
1899
+ if type == zmq.constants.PULL:
1900
+ socket.setsockopt(zmq.constants.RCVHWM, 0)
1901
+ socket.setsockopt(zmq.constants.RCVBUF, buf_size)
1902
+ socket.connect(path)
1903
+ elif type == zmq.constants.PUSH:
1904
+ socket.setsockopt(zmq.constants.SNDHWM, 0)
1905
+ socket.setsockopt(zmq.constants.SNDBUF, buf_size)
1906
+ socket.bind(path)
1907
+ else:
1908
+ raise ValueError(f"Unknown Socket Type: {type}")
1909
+
1910
+ return socket
1911
+
1912
+
1913
+@contextlib.contextmanager
1914
+def zmq_socket_ctx(
1915
+ path: str,
1916
+ type: Any) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
1917
+ """Context manager for a ZMQ socket"""
1918
+
1919
+ ctx = zmq.Context(io_threads=2) # type: ignore[attr-defined]
1920
+ try:
1921
+ yield make_zmq_socket(ctx, path, type)
1922
+
1923
+ except KeyboardInterrupt:
1924
+ logger.debug("Got Keyboard Interrupt.")
1925
+
1926
+ finally:
1927
+ ctx.destroy(linger=0)
1928
+
1929
+
1930
+def _check_multiproc_method():
1931
+ if (cuda_is_initialized()
1932
+ and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
1933
+ logger.warning("CUDA was previously initialized. We must use "
1934
+ "the `spawn` multiprocessing start method. Setting "
1935
+ "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
1936
+ "See https://docs.vllm.ai/en/latest/getting_started/"
1937
+ "debugging.html#python-multiprocessing "
1938
+ "for more information.")
1939
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
1940
+
1941
+
1942
+def get_mp_context():
1943
+ _check_multiproc_method()
1944
+ mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
1945
+ return multiprocessing.get_context(mp_method)
vllm/v1/engine/async_llm.py
+3
-3
@@ -75,11 +75,11 @@ class AsyncLLM(EngineClient):
75
76
# EngineCore (starts the engine in background process).
77
self.engine_core = EngineCoreClient.make_client(
78
- vllm_config=vllm_config,
79
- executor_class=executor_class,
80
- usage_context=usage_context,
78
multiprocess_mode=True,
79
asyncio_mode=True,
80
+ vllm_config=vllm_config,
81
+ executor_class=executor_class,
82
+ log_stats=self.log_stats,
83
)
84
85
self.output_handler: Optional[asyncio.Task] = None
vllm/v1/engine/core.py
+22
-89
@@ -3,20 +3,19 @@ import queue
3
import signal
4
import threading
5
import time
6
-from dataclasses import dataclass
7
-from multiprocessing.process import BaseProcess
6
+from multiprocessing.connection import Connection
7
from typing import List, Tuple, Type
8
9
+import psutil
10
import zmq
11
import zmq.asyncio
12
from msgspec import msgpack
13
14
from vllm.config import CacheConfig, VllmConfig
15
-from vllm.executor.multiproc_worker_utils import get_mp_context
15
from vllm.logger import init_logger
16
from vllm.transformers_utils.config import (
17
maybe_register_config_serialize_by_value)
19
-from vllm.usage.usage_lib import UsageContext
18
+from vllm.utils import get_exception_traceback, zmq_socket_ctx
19
from vllm.v1.core.scheduler import Scheduler
20
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
21
EngineCoreProfile, EngineCoreRequest,
@@ -25,14 +24,13 @@ from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
24
from vllm.v1.executor.abstract import Executor
25
from vllm.v1.request import Request, RequestStatus
26
from vllm.v1.serial_utils import PickleEncoder
28
-from vllm.v1.utils import make_zmq_socket
27
from vllm.version import __version__ as VLLM_VERSION
28
29
logger = init_logger(__name__)
30
31
POLLING_TIMEOUT_MS = 5000
32
POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
35
-LOGGING_TIME_S = POLLING_TIMEOUT_S
33
+LOGGING_TIME_S = 5
34
35
36
class EngineCore:
@@ -42,9 +40,10 @@ class EngineCore:
40
self,
41
vllm_config: VllmConfig,
42
executor_class: Type[Executor],
45
- usage_context: UsageContext,
43
+ log_stats: bool = False,
44
):
45
assert vllm_config.model_config.runner_type != "pooling"
46
+ self.log_stats = log_stats
47
48
logger.info("Initializing an LLM engine (v%s) with config: %s",
49
VLLM_VERSION, vllm_config)
@@ -134,29 +133,19 @@ class EngineCore:
133
self.model_executor.profile(is_start)
134
135
137
-@dataclass
138
-class EngineCoreProcHandle:
139
- proc: BaseProcess
140
- ready_path: str
141
- input_path: str
142
- output_path: str
143
-
144
-
136
class EngineCoreProc(EngineCore):
137
"""ZMQ-wrapper for running EngineCore in background process."""
138
148
- READY_STR = "READY"
149
-
139
def __init__(
140
self,
152
- vllm_config: VllmConfig,
153
- executor_class: Type[Executor],
154
- usage_context: UsageContext,
141
input_path: str,
142
output_path: str,
157
- ready_path: str,
143
+ ready_pipe: Connection,
144
+ vllm_config: VllmConfig,
145
+ executor_class: Type[Executor],
146
+ log_stats: bool = False,
147
):
159
- super().__init__(vllm_config, executor_class, usage_context)
148
+ super().__init__(vllm_config, executor_class, log_stats)
149
150
# Background Threads and Queues for IO. These enable us to
151
# overlap ZMQ socket IO with GPU since they release the GIL,
@@ -173,68 +162,7 @@ class EngineCoreProc(EngineCore):
162
daemon=True).start()
163
164
# Send Readiness signal to EngineClient.
176
- with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
177
- ready_socket.send_string(EngineCoreProc.READY_STR)
178
-
179
- @staticmethod
180
- def wait_for_startup(
181
- proc: BaseProcess,
182
- ready_path: str,
183
- ) -> None:
184
- """Wait until the EngineCore is ready."""
185
-
186
- try:
187
- sync_ctx = zmq.Context() # type: ignore[attr-defined]
188
- socket = sync_ctx.socket(zmq.constants.PULL)
189
- socket.connect(ready_path)
190
-
191
- # Wait for EngineCore to send EngineCoreProc.READY_STR.
192
- while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
193
- logger.debug("Waiting for EngineCoreProc to startup.")
194
-
195
- if not proc.is_alive():
196
- raise RuntimeError("EngineCoreProc failed to start.")
197
-
198
- message = socket.recv_string()
199
- assert message == EngineCoreProc.READY_STR
200
-
201
- except BaseException as e:
202
- logger.exception(e)
203
- raise e
204
-
205
- finally:
206
- sync_ctx.destroy(linger=0)
207
-
208
- @staticmethod
209
- def make_engine_core_process(
210
- vllm_config: VllmConfig,
211
- executor_class: Type[Executor],
212
- usage_context: UsageContext,
213
- input_path: str,
214
- output_path: str,
215
- ready_path: str,
216
- ) -> EngineCoreProcHandle:
217
- context = get_mp_context()
218
-
219
- process_kwargs = {
220
- "input_path": input_path,
221
- "output_path": output_path,
222
- "ready_path": ready_path,
223
- "vllm_config": vllm_config,
224
- "executor_class": executor_class,
225
- "usage_context": usage_context,
226
- }
227
- # Run EngineCore busy loop in background process.
228
- proc = context.Process(target=EngineCoreProc.run_engine_core,
229
- kwargs=process_kwargs)
230
- proc.start()
231
-
232
- # Wait for startup
233
- EngineCoreProc.wait_for_startup(proc, ready_path)
234
- return EngineCoreProcHandle(proc=proc,
235
- ready_path=ready_path,
236
- input_path=input_path,
237
- output_path=output_path)
165
+ ready_pipe.send({"status": "READY"})
166
167
@staticmethod
168
def run_engine_core(*args, **kwargs):
@@ -258,6 +186,7 @@ class EngineCoreProc(EngineCore):
186
signal.signal(signal.SIGTERM, signal_handler)
187
signal.signal(signal.SIGINT, signal_handler)
188
189
+ parent_process = psutil.Process().parent()
190
engine_core = None
191
try:
192
engine_core = EngineCoreProc(*args, **kwargs)
@@ -266,9 +195,10 @@ class EngineCoreProc(EngineCore):
195
except SystemExit:
196
logger.debug("EngineCore interrupted.")
197
269
- except BaseException as e:
270
- logger.exception(e)
271
- raise e
198
+ except Exception:
199
+ traceback = get_exception_traceback()
200
+ logger.error("EngineCore hit an exception: %s", traceback)
201
+ parent_process.send_signal(signal.SIGQUIT)
202
203
finally:
204
if engine_core is not None:
@@ -309,6 +239,9 @@ class EngineCoreProc(EngineCore):
239
def _log_stats(self):
240
"""Log basic stats every LOGGING_TIME_S"""
241
242
+ if not self.log_stats:
243
+ return
244
+
245
now = time.time()
246
247
if now - self._last_logging_time > LOGGING_TIME_S:
@@ -339,7 +272,7 @@ class EngineCoreProc(EngineCore):
272
decoder_add_req = PickleEncoder()
273
decoder_abort_req = PickleEncoder()
274
342
- with make_zmq_socket(input_path, zmq.constants.PULL) as socket:
275
+ with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
276
while True:
277
# (RequestType, RequestData)
278
type_frame, data_frame = socket.recv_multipart(copy=False)
@@ -367,7 +300,7 @@ class EngineCoreProc(EngineCore):
300
# Reuse send buffer.
301
buffer = bytearray()
302
370
- with make_zmq_socket(output_path, zmq.constants.PUSH) as socket:
303
+ with zmq_socket_ctx(output_path, zmq.constants.PUSH) as socket:
304
while True:
305
engine_core_outputs = self.output_queue.get()
306
outputs = EngineCoreOutputs(outputs=engine_core_outputs)
vllm/v1/engine/core_client.py
+47
-45
@@ -1,19 +1,19 @@
1
-import os
2
-import weakref
3
-from typing import List, Optional
1
+from typing import List, Optional, Type
2
3
import msgspec
4
import zmq
5
import zmq.asyncio
6
7
+from vllm.config import VllmConfig
8
from vllm.logger import init_logger
10
-from vllm.utils import get_open_zmq_ipc_path, kill_process_tree
9
+from vllm.utils import get_open_zmq_ipc_path
10
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
11
EngineCoreProfile, EngineCoreRequest,
12
EngineCoreRequestType, EngineCoreRequestUnion)
14
-from vllm.v1.engine.core import (EngineCore, EngineCoreProc,
15
- EngineCoreProcHandle)
13
+from vllm.v1.engine.core import EngineCore, EngineCoreProc
14
+from vllm.v1.executor.abstract import Executor
15
from vllm.v1.serial_utils import PickleEncoder
16
+from vllm.v1.utils import BackgroundProcHandle
17
18
logger = init_logger(__name__)
19
@@ -31,10 +31,11 @@ class EngineCoreClient:
31
32
@staticmethod
33
def make_client(
34
- *args,
34
multiprocess_mode: bool,
35
asyncio_mode: bool,
37
- **kwargs,
36
+ vllm_config: VllmConfig,
37
+ executor_class: Type[Executor],
38
+ log_stats: bool = False,
39
) -> "EngineCoreClient":
40
41
# TODO: support this for debugging purposes.
@@ -44,12 +45,12 @@ class EngineCoreClient:
45
"is not currently supported.")
46
47
if multiprocess_mode and asyncio_mode:
47
- return AsyncMPClient(*args, **kwargs)
48
+ return AsyncMPClient(vllm_config, executor_class, log_stats)
49
50
if multiprocess_mode and not asyncio_mode:
50
- return SyncMPClient(*args, **kwargs)
51
+ return SyncMPClient(vllm_config, executor_class, log_stats)
52
52
- return InprocClient(*args, **kwargs)
53
+ return InprocClient(vllm_config, executor_class, log_stats)
54
55
def shutdown(self):
56
pass
@@ -128,9 +129,10 @@ class MPClient(EngineCoreClient):
129
130
def __init__(
131
self,
131
- *args,
132
asyncio_mode: bool,
133
- **kwargs,
133
+ vllm_config: VllmConfig,
134
+ executor_class: Type[Executor],
135
+ log_stats: bool = False,
136
):
137
# Serialization setup.
138
self.encoder = PickleEncoder()
@@ -143,7 +145,6 @@ class MPClient(EngineCoreClient):
145
self.ctx = zmq.Context() # type: ignore[attr-defined]
146
147
# Path for IPC.
146
- ready_path = get_open_zmq_ipc_path()
148
output_path = get_open_zmq_ipc_path()
149
input_path = get_open_zmq_ipc_path()
150
@@ -156,47 +157,40 @@ class MPClient(EngineCoreClient):
157
self.input_socket.bind(input_path)
158
159
# Start EngineCore in background process.
159
- self.proc_handle: Optional[EngineCoreProcHandle]
160
- self.proc_handle = EngineCoreProc.make_engine_core_process(
161
- *args,
162
- input_path=
163
- input_path, # type: ignore[misc] # MyPy incorrectly flags duplicate keywords
164
- output_path=output_path, # type: ignore[misc]
165
- ready_path=ready_path, # type: ignore[misc]
166
- **kwargs,
167
- )
168
- self._finalizer = weakref.finalize(self, self.shutdown)
160
+ self.proc_handle: Optional[BackgroundProcHandle]
161
+ self.proc_handle = BackgroundProcHandle(
162
+ input_path=input_path,
163
+ output_path=output_path,
164
+ process_name="EngineCore",
165
+ target_fn=EngineCoreProc.run_engine_core,
166
+ process_kwargs={
167
+ "vllm_config": vllm_config,
168
+ "executor_class": executor_class,
169
+ "log_stats": log_stats,
170
+ })
171
172
def shutdown(self):
173
# Shut down the zmq context.
174
self.ctx.destroy(linger=0)
175
176
if hasattr(self, "proc_handle") and self.proc_handle:
175
- # Shutdown the process if needed.
176
- if self.proc_handle.proc.is_alive():
177
- self.proc_handle.proc.terminate()
178
- self.proc_handle.proc.join(5)
179
-
180
- if self.proc_handle.proc.is_alive():
181
- kill_process_tree(self.proc_handle.proc.pid)
182
-
183
- # Remove zmq ipc socket files
184
- ipc_sockets = [
185
- self.proc_handle.ready_path, self.proc_handle.output_path,
186
- self.proc_handle.input_path
187
- ]
188
- for ipc_socket in ipc_sockets:
189
- socket_file = ipc_socket.replace("ipc://", "")
190
- if os and os.path.exists(socket_file):
191
- os.remove(socket_file)
177
+ self.proc_handle.shutdown()
178
self.proc_handle = None
179
180
181
class SyncMPClient(MPClient):
182
"""Synchronous client for multi-proc EngineCore."""
183
198
- def __init__(self, *args, **kwargs):
199
- super().__init__(*args, asyncio_mode=False, **kwargs)
184
+ def __init__(self,
185
+ vllm_config: VllmConfig,
186
+ executor_class: Type[Executor],
187
+ log_stats: bool = False):
188
+ super().__init__(
189
+ asyncio_mode=False,
190
+ vllm_config=vllm_config,
191
+ executor_class=executor_class,
192
+ log_stats=log_stats,
193
+ )
194
195
def get_output(self) -> List[EngineCoreOutput]:
196
@@ -225,8 +219,16 @@ class SyncMPClient(MPClient):
219
class AsyncMPClient(MPClient):
220
"""Asyncio-compatible client for multi-proc EngineCore."""
221
228
- def __init__(self, *args, **kwargs):
229
- super().__init__(*args, asyncio_mode=True, **kwargs)
222
+ def __init__(self,
223
+ vllm_config: VllmConfig,
224
+ executor_class: Type[Executor],
225
+ log_stats: bool = False):
226
+ super().__init__(
227
+ asyncio_mode=True,
228
+ vllm_config=vllm_config,
229
+ executor_class=executor_class,
230
+ log_stats=log_stats,
231
+ )
232
233
async def get_output_async(self) -> List[EngineCoreOutput]:
234
vllm/v1/engine/llm_engine.py
+3
-3
@@ -72,11 +72,11 @@ class LLMEngine:
72
73
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
74
self.engine_core = EngineCoreClient.make_client(
75
- vllm_config,
76
- executor_class,
77
- usage_context,
75
multiprocess_mode=multiprocess_mode,
76
asyncio_mode=False,
77
+ vllm_config=vllm_config,
78
+ executor_class=executor_class,
79
+ log_stats=False,
80
)
81
82
@classmethod
vllm/v1/executor/multiproc_executor.py
+5
-6
@@ -17,13 +17,12 @@ from vllm.distributed import (destroy_distributed_environment,
17
from vllm.distributed.device_communicators.shm_broadcast import (Handle,
18
MessageQueue)
19
from vllm.executor.multiproc_worker_utils import (
20
- _add_prefix, get_mp_context, set_multiprocessing_worker_envs)
20
+ _add_prefix, set_multiprocessing_worker_envs)
21
from vllm.logger import init_logger
22
-from vllm.utils import (get_distributed_init_method, get_open_port,
23
- get_open_zmq_ipc_path)
22
+from vllm.utils import (get_distributed_init_method, get_mp_context,
23
+ get_open_port, get_open_zmq_ipc_path, zmq_socket_ctx)
24
from vllm.v1.executor.abstract import Executor
25
from vllm.v1.outputs import ModelRunnerOutput
26
-from vllm.v1.utils import make_zmq_socket
26
from vllm.worker.worker_base import WorkerWrapperBase
27
28
logger = init_logger(__name__)
@@ -250,7 +249,7 @@ class WorkerProc:
249
worker_response_mq_handle = self.worker_response_mq.export_handle()
250
251
# Send Readiness signal to EngineCore process.
253
- with make_zmq_socket(ready_path, zmq.constants.PUSH) as ready_socket:
252
+ with zmq_socket_ctx(ready_path, zmq.constants.PUSH) as ready_socket:
253
payload = pickle.dumps(worker_response_mq_handle,
254
protocol=pickle.HIGHEST_PROTOCOL)
255
ready_socket.send_string(WorkerProc.READY_STR)
@@ -352,7 +351,7 @@ class WorkerProc:
351
ready_path: str,
352
) -> Optional[Handle]:
353
"""Wait until the Worker is ready."""
355
- with make_zmq_socket(ready_path, zmq.constants.PULL) as socket:
354
+ with zmq_socket_ctx(ready_path, zmq.constants.PULL) as socket:
355
356
# Wait for Worker to send READY.
357
while socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
vllm/v1/utils.py
+60
-29
@@ -1,11 +1,11 @@
1
+import os
2
+import weakref
3
from collections.abc import Sequence
2
-from contextlib import contextmanager
3
-from typing import (Any, Generic, Iterator, List, Optional, TypeVar, Union,
4
- overload)
5
-
6
-import zmq
4
+from typing import (Any, Callable, Dict, Generic, List, Optional, TypeVar,
5
+ Union, overload)
6
7
from vllm.logger import init_logger
8
+from vllm.utils import get_mp_context, kill_process_tree
9
10
logger = init_logger(__name__)
11
@@ -77,27 +77,58 @@ class ConstantList(Generic[T], Sequence):
77
return len(self._x)
78
79
80
-@contextmanager
81
-def make_zmq_socket(
82
- path: str,
83
- type: Any) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
84
- """Context manager for a ZMQ socket"""
85
-
86
- ctx = zmq.Context() # type: ignore[attr-defined]
87
- try:
88
- socket = ctx.socket(type)
89
-
90
- if type == zmq.constants.PULL:
91
- socket.connect(path)
92
- elif type == zmq.constants.PUSH:
93
- socket.bind(path)
94
- else:
95
- raise ValueError(f"Unknown Socket Type: {type}")
96
-
97
- yield socket
98
-
99
- except KeyboardInterrupt:
100
- logger.debug("Worker had Keyboard Interrupt.")
101
-
102
- finally:
103
- ctx.destroy(linger=0)
80
+class BackgroundProcHandle:
81
+ """
82
+ Utility class to handle creation, readiness, and shutdown
83
+ of background processes used by the AsyncLLM and LLMEngine.
84
+ """
85
+
86
+ def __init__(
87
+ self,
88
+ input_path: str,
89
+ output_path: str,
90
+ process_name: str,
91
+ target_fn: Callable,
92
+ process_kwargs: Dict[Any, Any],
93
+ ):
94
+ self._finalizer = weakref.finalize(self, self.shutdown)
95
+
96
+ context = get_mp_context()
97
+ reader, writer = context.Pipe(duplex=False)
98
+
99
+ assert ("ready_pipe" not in process_kwargs
100
+ and "input_path" not in process_kwargs
101
+ and "output_path" not in process_kwargs)
102
+ process_kwargs["ready_pipe"] = writer
103
+ process_kwargs["input_path"] = input_path
104
+ process_kwargs["output_path"] = output_path
105
+ self.input_path = input_path
106
+ self.output_path = output_path
107
+
108
+ # Run Detokenizer busy loop in background process.
109
+ self.proc = context.Process(target=target_fn, kwargs=process_kwargs)
110
+ self.proc.start()
111
+
112
+ # Wait for startup.
113
+ if reader.recv()["status"] != "READY":
114
+ raise RuntimeError(f"{process_name} initialization failed. "
115
+ "See root cause above.")
116
+
117
+ def __del__(self):
118
+ self.shutdown()
119
+
120
+ def shutdown(self):
121
+ # Shutdown the process if needed.
122
+ if hasattr(self, "proc") and self.proc.is_alive():
123
+ self.proc.terminate()
124
+ self.proc.join(5)
125
+
126
+ if self.proc.is_alive():
127
+ kill_process_tree(self.proc.pid)
128
+
129
+ # Remove zmq ipc socket files
130
+ ipc_sockets = [self.output_path, self.input_path]
131
+ for ipc_socket in ipc_sockets:
132
+ socket_file = ipc_socket.replace("ipc://", "")
133
+ if os and os.path.exists(socket_file):
134
+ os.remove(socket_file)