A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[ BugFix ] Move `zmq` frontend to IPC instead of TCP (#7222)
Robert Shaw committed
Aug 7, 2024 at 12:24 UTC
564985729abc267af281de11f737cfb29b5c0abb
5 files changed
+29
-22
vllm/entrypoints/openai/api_server.py
+8
-4
@@ -43,7 +43,7 @@ from vllm.entrypoints.openai.serving_tokenization import (
43
OpenAIServingTokenization)
44
from vllm.logger import init_logger
45
from vllm.usage.usage_lib import UsageContext
46
-from vllm.utils import FlexibleArgumentParser, get_open_port
46
+from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
47
from vllm.version import __version__ as VLLM_VERSION
48
49
TIMEOUT_KEEP_ALIVE = 5 # seconds
@@ -106,16 +106,20 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
106
107
# Otherwise, use the multiprocessing AsyncLLMEngine.
108
else:
109
+ # Select random path for IPC.
110
+ rpc_path = get_open_zmq_ipc_path()
111
+ logger.info("Multiprocessing frontend to use %s for RPC Path.",
112
+ rpc_path)
113
+
114
# Start RPCServer in separate process (holds the AsyncLLMEngine).
110
- port = get_open_port(envs.VLLM_RPC_PORT)
115
rpc_server_process = Process(target=run_rpc_server,
116
args=(engine_args,
117
UsageContext.OPENAI_API_SERVER,
114
- port))
118
+ rpc_path))
119
rpc_server_process.start()
120
121
# Build RPCClient, which conforms to AsyncEngineClient Protocol.
118
- async_engine_client = AsyncEngineRPCClient(port)
122
+ async_engine_client = AsyncEngineRPCClient(rpc_path)
123
await async_engine_client.setup()
124
125
try:
vllm/entrypoints/openai/rpc/client.py
+3
-3
@@ -21,9 +21,9 @@ from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
21
22
class AsyncEngineRPCClient:
23
24
- def __init__(self, port: int):
24
+ def __init__(self, rpc_path: str):
25
self.context = zmq.asyncio.Context()
26
- self.path = f"tcp://localhost:{port}"
26
+ self.rpc_path = rpc_path
27
28
async def setup(self):
29
"""Setup the client before it starts sending server requests."""
@@ -58,7 +58,7 @@ class AsyncEngineRPCClient:
58
# to enable streaming.
59
socket = self.context.socket(zmq.constants.DEALER)
60
try:
61
- socket.connect(self.path)
61
+ socket.connect(self.rpc_path)
62
yield socket
63
finally:
64
socket.close()
vllm/entrypoints/openai/rpc/server.py
+4
-6
@@ -20,7 +20,7 @@ logger = init_logger(__name__)
20
class AsyncEngineRPCServer:
21
22
def __init__(self, async_engine_args: AsyncEngineArgs,
23
- usage_context: UsageContext, port: int):
23
+ usage_context: UsageContext, rpc_path: str):
24
# Initialize engine first.
25
self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
26
usage_context)
@@ -30,9 +30,7 @@ class AsyncEngineRPCServer:
30
31
# Init socket for readiness state.
32
self.socket = self.context.socket(zmq.constants.ROUTER)
33
- # Note numeric form of localhost should be used for zmq bind(),
34
- # see https://stackoverflow.com/a/8958414
35
- self.socket.bind(f"tcp://127.0.0.1:{port}")
33
+ self.socket.bind(rpc_path)
34
35
def cleanup(self):
36
"""Cleanup all resources."""
@@ -213,6 +211,6 @@ async def run_server(server: AsyncEngineRPCServer):
211
212
213
def run_rpc_server(async_engine_args: AsyncEngineArgs,
216
- usage_context: UsageContext, port: int):
217
- server = AsyncEngineRPCServer(async_engine_args, usage_context, port)
214
+ usage_context: UsageContext, rpc_path: str):
215
+ server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
216
asyncio.run(run_server(server))
vllm/envs.py
+6
-5
@@ -1,10 +1,11 @@
1
import os
2
+import tempfile
3
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
4
5
if TYPE_CHECKING:
6
VLLM_HOST_IP: str = ""
7
VLLM_PORT: Optional[int] = None
7
- VLLM_RPC_PORT: int = 5570
8
+ VLLM_RPC_BASE_PATH: str = tempfile.gettempdir()
9
VLLM_USE_MODELSCOPE: bool = False
10
VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
11
VLLM_INSTANCE_ID: Optional[str] = None
@@ -142,10 +143,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
143
lambda: int(os.getenv('VLLM_PORT', '0'))
144
if 'VLLM_PORT' in os.environ else None,
145
145
- # used when the frontend api server is running in multi-processing mode,
146
- # to communicate with the backend engine process over ZMQ.
147
- 'VLLM_RPC_PORT':
148
- lambda: int(os.getenv('VLLM_RPC_PORT', '5570')),
146
+ # path used for ipc when the frontend api server is running in
147
+ # multi-processing mode to communicate with the backend engine process.
148
+ 'VLLM_RPC_BASE_PATH':
149
+ lambda: os.getenv('VLLM_RPC_BASE_PATH', tempfile.gettempdir()),
150
151
# If true, will load models from ModelScope instead of Hugging Face Hub.
152
# note that the value is true or false, not numbers
vllm/utils.py
+8
-4
@@ -19,6 +19,7 @@ from platform import uname
19
from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
20
Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
21
Union, overload)
22
+from uuid import uuid4
23
24
import numpy as np
25
import numpy.typing as npt
@@ -484,10 +485,13 @@ def get_distributed_init_method(ip: str, port: int) -> str:
485
return f"tcp://[{ip}]:{port}" if ":" in ip else f"tcp://{ip}:{port}"
486
487
487
-def get_open_port(port: Optional[int] = None) -> int:
488
- if port is None:
489
- # Default behavior here is to return a port for multi-gpu communication
490
- port = envs.VLLM_PORT
488
+def get_open_zmq_ipc_path() -> str:
489
+ base_rpc_path = envs.VLLM_RPC_BASE_PATH
490
+ return f"ipc://{base_rpc_path}/{uuid4()}"
491
+
492
+
493
+def get_open_port() -> int:
494
+ port = envs.VLLM_PORT
495
if port is not None:
496
while True:
497
try: