@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 564985729a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[ BugFix ] Move `zmq` frontend to IPC instead of TCP (#7222)

Robert Shaw committed Aug 7, 2024 at 12:24 UTC 564985729abc267af281de11f737cfb29b5c0abb
5 files changed +29 -22
vllm/entrypoints/openai/api_server.py
+8 -4
@@ -43,7 +43,7 @@ from vllm.entrypoints.openai.serving_tokenization import (
43 OpenAIServingTokenization)
44 from vllm.logger import init_logger
45 from vllm.usage.usage_lib import UsageContext
46 -from vllm.utils import FlexibleArgumentParser, get_open_port
46 +from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
47 from vllm.version import __version__ as VLLM_VERSION
48
49 TIMEOUT_KEEP_ALIVE = 5 # seconds
@@ -106,16 +106,20 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
106
107 # Otherwise, use the multiprocessing AsyncLLMEngine.
108 else:
109 + # Select random path for IPC.
110 + rpc_path = get_open_zmq_ipc_path()
111 + logger.info("Multiprocessing frontend to use %s for RPC Path.",
112 + rpc_path)
113 +
114 # Start RPCServer in separate process (holds the AsyncLLMEngine).
110 - port = get_open_port(envs.VLLM_RPC_PORT)
115 rpc_server_process = Process(target=run_rpc_server,
116 args=(engine_args,
117 UsageContext.OPENAI_API_SERVER,
114 - port))
118 + rpc_path))
119 rpc_server_process.start()
120
121 # Build RPCClient, which conforms to AsyncEngineClient Protocol.
118 - async_engine_client = AsyncEngineRPCClient(port)
122 + async_engine_client = AsyncEngineRPCClient(rpc_path)
123 await async_engine_client.setup()
124
125 try:
vllm/entrypoints/openai/rpc/client.py
+3 -3
@@ -21,9 +21,9 @@ from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
21
22 class AsyncEngineRPCClient:
23
24 - def __init__(self, port: int):
24 + def __init__(self, rpc_path: str):
25 self.context = zmq.asyncio.Context()
26 - self.path = f"tcp://localhost:{port}"
26 + self.rpc_path = rpc_path
27
28 async def setup(self):
29 """Setup the client before it starts sending server requests."""
@@ -58,7 +58,7 @@ class AsyncEngineRPCClient:
58 # to enable streaming.
59 socket = self.context.socket(zmq.constants.DEALER)
60 try:
61 - socket.connect(self.path)
61 + socket.connect(self.rpc_path)
62 yield socket
63 finally:
64 socket.close()
vllm/entrypoints/openai/rpc/server.py
+4 -6
@@ -20,7 +20,7 @@ logger = init_logger(__name__)
20 class AsyncEngineRPCServer:
21
22 def __init__(self, async_engine_args: AsyncEngineArgs,
23 - usage_context: UsageContext, port: int):
23 + usage_context: UsageContext, rpc_path: str):
24 # Initialize engine first.
25 self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
26 usage_context)
@@ -30,9 +30,7 @@ class AsyncEngineRPCServer:
30
31 # Init socket for readiness state.
32 self.socket = self.context.socket(zmq.constants.ROUTER)
33 - # Note numeric form of localhost should be used for zmq bind(),
34 - # see https://stackoverflow.com/a/8958414
35 - self.socket.bind(f"tcp://127.0.0.1:{port}")
33 + self.socket.bind(rpc_path)
34
35 def cleanup(self):
36 """Cleanup all resources."""
@@ -213,6 +211,6 @@ async def run_server(server: AsyncEngineRPCServer):
211
212
213 def run_rpc_server(async_engine_args: AsyncEngineArgs,
216 - usage_context: UsageContext, port: int):
217 - server = AsyncEngineRPCServer(async_engine_args, usage_context, port)
214 + usage_context: UsageContext, rpc_path: str):
215 + server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
216 asyncio.run(run_server(server))
vllm/envs.py
+6 -5
@@ -1,10 +1,11 @@
1 import os
2 +import tempfile
3 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
4
5 if TYPE_CHECKING:
6 VLLM_HOST_IP: str = ""
7 VLLM_PORT: Optional[int] = None
7 - VLLM_RPC_PORT: int = 5570
8 + VLLM_RPC_BASE_PATH: str = tempfile.gettempdir()
9 VLLM_USE_MODELSCOPE: bool = False
10 VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
11 VLLM_INSTANCE_ID: Optional[str] = None
@@ -142,10 +143,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
143 lambda: int(os.getenv('VLLM_PORT', '0'))
144 if 'VLLM_PORT' in os.environ else None,
145
145 - # used when the frontend api server is running in multi-processing mode,
146 - # to communicate with the backend engine process over ZMQ.
147 - 'VLLM_RPC_PORT':
148 - lambda: int(os.getenv('VLLM_RPC_PORT', '5570')),
146 + # path used for ipc when the frontend api server is running in
147 + # multi-processing mode to communicate with the backend engine process.
148 + 'VLLM_RPC_BASE_PATH':
149 + lambda: os.getenv('VLLM_RPC_BASE_PATH', tempfile.gettempdir()),
150
151 # If true, will load models from ModelScope instead of Hugging Face Hub.
152 # note that the value is true or false, not numbers
vllm/utils.py
+8 -4
@@ -19,6 +19,7 @@ from platform import uname
19 from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
20 Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
21 Union, overload)
22 +from uuid import uuid4
23
24 import numpy as np
25 import numpy.typing as npt
@@ -484,10 +485,13 @@ def get_distributed_init_method(ip: str, port: int) -> str:
485 return f"tcp://[{ip}]:{port}" if ":" in ip else f"tcp://{ip}:{port}"
486
487
487 -def get_open_port(port: Optional[int] = None) -> int:
488 - if port is None:
489 - # Default behavior here is to return a port for multi-gpu communication
490 - port = envs.VLLM_PORT
488 +def get_open_zmq_ipc_path() -> str:
489 + base_rpc_path = envs.VLLM_RPC_BASE_PATH
490 + return f"ipc://{base_rpc_path}/{uuid4()}"
491 +
492 +
493 +def get_open_port() -> int:
494 + port = envs.VLLM_PORT
495 if port is not None:
496 while True:
497 try: