A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Add unix domain socket support (#18097)
Signed-off-by: <yyweiss@gmail.com> Signed-off-by: yyw <yyweiss@gmail.com>
yyweiss committed
Aug 9, 2025 at 02:23 UTC
baece8c3d26484b918fa123c884e6ea81237b661
5 files changed
+86
-16
docs/cli/README.md
+3
@@ -29,6 +29,9 @@ Start the vLLM OpenAI Compatible API server.
29
# Specify the port
30
vllm serve meta-llama/Llama-2-7b-hf --port 8100
31
32
+ # Serve over a Unix domain socket
33
+ vllm serve meta-llama/Llama-2-7b-hf --uds /tmp/vllm.sock
34
+
35
# Check with --help for more options
36
# To list all groups
37
vllm serve --help=listgroup
tests/entrypoints/openai/test_uds.py
new
+43
@@ -0,0 +1,43 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from tempfile import TemporaryDirectory
5
+
6
+import httpx
7
+import pytest
8
+
9
+from vllm.version import __version__ as VLLM_VERSION
10
+
11
+from ...utils import RemoteOpenAIServer
12
+
13
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
+
15
+
16
+@pytest.fixture(scope="module")
17
+def server():
18
+ with TemporaryDirectory() as tmpdir:
19
+ args = [
20
+ # use half precision for speed and memory savings in CI environment
21
+ "--dtype",
22
+ "bfloat16",
23
+ "--max-model-len",
24
+ "8192",
25
+ "--enforce-eager",
26
+ "--max-num-seqs",
27
+ "128",
28
+ "--uds",
29
+ f"{tmpdir}/vllm.sock",
30
+ ]
31
+
32
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
33
+ yield remote_server
34
+
35
+
36
+@pytest.mark.asyncio
37
+async def test_show_version(server: RemoteOpenAIServer):
38
+ transport = httpx.HTTPTransport(uds=server.uds)
39
+ client = httpx.Client(transport=transport)
40
+ response = client.get(server.url_for("version"))
41
+ response.raise_for_status()
42
+
43
+ assert response.json() == {"version": VLLM_VERSION}
tests/utils.py
+19
-8
@@ -17,6 +17,7 @@ from pathlib import Path
17
from typing import Any, Callable, Literal, Optional, Union
18
19
import cloudpickle
20
+import httpx
21
import openai
22
import pytest
23
import requests
@@ -88,10 +89,12 @@ class RemoteOpenAIServer:
89
raise ValueError("You have manually specified the port "
90
"when `auto_port=True`.")
91
91
- # Don't mutate the input args
92
- vllm_serve_args = vllm_serve_args + [
93
- "--port", str(get_open_port())
94
- ]
92
+ # No need for a port if using unix sockets
93
+ if "--uds" not in vllm_serve_args:
94
+ # Don't mutate the input args
95
+ vllm_serve_args = vllm_serve_args + [
96
+ "--port", str(get_open_port())
97
+ ]
98
if seed is not None:
99
if "--seed" in vllm_serve_args:
100
raise ValueError("You have manually specified the seed "
@@ -104,8 +107,13 @@ class RemoteOpenAIServer:
107
subparsers = parser.add_subparsers(required=False, dest="subparser")
108
parser = ServeSubcommand().subparser_init(subparsers)
109
args = parser.parse_args(["--model", model, *vllm_serve_args])
107
- self.host = str(args.host or 'localhost')
108
- self.port = int(args.port)
110
+ self.uds = args.uds
111
+ if args.uds:
112
+ self.host = None
113
+ self.port = None
114
+ else:
115
+ self.host = str(args.host or 'localhost')
116
+ self.port = int(args.port)
117
118
self.show_hidden_metrics = \
119
args.show_hidden_metrics_for_version is not None
@@ -150,9 +158,11 @@ class RemoteOpenAIServer:
158
def _wait_for_server(self, *, url: str, timeout: float):
159
# run health check
160
start = time.time()
161
+ client = (httpx.Client(transport=httpx.HTTPTransport(
162
+ uds=self.uds)) if self.uds else requests)
163
while True:
164
try:
155
- if requests.get(url).status_code == 200:
165
+ if client.get(url).status_code == 200:
166
break
167
except Exception:
168
# this exception can only be raised by requests.get,
@@ -170,7 +180,8 @@ class RemoteOpenAIServer:
180
181
@property
182
def url_root(self) -> str:
173
- return f"http://{self.host}:{self.port}"
183
+ return (f"http://{self.uds.split('/')[-1]}"
184
+ if self.uds else f"http://{self.host}:{self.port}")
185
186
def url_for(self, *parts: str) -> str:
187
return self.url_root + "/" + "/".join(parts)
vllm/entrypoints/openai/api_server.py
+19
-8
@@ -1777,6 +1777,12 @@ def create_server_socket(addr: tuple[str, int]) -> socket.socket:
1777
return sock
1778
1779
1780
+def create_server_unix_socket(path: str) -> socket.socket:
1781
+ sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
1782
+ sock.bind(path)
1783
+ return sock
1784
+
1785
+
1786
def validate_api_server_args(args):
1787
valid_tool_parses = ToolParserManager.tool_parsers.keys()
1788
if args.enable_auto_tool_choice \
@@ -1807,8 +1813,11 @@ def setup_server(args):
1813
# workaround to make sure that we bind the port before the engine is set up.
1814
# This avoids race conditions with ray.
1815
# see https://github.com/vllm-project/vllm/issues/8204
1810
- sock_addr = (args.host or "", args.port)
1811
- sock = create_server_socket(sock_addr)
1816
+ if args.uds:
1817
+ sock = create_server_unix_socket(args.uds)
1818
+ else:
1819
+ sock_addr = (args.host or "", args.port)
1820
+ sock = create_server_socket(sock_addr)
1821
1822
# workaround to avoid footguns where uvicorn drops requests with too
1823
# many concurrent requests active
@@ -1820,12 +1829,14 @@ def setup_server(args):
1829
1830
signal.signal(signal.SIGTERM, signal_handler)
1831
1823
- addr, port = sock_addr
1824
- is_ssl = args.ssl_keyfile and args.ssl_certfile
1825
- host_part = f"[{addr}]" if is_valid_ipv6_address(
1826
- addr) else addr or "0.0.0.0"
1827
- listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
1828
-
1832
+ if args.uds:
1833
+ listen_address = f"unix:{args.uds}"
1834
+ else:
1835
+ addr, port = sock_addr
1836
+ is_ssl = args.ssl_keyfile and args.ssl_certfile
1837
+ host_part = f"[{addr}]" if is_valid_ipv6_address(
1838
+ addr) else addr or "0.0.0.0"
1839
+ listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
1840
return listen_address, sock
1841
1842
vllm/entrypoints/openai/cli_args.py
+2
@@ -72,6 +72,8 @@ class FrontendArgs:
72
"""Host name."""
73
port: int = 8000
74
"""Port number."""
75
+ uds: Optional[str] = None
76
+ """Unix domain socket path. If set, host and port arguments are ignored."""
77
uvicorn_log_level: Literal["debug", "info", "warning", "error", "critical",
78
"trace"] = "info"
79
"""Log level for uvicorn."""