A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] fix OpenAI API server startup with --disable-frontend-multiprocessing (#8537)
Daniele committed
Oct 8, 2024 at 18:38 UTC
9a94ca4a5d31c0ba57ca67fc1c252233d3284012
2 files changed
+63
-5
tests/entrypoints/openai/test_basic.py
+57
-1
@@ -1,4 +1,5 @@
1
from http import HTTPStatus
2
+from typing import List
3
4
import openai
5
import pytest
@@ -12,8 +13,44 @@ from ...utils import RemoteOpenAIServer
13
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
15
16
+@pytest.fixture(scope='module')
17
+def server_args(request: pytest.FixtureRequest) -> List[str]:
18
+ """ Provide extra arguments to the server via indirect parametrization
19
+
20
+ Usage:
21
+
22
+ >>> @pytest.mark.parametrize(
23
+ >>> "server_args",
24
+ >>> [
25
+ >>> ["--disable-frontend-multiprocessing"],
26
+ >>> [
27
+ >>> "--model=NousResearch/Hermes-3-Llama-3.1-70B",
28
+ >>> "--enable-auto-tool-choice",
29
+ >>> ],
30
+ >>> ],
31
+ >>> indirect=True,
32
+ >>> )
33
+ >>> def test_foo(server, client):
34
+ >>> ...
35
+
36
+ This will run `test_foo` twice with servers with:
37
+ - `--disable-frontend-multiprocessing`
38
+ - `--model=NousResearch/Hermes-3-Llama-3.1-70B --enable-auto-tool-choice`.
39
+
40
+ """
41
+ if not hasattr(request, "param"):
42
+ return []
43
+
44
+ val = request.param
45
+
46
+ if isinstance(val, str):
47
+ return [val]
48
+
49
+ return request.param
50
+
51
+
52
@pytest.fixture(scope="module")
16
-def server():
53
+def server(server_args):
54
args = [
55
# use half precision for speed and memory savings in CI environment
56
"--dtype",
@@ -23,6 +60,7 @@ def server():
60
"--enforce-eager",
61
"--max-num-seqs",
62
"128",
63
+ *server_args,
64
]
65
66
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -35,6 +73,15 @@ async def client(server):
73
yield async_client
74
75
76
+@pytest.mark.parametrize(
77
+ "server_args",
78
+ [
79
+ pytest.param([], id="default-frontend-multiprocessing"),
80
+ pytest.param(["--disable-frontend-multiprocessing"],
81
+ id="disable-frontend-multiprocessing")
82
+ ],
83
+ indirect=True,
84
+)
85
@pytest.mark.asyncio
86
async def test_show_version(client: openai.AsyncOpenAI):
87
base_url = str(client.base_url)[:-3].strip("/")
@@ -45,6 +92,15 @@ async def test_show_version(client: openai.AsyncOpenAI):
92
assert response.json() == {"version": VLLM_VERSION}
93
94
95
+@pytest.mark.parametrize(
96
+ "server_args",
97
+ [
98
+ pytest.param([], id="default-frontend-multiprocessing"),
99
+ pytest.param(["--disable-frontend-multiprocessing"],
100
+ id="disable-frontend-multiprocessing")
101
+ ],
102
+ indirect=True,
103
+)
104
@pytest.mark.asyncio
105
async def test_check_health(client: openai.AsyncOpenAI):
106
base_url = str(client.base_url)[:-3].strip("/")
vllm/entrypoints/openai/api_server.py
+6
-4
@@ -537,8 +537,11 @@ async def run_server(args, **uvicorn_kwargs) -> None:
537
raise KeyError(f"invalid tool call parser: {args.tool_call_parser} "
538
f"(chose from {{ {','.join(valide_tool_parses)} }})")
539
540
- temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
541
- temp_socket.bind(("", args.port))
540
+ # workaround to make sure that we bind the port before the engine is set up.
541
+ # This avoids race conditions with ray.
542
+ # see https://github.com/vllm-project/vllm/issues/8204
543
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
544
+ sock.bind(("", args.port))
545
546
def signal_handler(*_) -> None:
547
# Interrupt server on sigterm while initializing
@@ -552,8 +555,6 @@ async def run_server(args, **uvicorn_kwargs) -> None:
555
model_config = await engine_client.get_model_config()
556
init_app_state(engine_client, model_config, app.state, args)
557
555
- temp_socket.close()
556
-
558
shutdown_task = await serve_http(
559
app,
560
host=args.host,
@@ -564,6 +565,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
565
ssl_certfile=args.ssl_certfile,
566
ssl_ca_certs=args.ssl_ca_certs,
567
ssl_cert_reqs=args.ssl_cert_reqs,
568
+ fd=sock.fileno(),
569
**uvicorn_kwargs,
570
)
571