@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a94ca4a5d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] fix OpenAI API server startup with --disable-frontend-multiprocessing (#8537)

Daniele committed Oct 8, 2024 at 18:38 UTC 9a94ca4a5d31c0ba57ca67fc1c252233d3284012
2 files changed +63 -5
tests/entrypoints/openai/test_basic.py
+57 -1
@@ -1,4 +1,5 @@
1 from http import HTTPStatus
2 +from typing import List
3
4 import openai
5 import pytest
@@ -12,8 +13,44 @@ from ...utils import RemoteOpenAIServer
13 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
15
16 +@pytest.fixture(scope='module')
17 +def server_args(request: pytest.FixtureRequest) -> List[str]:
18 + """ Provide extra arguments to the server via indirect parametrization
19 +
20 + Usage:
21 +
22 + >>> @pytest.mark.parametrize(
23 + >>> "server_args",
24 + >>> [
25 + >>> ["--disable-frontend-multiprocessing"],
26 + >>> [
27 + >>> "--model=NousResearch/Hermes-3-Llama-3.1-70B",
28 + >>> "--enable-auto-tool-choice",
29 + >>> ],
30 + >>> ],
31 + >>> indirect=True,
32 + >>> )
33 + >>> def test_foo(server, client):
34 + >>> ...
35 +
36 + This will run `test_foo` twice with servers with:
37 + - `--disable-frontend-multiprocessing`
38 + - `--model=NousResearch/Hermes-3-Llama-3.1-70B --enable-auto-tool-choice`.
39 +
40 + """
41 + if not hasattr(request, "param"):
42 + return []
43 +
44 + val = request.param
45 +
46 + if isinstance(val, str):
47 + return [val]
48 +
49 + return request.param
50 +
51 +
52 @pytest.fixture(scope="module")
16 -def server():
53 +def server(server_args):
54 args = [
55 # use half precision for speed and memory savings in CI environment
56 "--dtype",
@@ -23,6 +60,7 @@ def server():
60 "--enforce-eager",
61 "--max-num-seqs",
62 "128",
63 + *server_args,
64 ]
65
66 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -35,6 +73,15 @@ async def client(server):
73 yield async_client
74
75
76 +@pytest.mark.parametrize(
77 + "server_args",
78 + [
79 + pytest.param([], id="default-frontend-multiprocessing"),
80 + pytest.param(["--disable-frontend-multiprocessing"],
81 + id="disable-frontend-multiprocessing")
82 + ],
83 + indirect=True,
84 +)
85 @pytest.mark.asyncio
86 async def test_show_version(client: openai.AsyncOpenAI):
87 base_url = str(client.base_url)[:-3].strip("/")
@@ -45,6 +92,15 @@ async def test_show_version(client: openai.AsyncOpenAI):
92 assert response.json() == {"version": VLLM_VERSION}
93
94
95 +@pytest.mark.parametrize(
96 + "server_args",
97 + [
98 + pytest.param([], id="default-frontend-multiprocessing"),
99 + pytest.param(["--disable-frontend-multiprocessing"],
100 + id="disable-frontend-multiprocessing")
101 + ],
102 + indirect=True,
103 +)
104 @pytest.mark.asyncio
105 async def test_check_health(client: openai.AsyncOpenAI):
106 base_url = str(client.base_url)[:-3].strip("/")
vllm/entrypoints/openai/api_server.py
+6 -4
@@ -537,8 +537,11 @@ async def run_server(args, **uvicorn_kwargs) -> None:
537 raise KeyError(f"invalid tool call parser: {args.tool_call_parser} "
538 f"(chose from {{ {','.join(valide_tool_parses)} }})")
539
540 - temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
541 - temp_socket.bind(("", args.port))
540 + # workaround to make sure that we bind the port before the engine is set up.
541 + # This avoids race conditions with ray.
542 + # see https://github.com/vllm-project/vllm/issues/8204
543 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
544 + sock.bind(("", args.port))
545
546 def signal_handler(*_) -> None:
547 # Interrupt server on sigterm while initializing
@@ -552,8 +555,6 @@ async def run_server(args, **uvicorn_kwargs) -> None:
555 model_config = await engine_client.get_model_config()
556 init_app_state(engine_client, model_config, app.state, args)
557
555 - temp_socket.close()
556 -
558 shutdown_task = await serve_http(
559 app,
560 host=args.host,
@@ -564,6 +565,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
565 ssl_certfile=args.ssl_certfile,
566 ssl_ca_certs=args.ssl_ca_certs,
567 ssl_cert_reqs=args.ssl_cert_reqs,
568 + fd=sock.fileno(),
569 **uvicorn_kwargs,
570 )
571