A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 00:45 UTC
[Frontend] Move APIServerProcessManager target server fn (#38115)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Mar 25, 2026 at 11:14 UTC
72cad44d3c6eaac69d8353981b9f2b4439d12ecd
3 files changed
+26
-30
tests/entrypoints/test_api_server_process_manager.py
+1
-3
@@ -89,9 +89,7 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
89
assert not proc.is_alive()
90
91
92
-@patch(
93
- "vllm.entrypoints.cli.serve.run_api_server_worker_proc", mock_run_api_server_worker
94
-)
92
+@patch("vllm.v1.utils.run_api_server_worker_proc", mock_run_api_server_worker)
93
def test_wait_for_completion_or_failure(api_server_args):
94
"""Test that wait_for_completion_or_failure works with failures."""
95
global WORKER_RUNTIME_SECONDS
vllm/entrypoints/cli/serve.py
+1
-23
@@ -10,18 +10,13 @@ import uvloop
10
import vllm
11
import vllm.envs as envs
12
from vllm.entrypoints.cli.types import CLISubcommand
13
-from vllm.entrypoints.openai.api_server import (
14
- run_server,
15
- run_server_worker,
16
- setup_server,
17
-)
13
+from vllm.entrypoints.openai.api_server import run_server, setup_server
14
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
15
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
16
from vllm.logger import init_logger
17
from vllm.usage.usage_lib import UsageContext
18
from vllm.utils.argparse_utils import FlexibleArgumentParser
19
from vllm.utils.network_utils import get_tcp_uri
24
-from vllm.utils.system_utils import decorate_logs, set_process_title
20
from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
21
from vllm.v1.executor import Executor
22
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
@@ -293,7 +288,6 @@ def run_multi_api_server(args: argparse.Namespace):
288
) as (local_engine_manager, coordinator, addresses, tensor_queue):
289
# Construct common args for the APIServerProcessManager up-front.
290
api_server_manager_kwargs = dict(
296
- target_server_fn=run_api_server_worker_proc,
291
listen_address=listen_address,
292
sock=sock,
293
args=args,
@@ -346,19 +340,3 @@ def run_multi_api_server(args: argparse.Namespace):
340
local_engine_manager.shutdown(timeout=to_timeout(shutdown_by))
341
if coordinator:
342
coordinator.shutdown(timeout=to_timeout(shutdown_by))
349
-
350
-
351
-def run_api_server_worker_proc(
352
- listen_address, sock, args, client_config=None, **uvicorn_kwargs
353
-) -> None:
354
- """Entrypoint for individual API server worker processes."""
355
- client_config = client_config or {}
356
- server_index = client_config.get("client_index", 0)
357
-
358
- # Set process title and add process-specific prefix to stdout and stderr.
359
- set_process_title("APIServer", str(server_index))
360
- decorate_logs()
361
-
362
- uvloop.run(
363
- run_server_worker(listen_address, sock, args, client_config, **uvicorn_kwargs)
364
- )
vllm/v1/utils.py
+24
-4
@@ -21,13 +21,14 @@ from typing import (
21
)
22
23
import torch
24
+import uvloop
25
from torch.autograd.profiler import record_function
26
27
import vllm.envs as envs
28
from vllm.logger import init_logger
29
from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message
30
from vllm.utils.network_utils import get_open_port, get_open_zmq_ipc_path, get_tcp_uri
30
-from vllm.utils.system_utils import kill_process_tree
31
+from vllm.utils.system_utils import decorate_logs, kill_process_tree, set_process_title
32
from vllm.v1.core.sched.output import SchedulerOutput
33
34
if TYPE_CHECKING:
@@ -166,20 +167,20 @@ class APIServerProcessManager:
167
168
def __init__(
169
self,
169
- target_server_fn: Callable,
170
listen_address: str,
171
sock: Any,
172
args: argparse.Namespace,
173
num_servers: int,
174
input_addresses: list[str],
175
output_addresses: list[str],
176
+ target_server_fn: Callable | None = None,
177
stats_update_address: str | None = None,
178
tensor_queue: Queue | None = None,
179
):
180
"""Initialize and start API server worker processes.
181
182
Args:
182
- target_server_fn: Function to call for each API server process
183
+ target_server_fn: Override function to call for each API server process
184
listen_address: Address to listen for client connections
185
sock: Socket for client connections
186
args: Command line arguments
@@ -212,7 +213,7 @@ class APIServerProcessManager:
213
client_config["tensor_queue"] = tensor_queue
214
215
proc = spawn_context.Process(
215
- target=target_server_fn,
216
+ target=target_server_fn or run_api_server_worker_proc,
217
name=f"ApiServer_{i}",
218
args=(listen_address, sock, args, client_config),
219
)
@@ -231,6 +232,25 @@ class APIServerProcessManager:
232
shutdown(self.processes, timeout=timeout)
233
234
235
+def run_api_server_worker_proc(
236
+ listen_address, sock, args, client_config=None, **uvicorn_kwargs
237
+) -> None:
238
+ """Entrypoint for individual API server worker processes."""
239
+
240
+ from vllm.entrypoints.openai.api_server import run_server_worker
241
+
242
+ client_config = client_config or {}
243
+ server_index = client_config.get("client_index", 0)
244
+
245
+ # Set process title and add process-specific prefix to stdout and stderr.
246
+ set_process_title("APIServer", str(server_index))
247
+ decorate_logs()
248
+
249
+ uvloop.run(
250
+ run_server_worker(listen_address, sock, args, client_config, **uvicorn_kwargs)
251
+ )
252
+
253
+
254
def wait_for_completion_or_failure(
255
api_server_manager: APIServerProcessManager,
256
engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]