A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
preload heavy modules when mp method is forkserver (#22214)
Signed-off-by: Lionel Villard <villard@us.ibm.com>
Lionel Villard committed
Aug 6, 2025 at 23:33 UTC
ad6c655dde487c256292ad85a538cdf5133ee28b
2 files changed
+13
-1
vllm/benchmarks/latency.py
+3
-1
@@ -13,7 +13,6 @@ import numpy as np
13
from tqdm import tqdm
14
15
import vllm.envs as envs
16
-from vllm import LLM, SamplingParams
16
from vllm.benchmarks.lib.utils import (convert_to_pytorch_benchmark_format,
17
write_to_json)
18
from vllm.engine.arg_utils import EngineArgs
@@ -85,6 +84,9 @@ def main(args: argparse.Namespace):
84
"Please set it to a valid path to use torch profiler.")
85
engine_args = EngineArgs.from_cli_args(args)
86
87
+ # Lazy import to avoid importing LLM when the bench command is not selected.
88
+ from vllm import LLM, SamplingParams
89
+
90
# NOTE(woosuk): If the request cannot be processed in a single batch,
91
# the engine will automatically process the request in multiple batches.
92
llm = LLM(**dataclasses.asdict(engine_args))
vllm/entrypoints/openai/api_server.py
+10
@@ -8,6 +8,7 @@ import importlib
8
import inspect
9
import json
10
import multiprocessing
11
+import multiprocessing.forkserver as forkserver
12
import os
13
import signal
14
import socket
@@ -155,6 +156,15 @@ async def build_async_engine_client(
156
client_config: Optional[dict[str, Any]] = None,
157
) -> AsyncIterator[EngineClient]:
158
159
+ if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
160
+ # The executor is expected to be mp.
161
+ # Pre-import heavy modules in the forkserver process
162
+ logger.debug("Setup forkserver with pre-imports")
163
+ multiprocessing.set_start_method('forkserver')
164
+ multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
165
+ forkserver.ensure_running()
166
+ logger.debug("Forkserver setup complete!")
167
+
168
# Context manager to handle engine_client lifecycle
169
# Ensures everything is shutdown and cleaned up on error/exit
170
engine_args = AsyncEngineArgs.from_cli_args(args)