A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Improve choice of Python multiprocessing method (#8823)
Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: youkaichao <youkaichao@126.com>
Russell Bryant committed
Sep 28, 2024 at 21:17 UTC
d1537039ce7e6018db510d0c0d9b0c0fccb62b63
4 files changed
+52
-9
vllm/executor/multiproc_gpu_executor.py
+9
-2
@@ -15,8 +15,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.sequence import ExecuteModelRequest
16
from vllm.triton_utils import maybe_set_triton_cache_manager
17
from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18
- get_distributed_init_method, get_open_port,
19
- get_vllm_instance_id, make_async,
18
+ cuda_is_initialized, get_distributed_init_method,
19
+ get_open_port, get_vllm_instance_id, make_async,
20
update_environment_variables)
21
22
logger = init_logger(__name__)
@@ -122,6 +122,13 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
122
"CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
123
})
124
125
+ if (cuda_is_initialized()
126
+ and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
127
+ logger.warning("CUDA was previously initialized. We must use "
128
+ "the `spawn` multiprocessing start method. Setting "
129
+ "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.")
130
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
131
+
132
cuda_device_count = cuda_device_count_stateless()
133
# Use confusing message for more common TP-only case.
134
assert tensor_parallel_size <= cuda_device_count, (
vllm/executor/multiproc_worker_utils.py
+10
-7
@@ -27,9 +27,6 @@ RESET = '\033[0;0m'
27
28
JOIN_TIMEOUT_S = 2
29
30
-mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
31
-mp = multiprocessing.get_context(mp_method)
32
-
30
31
@dataclass
32
class Result(Generic[T]):
@@ -77,7 +74,7 @@ class ResultHandler(threading.Thread):
74
75
def __init__(self) -> None:
76
super().__init__(daemon=True)
80
- self.result_queue = mp.Queue()
77
+ self.result_queue = get_mp_context().Queue()
78
self.tasks: Dict[uuid.UUID, Union[ResultFuture, asyncio.Future]] = {}
79
80
def run(self):
@@ -147,10 +144,11 @@ class ProcessWorkerWrapper:
144
145
def __init__(self, result_handler: ResultHandler,
146
worker_factory: Callable[[], Any]) -> None:
150
- self._task_queue = mp.Queue()
147
+ self.mp = get_mp_context()
148
+ self._task_queue = self.mp.Queue()
149
self.result_queue = result_handler.result_queue
150
self.tasks = result_handler.tasks
153
- self.process: BaseProcess = mp.Process( # type: ignore[attr-defined]
151
+ self.process: BaseProcess = self.mp.Process( # type: ignore[attr-defined]
152
target=_run_worker_process,
153
name="VllmWorkerProcess",
154
kwargs=dict(
@@ -204,7 +202,7 @@ def _run_worker_process(
202
"""Worker process event loop"""
203
204
# Add process-specific prefix to stdout and stderr
207
- process_name = mp.current_process().name
205
+ process_name = get_mp_context().current_process().name
206
pid = os.getpid()
207
_add_prefix(sys.stdout, process_name, pid)
208
_add_prefix(sys.stderr, process_name, pid)
@@ -269,3 +267,8 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
267
268
file.start_new_line = True # type: ignore[attr-defined]
269
file.write = write_with_prefix # type: ignore[method-assign]
270
+
271
+
272
+def get_mp_context():
273
+ mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
274
+ return multiprocessing.get_context(mp_method)
vllm/scripts.py
+26
@@ -12,8 +12,11 @@ from openai.types.chat import ChatCompletionMessageParam
12
from vllm.engine.arg_utils import EngineArgs
13
from vllm.entrypoints.openai.api_server import run_server
14
from vllm.entrypoints.openai.cli_args import make_arg_parser
15
+from vllm.logger import init_logger
16
from vllm.utils import FlexibleArgumentParser
17
18
+logger = init_logger(__name__)
19
+
20
21
def register_signal_handlers():
22
@@ -114,7 +117,30 @@ def _add_query_options(
117
return parser
118
119
120
+def env_setup():
121
+ # The safest multiprocessing method is `spawn`, as the default `fork` method
122
+ # is not compatible with some accelerators. The default method will be
123
+ # changing in future versions of Python, so we should use it explicitly when
124
+ # possible.
125
+ #
126
+ # We only set it here in the CLI entrypoint, because changing to `spawn`
127
+ # could break some existing code using vLLM as a library. `spawn` will cause
128
+ # unexpected behavior if the code is not protected by
129
+ # `if __name__ == "__main__":`.
130
+ #
131
+ # References:
132
+ # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
133
+ # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
134
+ # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors
135
+ # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders
136
+ if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
137
+ logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'")
138
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
139
+
140
+
141
def main():
142
+ env_setup()
143
+
144
parser = FlexibleArgumentParser(description="vLLM CLI")
145
subparsers = parser.add_subparsers(required=True)
146
vllm/utils.py
+7
@@ -1091,6 +1091,13 @@ def cuda_device_count_stateless() -> int:
1091
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
1092
1093
1094
+def cuda_is_initialized() -> bool:
1095
+ """Check if CUDA is initialized."""
1096
+ if not torch.cuda._is_compiled():
1097
+ return False
1098
+ return torch.cuda.is_initialized()
1099
+
1100
+
1101
def weak_bind(bound_method: Callable[..., Any], ) -> Callable[..., None]:
1102
"""Make an instance method that weakly references
1103
its associated instance and no-ops once that