@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c5b7af152
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[distributed][misc] use fork by default for mp (#5669)

youkaichao committed Jun 20, 2024 at 17:06 UTC 6c5b7af1525a2013d7b1806dd6c0c9a53404be6d
3 files changed +38 -3
.buildkite/test-pipeline.yaml
+9
@@ -37,6 +37,9 @@ steps:
37 working_dir: "/vllm-workspace/tests"
38 num_gpus: 2
39 commands:
40 + # FIXIT: find out which code initialize cuda before running the test
41 + # before the fix, we need to use spawn to test it
42 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
43 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
44 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
45 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -55,6 +58,9 @@ steps:
58 working_dir: "/vllm-workspace/tests"
59 num_gpus: 4
60 commands:
61 + # FIXIT: find out which code initialize cuda before running the test
62 + # before the fix, we need to use spawn to test it
63 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
64 - pytest -v -s distributed/test_pynccl.py
65 # We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
66 # See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
@@ -145,6 +151,9 @@ steps:
151 num_gpus: 4
152 # This test runs llama 13B, so it is required to run on 4 GPUs.
153 commands:
154 + # FIXIT: find out which code initialize cuda before running the test
155 + # before the fix, we need to use spawn to test it
156 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
157 - pytest -v -s -x lora/test_long_context.py
158
159 - label: Tensorizer Test
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+27 -1
@@ -1,6 +1,9 @@
1 import ctypes
2 import json
3 import os
4 +import pickle
5 +import subprocess
6 +import sys
7 from itertools import product
8 from typing import Dict, List, Optional, Sequence
9
@@ -198,7 +201,25 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
201 ids = list(range(num_dev))
202 # batch of all pairs of GPUs
203 batch_src, batch_tgt = zip(*list(product(ids, ids)))
201 - result = can_actually_p2p(batch_src, batch_tgt)
204 + # NOTE: we use `subprocess` rather than `multiprocessing` here
205 + # because the caller might not have `if __name__ == "__main__":`,
206 + # in that case we cannot use spawn method in multiprocessing.
207 + # However, `can_actually_p2p` requires spawn method.
208 + # The fix is, we use `subprocess` to call the function,
209 + # where we have `if __name__ == "__main__":` in this file.
210 + input_bytes = pickle.dumps((batch_src, batch_tgt))
211 + returned = subprocess.run([sys.executable, __file__],
212 + input=input_bytes,
213 + capture_output=True)
214 + # check if the subprocess is successful
215 + try:
216 + returned.check_returncode()
217 + except Exception as e:
218 + # wrap raised exception to provide more information
219 + raise RuntimeError(
220 + f"Error happened when batch testing "
221 + f"peer-to-peer access from {batch_src} to {batch_tgt}") from e
222 + result = pickle.loads(returned.stdout)
223 for _i, _j, r in zip(batch_src, batch_tgt, result):
224 cache[f"{_i}->{_j}"] = r
225 with open(path, "w") as f:
@@ -213,3 +234,8 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
234
235
236 __all__ = ["gpu_p2p_access_check"]
237 +
238 +if __name__ == "__main__":
239 + batch_src, batch_tgt = pickle.loads(sys.stdin.buffer.read())
240 + result = can_actually_p2p(batch_src, batch_tgt)
241 + sys.stdout.buffer.write(pickle.dumps(result))
vllm/envs.py
+2 -2
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
29 VLLM_CPU_KVCACHE_SPACE: int = 0
30 VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
31 VLLM_USE_RAY_COMPILED_DAG: bool = False
32 - VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
32 + VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
33 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
34 VLLM_TARGET_DEVICE: str = "cuda"
35 MAX_JOBS: Optional[str] = None
@@ -212,7 +212,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
212 # Use dedicated multiprocess context for workers.
213 # Both spawn and fork work
214 "VLLM_WORKER_MULTIPROC_METHOD":
215 - lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn"),
215 + lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "fork"),
216
217 # Timeout for fetching images when serving multimodal models
218 # Default is 5 seconds