1
+import os
2
+import subprocess
3
+import sys
4
+import time
5
+
6
+import ray
7
+import requests
8
+
9
+from vllm.distributed import (ensure_model_parallel_initialized,
10
+ init_distributed_environment)
11
+from vllm.utils import get_open_port
12
+
13
+# Path to root of repository so that utilities can be imported by ray workers
14
+VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
15
+
16
+
17
+@ray.remote(num_gpus=1)
18
+class ServerRunner:
19
+ MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
20
+
21
+ def __init__(self, args):
22
+ env = os.environ.copy()
23
+ env["PYTHONUNBUFFERED"] = "1"
24
+ self.proc = subprocess.Popen(
25
+ ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
26
+ env=env,
27
+ stdout=sys.stdout,
28
+ stderr=sys.stderr,
29
+ )
30
+ self._wait_for_server()
31
+
32
+ def ready(self):
33
+ return True
34
+
35
+ def _wait_for_server(self):
36
+ # run health check
37
+ start = time.time()
38
+ while True:
39
+ try:
40
+ if requests.get(
41
+ "http://localhost:8000/health").status_code == 200:
42
+ break
43
+ except Exception as err:
44
+ if self.proc.poll() is not None:
45
+ raise RuntimeError("Server exited unexpectedly.") from err
46
+
47
+ time.sleep(0.5)
48
+ if time.time() - start > self.MAX_SERVER_START_WAIT_S:
49
+ raise RuntimeError(
50
+ "Server failed to start in time.") from err
51
+
52
+ def __del__(self):
53
+ if hasattr(self, "proc"):
54
+ self.proc.terminate()
55
+
56
+
57
+def init_test_distributed_environment(
58
+ tp_size: int,
59
+ pp_size: int,
60
+ rank: int,
61
+ distributed_init_port: str,
62
+ local_rank: int = -1,
63
+) -> None:
64
+ distributed_init_method = f"tcp://localhost:{distributed_init_port}"
65
+ init_distributed_environment(
66
+ world_size=pp_size * tp_size,
67
+ rank=rank,
68
+ distributed_init_method=distributed_init_method,
69
+ local_rank=local_rank)
70
+ ensure_model_parallel_initialized(tp_size, pp_size)
71
+
72
+
73
+def multi_process_tensor_parallel(
74
+ tp_size: int,
75
+ pp_size: int,
76
+ test_target,
77
+) -> None:
78
+ # Using ray helps debugging the error when it failed
79
+ # as compared to multiprocessing.
80
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
81
+
82
+ distributed_init_port = get_open_port()
83
+ refs = []
84
+ for rank in range(tp_size * pp_size):
85
+ refs.append(
86
+ test_target.remote(tp_size, pp_size, rank, distributed_init_port))
87
+ ray.get(refs)
88
+
89
+ ray.shutdown()