@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 515386ef3c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Support multi-node inference(eager and cuda graph) (#3686)

Roy committed Mar 29, 2024 at 06:01 UTC 515386ef3cacb44a2bcfab9d66eaee6143d94e95
7 files changed +25 -22
tests/distributed/test_comm_ops.py
+3 -3
@@ -24,7 +24,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
24 del os.environ["CUDA_VISIBLE_DEVICES"]
25 device = torch.device(f"cuda:{rank}")
26 torch.cuda.set_device(device)
27 - init_test_distributed_environment(1, tensor_parallel_size, rank,
27 + init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
28 distributed_init_port)
29 num_elements = 8
30 all_tensors = [
@@ -46,7 +46,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
46 del os.environ["CUDA_VISIBLE_DEVICES"]
47 device = torch.device(f"cuda:{rank}")
48 torch.cuda.set_device(device)
49 - init_test_distributed_environment(1, tensor_parallel_size, rank,
49 + init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
50 distributed_init_port)
51 num_dimensions = 3
52 tensor_size = list(range(2, num_dimensions + 2))
@@ -74,7 +74,7 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
74 del os.environ["CUDA_VISIBLE_DEVICES"]
75 device = torch.device(f"cuda:{rank}")
76 torch.cuda.set_device(device)
77 - init_test_distributed_environment(1, tensor_parallel_size, rank,
77 + init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
78 distributed_init_port)
79 test_dict = {
80 "a": torch.arange(8, dtype=torch.float32, device="cuda"),
tests/distributed/test_custom_all_reduce.py
+2 -2
@@ -23,7 +23,7 @@ def graph_allreduce(world_size, rank, distributed_init_port):
23 del os.environ["CUDA_VISIBLE_DEVICES"]
24 device = torch.device(f"cuda:{rank}")
25 torch.cuda.set_device(device)
26 - init_test_distributed_environment(1, world_size, rank,
26 + init_test_distributed_environment(1, world_size, rank, rank,
27 distributed_init_port)
28
29 custom_ar.init_custom_ar()
@@ -58,7 +58,7 @@ def eager_allreduce(world_size, rank, distributed_init_port):
58 del os.environ["CUDA_VISIBLE_DEVICES"]
59 device = torch.device(f"cuda:{rank}")
60 torch.cuda.set_device(device)
61 - init_test_distributed_environment(1, world_size, rank,
61 + init_test_distributed_environment(1, world_size, rank, rank,
62 distributed_init_port)
63
64 sz = 1024
vllm/executor/ray_gpu_executor.py
-2
@@ -188,8 +188,6 @@ class RayGPUExecutor(ExecutorBase):
188 is_driver_worker=True,
189 )
190
191 - # FIXME(woosuk): We are not properly initializing pynccl when
192 - # we have multiple nodes.
191 self._run_workers("init_device")
192 self._run_workers(
193 "load_model",
vllm/model_executor/parallel_utils/pynccl.py
+8 -10
@@ -202,6 +202,7 @@ class NCCLCommunicator:
202 init_method=None,
203 timeout=datetime.timedelta(seconds=10),
204 world_size: int = -1,
205 + local_rank: int = -1,
206 rank: int = -1,
207 store=None,
208 group_name: str = "",
@@ -219,25 +220,22 @@ class NCCLCommunicator:
220 store=store,
221 group_name=group_name,
222 pg_options=pg_options)
222 - self.world_size = dist.get_world_size()
223 - self.rank = dist.get_rank()
224 - torch.cuda.set_device(self.rank)
225 - if self.rank == 0:
223 + torch.cuda.set_device(local_rank)
224 + if rank == 0:
225 self.unique_id = ncclGetUniqueId()
226 else:
227 self.unique_id = NcclUniqueId()
229 - tensor = torch.ByteTensor(list(self.unique_id.internal)).cuda(
230 - self.rank)
228 + tensor = torch.ByteTensor(list(
229 + self.unique_id.internal)).cuda(local_rank)
230 dist.broadcast(tensor, src=0)
231 byte_list = tensor.cpu().tolist()
233 - self.unique_id = NcclUniqueId()
232 for i, byte in enumerate(byte_list):
233 self.unique_id.internal[i] = byte
234 self.comm = ctypes.c_void_p()
237 - result = _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
238 - self.unique_id, self.rank)
235 + result = _c_ncclCommInitRank(ctypes.byref(self.comm), world_size,
236 + self.unique_id, rank)
237 assert result == 0
240 - self.stream = torch.cuda.Stream(device=f"cuda:{self.rank}")
238 + self.stream = torch.cuda.Stream(device=f"cuda:{local_rank}")
239
240 def all_reduce(self,
241 tensor: torch.Tensor,
vllm/model_executor/parallel_utils/pynccl_utils.py
+3 -1
@@ -36,11 +36,13 @@ def set_pynccl_stream(stream: torch.cuda.Stream):
36 pass
37
38
39 -def init_process_group(world_size: int, rank: int, init_method: str) -> None:
39 +def init_process_group(world_size: int, local_rank: int, rank: int,
40 + init_method: str) -> None:
41 assert not is_initialized()
42 global comm
43 comm = NCCLCommunicator(init_method=init_method,
44 world_size=world_size,
45 + local_rank=local_rank,
46 rank=rank)
47
48
vllm/test_utils.py
+5 -1
@@ -8,6 +8,7 @@ from vllm.worker.worker import init_distributed_environment
8 def init_test_distributed_environment(
9 pipeline_parallel_size: int,
10 tensor_parallel_size: int,
11 + local_rank: int,
12 rank: int,
13 distributed_init_port: str,
14 ) -> None:
@@ -16,7 +17,10 @@ def init_test_distributed_environment(
17 worker_use_ray=True)
18 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
19 init_distributed_environment(
19 - parallel_config, rank, distributed_init_method=distributed_init_method)
20 + parallel_config,
21 + local_rank,
22 + rank,
23 + distributed_init_method=distributed_init_method)
24
25
26 def multi_process_tensor_parallel(
vllm/worker/worker.py
+4 -3
@@ -97,8 +97,8 @@ class Worker:
97 raise RuntimeError(
98 f"Not support device type: {self.device_config.device}")
99 # Initialize the distributed environment.
100 - init_distributed_environment(self.parallel_config, self.rank,
101 - self.distributed_init_method)
100 + init_distributed_environment(self.parallel_config, self.local_rank,
101 + self.rank, self.distributed_init_method)
102 # Set random seed.
103 set_random_seed(self.model_config.seed)
104
@@ -249,6 +249,7 @@ class Worker:
249
250 def init_distributed_environment(
251 parallel_config: ParallelConfig,
252 + local_rank: int,
253 rank: int,
254 distributed_init_method: Optional[str] = None,
255 ) -> None:
@@ -282,9 +283,9 @@ def init_distributed_environment(
283 elif parallel_config.world_size > 1:
284 # NOTE(woosuk): We don't initialize pynccl process group when world size
285 # is 1.
285 - # TODO(woosuk): Support multi-node connection.
286 pynccl_utils.init_process_group(
287 world_size=parallel_config.world_size,
288 + local_rank=local_rank,
289 rank=rank,
290 init_method=distributed_init_method,
291 )