@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 756b30a5f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core][Test] move local_rank to the last arg with default value(#3711)

[Core][Test] move local_rank to the last arg with default value to keep api compatible (#3711)

youkaichao committed Mar 28, 2024 at 21:19 UTC 756b30a5f30ee08b97243e1077419d8d74442b02
7 files changed +24 -14
tests/distributed/test_comm_ops.py
+3 -3
@@ -24,7 +24,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
24 del os.environ["CUDA_VISIBLE_DEVICES"]
25 device = torch.device(f"cuda:{rank}")
26 torch.cuda.set_device(device)
27 - init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
27 + init_test_distributed_environment(1, tensor_parallel_size, rank,
28 distributed_init_port)
29 num_elements = 8
30 all_tensors = [
@@ -46,7 +46,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
46 del os.environ["CUDA_VISIBLE_DEVICES"]
47 device = torch.device(f"cuda:{rank}")
48 torch.cuda.set_device(device)
49 - init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
49 + init_test_distributed_environment(1, tensor_parallel_size, rank,
50 distributed_init_port)
51 num_dimensions = 3
52 tensor_size = list(range(2, num_dimensions + 2))
@@ -74,7 +74,7 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
74 del os.environ["CUDA_VISIBLE_DEVICES"]
75 device = torch.device(f"cuda:{rank}")
76 torch.cuda.set_device(device)
77 - init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
77 + init_test_distributed_environment(1, tensor_parallel_size, rank,
78 distributed_init_port)
79 test_dict = {
80 "a": torch.arange(8, dtype=torch.float32, device="cuda"),
tests/distributed/test_custom_all_reduce.py
+2 -2
@@ -23,7 +23,7 @@ def graph_allreduce(world_size, rank, distributed_init_port):
23 del os.environ["CUDA_VISIBLE_DEVICES"]
24 device = torch.device(f"cuda:{rank}")
25 torch.cuda.set_device(device)
26 - init_test_distributed_environment(1, world_size, rank, rank,
26 + init_test_distributed_environment(1, world_size, rank,
27 distributed_init_port)
28
29 custom_ar.init_custom_ar()
@@ -58,7 +58,7 @@ def eager_allreduce(world_size, rank, distributed_init_port):
58 del os.environ["CUDA_VISIBLE_DEVICES"]
59 device = torch.device(f"cuda:{rank}")
60 torch.cuda.set_device(device)
61 - init_test_distributed_environment(1, world_size, rank, rank,
61 + init_test_distributed_environment(1, world_size, rank,
62 distributed_init_port)
63
64 sz = 1024
tests/distributed/test_pynccl.py
+2
@@ -14,7 +14,9 @@ def distributed_run(fn, world_size):
14 for i in range(number_of_processes):
15 env = os.environ.copy()
16 env['RANK'] = str(i)
17 + env['LOCAL_RANK'] = str(i)
18 env['WORLD_SIZE'] = str(number_of_processes)
19 + env['LOCAL_WORLD_SIZE'] = str(number_of_processes)
20 env['MASTER_ADDR'] = 'localhost'
21 env['MASTER_PORT'] = '12345'
22 p = multiprocessing.Process(target=fn, args=(env, ))
vllm/model_executor/parallel_utils/pynccl.py
+6 -1
@@ -202,11 +202,11 @@ class NCCLCommunicator:
202 init_method=None,
203 timeout=datetime.timedelta(seconds=10),
204 world_size: int = -1,
205 - local_rank: int = -1,
205 rank: int = -1,
206 store=None,
207 group_name: str = "",
208 pg_options=None,
209 + local_rank: int = -1,
210 ):
211 if not dist.is_initialized():
212 backend = backend or "nccl"
@@ -220,6 +220,11 @@ class NCCLCommunicator:
220 store=store,
221 group_name=group_name,
222 pg_options=pg_options)
223 + self.rank = dist.get_rank()
224 + self.world_size = dist.get_world_size()
225 + if local_rank == -1:
226 + local_rank = self.rank
227 + self.local_rank = local_rank
228 torch.cuda.set_device(local_rank)
229 if rank == 0:
230 self.unique_id = ncclGetUniqueId()
vllm/model_executor/parallel_utils/pynccl_utils.py
+4 -2
@@ -35,8 +35,10 @@ def set_pynccl_stream(stream: torch.cuda.Stream):
35 pass
36
37
38 -def init_process_group(world_size: int, local_rank: int, rank: int,
39 - init_method: str) -> None:
38 +def init_process_group(world_size: int,
39 + rank: int,
40 + init_method: str,
41 + local_rank: int = -1) -> None:
42 assert not is_initialized()
43 global comm
44 logger.info(f"vLLM is using nccl=={ncclGetVersion()}")
vllm/test_utils.py
+3 -3
@@ -8,9 +8,9 @@ from vllm.worker.worker import init_distributed_environment
8 def init_test_distributed_environment(
9 pipeline_parallel_size: int,
10 tensor_parallel_size: int,
11 - local_rank: int,
11 rank: int,
12 distributed_init_port: str,
13 + local_rank: int = -1,
14 ) -> None:
15 parallel_config = ParallelConfig(pipeline_parallel_size,
16 tensor_parallel_size,
@@ -18,9 +18,9 @@ def init_test_distributed_environment(
18 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
19 init_distributed_environment(
20 parallel_config,
21 - local_rank,
21 rank,
23 - distributed_init_method=distributed_init_method)
22 + distributed_init_method=distributed_init_method,
23 + local_rank=local_rank)
24
25
26 def multi_process_tensor_parallel(
vllm/worker/worker.py
+4 -3
@@ -97,8 +97,9 @@ class Worker:
97 raise RuntimeError(
98 f"Not support device type: {self.device_config.device}")
99 # Initialize the distributed environment.
100 - init_distributed_environment(self.parallel_config, self.local_rank,
101 - self.rank, self.distributed_init_method)
100 + init_distributed_environment(self.parallel_config, self.rank,
101 + self.distributed_init_method,
102 + self.local_rank)
103 # Set random seed.
104 set_random_seed(self.model_config.seed)
105
@@ -249,9 +250,9 @@ class Worker:
250
251 def init_distributed_environment(
252 parallel_config: ParallelConfig,
252 - local_rank: int,
253 rank: int,
254 distributed_init_method: Optional[str] = None,
255 + local_rank: int = -1,
256 ) -> None:
257 """Initialize the distributed environment."""
258 if torch.distributed.is_initialized():