A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Test] move local_rank to the last arg with default value(#3711)
[Core][Test] move local_rank to the last arg with default value to keep api compatible (#3711)
youkaichao committed
Mar 28, 2024 at 21:19 UTC
756b30a5f30ee08b97243e1077419d8d74442b02
7 files changed
+24
-14
tests/distributed/test_comm_ops.py
+3
-3
@@ -24,7 +24,7 @@ def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
24
del os.environ["CUDA_VISIBLE_DEVICES"]
25
device = torch.device(f"cuda:{rank}")
26
torch.cuda.set_device(device)
27
- init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
27
+ init_test_distributed_environment(1, tensor_parallel_size, rank,
28
distributed_init_port)
29
num_elements = 8
30
all_tensors = [
@@ -46,7 +46,7 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
46
del os.environ["CUDA_VISIBLE_DEVICES"]
47
device = torch.device(f"cuda:{rank}")
48
torch.cuda.set_device(device)
49
- init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
49
+ init_test_distributed_environment(1, tensor_parallel_size, rank,
50
distributed_init_port)
51
num_dimensions = 3
52
tensor_size = list(range(2, num_dimensions + 2))
@@ -74,7 +74,7 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
74
del os.environ["CUDA_VISIBLE_DEVICES"]
75
device = torch.device(f"cuda:{rank}")
76
torch.cuda.set_device(device)
77
- init_test_distributed_environment(1, tensor_parallel_size, rank, rank,
77
+ init_test_distributed_environment(1, tensor_parallel_size, rank,
78
distributed_init_port)
79
test_dict = {
80
"a": torch.arange(8, dtype=torch.float32, device="cuda"),
tests/distributed/test_custom_all_reduce.py
+2
-2
@@ -23,7 +23,7 @@ def graph_allreduce(world_size, rank, distributed_init_port):
23
del os.environ["CUDA_VISIBLE_DEVICES"]
24
device = torch.device(f"cuda:{rank}")
25
torch.cuda.set_device(device)
26
- init_test_distributed_environment(1, world_size, rank, rank,
26
+ init_test_distributed_environment(1, world_size, rank,
27
distributed_init_port)
28
29
custom_ar.init_custom_ar()
@@ -58,7 +58,7 @@ def eager_allreduce(world_size, rank, distributed_init_port):
58
del os.environ["CUDA_VISIBLE_DEVICES"]
59
device = torch.device(f"cuda:{rank}")
60
torch.cuda.set_device(device)
61
- init_test_distributed_environment(1, world_size, rank, rank,
61
+ init_test_distributed_environment(1, world_size, rank,
62
distributed_init_port)
63
64
sz = 1024
tests/distributed/test_pynccl.py
+2
@@ -14,7 +14,9 @@ def distributed_run(fn, world_size):
14
for i in range(number_of_processes):
15
env = os.environ.copy()
16
env['RANK'] = str(i)
17
+ env['LOCAL_RANK'] = str(i)
18
env['WORLD_SIZE'] = str(number_of_processes)
19
+ env['LOCAL_WORLD_SIZE'] = str(number_of_processes)
20
env['MASTER_ADDR'] = 'localhost'
21
env['MASTER_PORT'] = '12345'
22
p = multiprocessing.Process(target=fn, args=(env, ))
vllm/model_executor/parallel_utils/pynccl.py
+6
-1
@@ -202,11 +202,11 @@ class NCCLCommunicator:
202
init_method=None,
203
timeout=datetime.timedelta(seconds=10),
204
world_size: int = -1,
205
- local_rank: int = -1,
205
rank: int = -1,
206
store=None,
207
group_name: str = "",
208
pg_options=None,
209
+ local_rank: int = -1,
210
):
211
if not dist.is_initialized():
212
backend = backend or "nccl"
@@ -220,6 +220,11 @@ class NCCLCommunicator:
220
store=store,
221
group_name=group_name,
222
pg_options=pg_options)
223
+ self.rank = dist.get_rank()
224
+ self.world_size = dist.get_world_size()
225
+ if local_rank == -1:
226
+ local_rank = self.rank
227
+ self.local_rank = local_rank
228
torch.cuda.set_device(local_rank)
229
if rank == 0:
230
self.unique_id = ncclGetUniqueId()
vllm/model_executor/parallel_utils/pynccl_utils.py
+4
-2
@@ -35,8 +35,10 @@ def set_pynccl_stream(stream: torch.cuda.Stream):
35
pass
36
37
38
-def init_process_group(world_size: int, local_rank: int, rank: int,
39
- init_method: str) -> None:
38
+def init_process_group(world_size: int,
39
+ rank: int,
40
+ init_method: str,
41
+ local_rank: int = -1) -> None:
42
assert not is_initialized()
43
global comm
44
logger.info(f"vLLM is using nccl=={ncclGetVersion()}")
vllm/test_utils.py
+3
-3
@@ -8,9 +8,9 @@ from vllm.worker.worker import init_distributed_environment
8
def init_test_distributed_environment(
9
pipeline_parallel_size: int,
10
tensor_parallel_size: int,
11
- local_rank: int,
11
rank: int,
12
distributed_init_port: str,
13
+ local_rank: int = -1,
14
) -> None:
15
parallel_config = ParallelConfig(pipeline_parallel_size,
16
tensor_parallel_size,
@@ -18,9 +18,9 @@ def init_test_distributed_environment(
18
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
19
init_distributed_environment(
20
parallel_config,
21
- local_rank,
21
rank,
23
- distributed_init_method=distributed_init_method)
22
+ distributed_init_method=distributed_init_method,
23
+ local_rank=local_rank)
24
25
26
def multi_process_tensor_parallel(
vllm/worker/worker.py
+4
-3
@@ -97,8 +97,9 @@ class Worker:
97
raise RuntimeError(
98
f"Not support device type: {self.device_config.device}")
99
# Initialize the distributed environment.
100
- init_distributed_environment(self.parallel_config, self.local_rank,
101
- self.rank, self.distributed_init_method)
100
+ init_distributed_environment(self.parallel_config, self.rank,
101
+ self.distributed_init_method,
102
+ self.local_rank)
103
# Set random seed.
104
set_random_seed(self.model_config.seed)
105
@@ -249,9 +250,9 @@ class Worker:
250
251
def init_distributed_environment(
252
parallel_config: ParallelConfig,
252
- local_rank: int,
253
rank: int,
254
distributed_init_method: Optional[str] = None,
255
+ local_rank: int = -1,
256
) -> None:
257
"""Initialize the distributed environment."""
258
if torch.distributed.is_initialized():