A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Test] fix function name typo in custom allreduce (#4750)
youkaichao committed
May 10, 2024 at 15:14 UTC
4e12131089f192334f6e09c8fe5cd85af1e25327
2 files changed
+6
-2
tests/distributed/test_custom_all_reduce.py
+2
-2
@@ -25,7 +25,7 @@ def graph_allreduce(world_size, rank, distributed_init_port):
25
init_test_distributed_environment(1, world_size, rank,
26
distributed_init_port)
27
28
- custom_all_reduce.init_custom_all_reduce()
28
+ custom_all_reduce.init_custom_ar()
29
for sz in test_sizes:
30
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
31
with custom_all_reduce.capture():
@@ -61,7 +61,7 @@ def eager_allreduce(world_size, rank, distributed_init_port):
61
distributed_init_port)
62
63
sz = 1024
64
- custom_all_reduce.init_custom_all_reduce()
64
+ custom_all_reduce.init_custom_ar()
65
fa = custom_all_reduce.get_handle()
66
inp = torch.ones(sz, dtype=torch.float32, device=device)
67
out = fa.all_reduce_unreg(inp)
vllm/distributed/device_communicators/custom_all_reduce.py
+4
@@ -52,6 +52,10 @@ def init_custom_ar() -> None:
52
"current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53
" is set.")
54
return
55
+
56
+ # we only use a subset of GPUs here
57
+ # so we only need to check the nvlink connectivity of these GPUs
58
+ num_dev = world_size
59
# test nvlink first, this will filter out most of the cases
60
# where custom allreduce is not supported
61
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES