@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4e12131089
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Test] fix function name typo in custom allreduce (#4750)

youkaichao committed May 10, 2024 at 15:14 UTC 4e12131089f192334f6e09c8fe5cd85af1e25327
2 files changed +6 -2
tests/distributed/test_custom_all_reduce.py
+2 -2
@@ -25,7 +25,7 @@ def graph_allreduce(world_size, rank, distributed_init_port):
25 init_test_distributed_environment(1, world_size, rank,
26 distributed_init_port)
27
28 - custom_all_reduce.init_custom_all_reduce()
28 + custom_all_reduce.init_custom_ar()
29 for sz in test_sizes:
30 for dtype in [torch.float32, torch.float16, torch.bfloat16]:
31 with custom_all_reduce.capture():
@@ -61,7 +61,7 @@ def eager_allreduce(world_size, rank, distributed_init_port):
61 distributed_init_port)
62
63 sz = 1024
64 - custom_all_reduce.init_custom_all_reduce()
64 + custom_all_reduce.init_custom_ar()
65 fa = custom_all_reduce.get_handle()
66 inp = torch.ones(sz, dtype=torch.float32, device=device)
67 out = fa.all_reduce_unreg(inp)
vllm/distributed/device_communicators/custom_all_reduce.py
+4
@@ -52,6 +52,10 @@ def init_custom_ar() -> None:
52 "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53 " is set.")
54 return
55 +
56 + # we only use a subset of GPUs here
57 + # so we only need to check the nvlink connectivity of these GPUs
58 + num_dev = world_size
59 # test nvlink first, this will filter out most of the cases
60 # where custom allreduce is not supported
61 cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES