A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[doc] update test script to include cudagraph (#7501)
youkaichao committed
Aug 13, 2024 at 21:52 UTC
199adbb7cfb7e8185c9418de56db006f8cf1c294
1 file changed
+37
-2
docs/source/getting_started/debugging.rst
+37
-2
@@ -30,24 +30,59 @@ Here are some common issues that can cause hangs:
30
31
.. code-block:: python
32
33
+ # Test PyTorch NCCL
34
import torch
35
import torch.distributed as dist
36
dist.init_process_group(backend="nccl")
37
local_rank = dist.get_rank() % torch.cuda.device_count()
37
- data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}")
38
+ torch.cuda.set_device(local_rank)
39
+ data = torch.FloatTensor([1,] * 128).to("cuda")
40
dist.all_reduce(data, op=dist.ReduceOp.SUM)
41
torch.cuda.synchronize()
42
value = data.mean().item()
43
world_size = dist.get_world_size()
44
assert value == world_size, f"Expected {world_size}, got {value}"
45
46
+ print("PyTorch NCCL is successful!")
47
+
48
+ # Test PyTorch GLOO
49
gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo")
50
cpu_data = torch.FloatTensor([1,] * 128)
51
dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group)
52
value = cpu_data.mean().item()
53
assert value == world_size, f"Expected {world_size}, got {value}"
54
50
- print("sanity check is successful!")
55
+ print("PyTorch GLOO is successful!")
56
+
57
+ # Test vLLM NCCL, with cuda graph
58
+ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
59
+
60
+ pynccl = PyNcclCommunicator(group=gloo_group, device=local_rank)
61
+ pynccl.disabled = False
62
+
63
+ s = torch.cuda.Stream()
64
+ with torch.cuda.stream(s):
65
+ data.fill_(1)
66
+ pynccl.all_reduce(data, stream=s)
67
+ value = data.mean().item()
68
+ assert value == world_size, f"Expected {world_size}, got {value}"
69
+
70
+ print("vLLM NCCL is successful!")
71
+
72
+ g = torch.cuda.CUDAGraph()
73
+ with torch.cuda.graph(cuda_graph=g, stream=s):
74
+ pynccl.all_reduce(data, stream=torch.cuda.current_stream())
75
+
76
+ data.fill_(1)
77
+ g.replay()
78
+ torch.cuda.current_stream().synchronize()
79
+ value = data.mean().item()
80
+ assert value == world_size, f"Expected {world_size}, got {value}"
81
+
82
+ print("vLLM NCCL with cuda graph is successful!")
83
+
84
+ dist.destroy_process_group(gloo_group)
85
+ dist.destroy_process_group()
86
87
.. tip::
88