@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 199adbb7cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[doc] update test script to include cudagraph (#7501)

youkaichao committed Aug 13, 2024 at 21:52 UTC 199adbb7cfb7e8185c9418de56db006f8cf1c294
1 file changed +37 -2
docs/source/getting_started/debugging.rst
+37 -2
@@ -30,24 +30,59 @@ Here are some common issues that can cause hangs:
30
31 .. code-block:: python
32
33 + # Test PyTorch NCCL
34 import torch
35 import torch.distributed as dist
36 dist.init_process_group(backend="nccl")
37 local_rank = dist.get_rank() % torch.cuda.device_count()
37 - data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}")
38 + torch.cuda.set_device(local_rank)
39 + data = torch.FloatTensor([1,] * 128).to("cuda")
40 dist.all_reduce(data, op=dist.ReduceOp.SUM)
41 torch.cuda.synchronize()
42 value = data.mean().item()
43 world_size = dist.get_world_size()
44 assert value == world_size, f"Expected {world_size}, got {value}"
45
46 + print("PyTorch NCCL is successful!")
47 +
48 + # Test PyTorch GLOO
49 gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo")
50 cpu_data = torch.FloatTensor([1,] * 128)
51 dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group)
52 value = cpu_data.mean().item()
53 assert value == world_size, f"Expected {world_size}, got {value}"
54
50 - print("sanity check is successful!")
55 + print("PyTorch GLOO is successful!")
56 +
57 + # Test vLLM NCCL, with cuda graph
58 + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
59 +
60 + pynccl = PyNcclCommunicator(group=gloo_group, device=local_rank)
61 + pynccl.disabled = False
62 +
63 + s = torch.cuda.Stream()
64 + with torch.cuda.stream(s):
65 + data.fill_(1)
66 + pynccl.all_reduce(data, stream=s)
67 + value = data.mean().item()
68 + assert value == world_size, f"Expected {world_size}, got {value}"
69 +
70 + print("vLLM NCCL is successful!")
71 +
72 + g = torch.cuda.CUDAGraph()
73 + with torch.cuda.graph(cuda_graph=g, stream=s):
74 + pynccl.all_reduce(data, stream=torch.cuda.current_stream())
75 +
76 + data.fill_(1)
77 + g.replay()
78 + torch.cuda.current_stream().synchronize()
79 + value = data.mean().item()
80 + assert value == world_size, f"Expected {world_size}, got {value}"
81 +
82 + print("vLLM NCCL with cuda graph is successful!")
83 +
84 + dist.destroy_process_group(gloo_group)
85 + dist.destroy_process_group()
86
87 .. tip::
88