A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[misc] fix debugging code (#13487)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 19, 2025 at 01:37 UTC
7b203b76947de241659d0f1a8b77183bc48a6dd2
1 file changed
+4
-4
docs/source/getting_started/troubleshooting.md
+4
-4
@@ -94,20 +94,20 @@ pynccl.disabled = False
94
s = torch.cuda.Stream()
95
with torch.cuda.stream(s):
96
data.fill_(1)
97
- pynccl.all_reduce(data, stream=s)
98
- value = data.mean().item()
97
+ out = pynccl.all_reduce(data, stream=s)
98
+ value = out.mean().item()
99
assert value == world_size, f"Expected {world_size}, got {value}"
100
101
print("vLLM NCCL is successful!")
102
103
g = torch.cuda.CUDAGraph()
104
with torch.cuda.graph(cuda_graph=g, stream=s):
105
- pynccl.all_reduce(data, stream=torch.cuda.current_stream())
105
+ out = pynccl.all_reduce(data, stream=torch.cuda.current_stream())
106
107
data.fill_(1)
108
g.replay()
109
torch.cuda.current_stream().synchronize()
110
-value = data.mean().item()
110
+value = out.mean().item()
111
assert value == world_size, f"Expected {world_size}, got {value}"
112
113
print("vLLM NCCL with cuda graph is successful!")