@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8674f9880e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Fixup for CUTLASS kernels in CUDA graphs (#4954)

Pass the CUDA stream into the CUTLASS GEMMs, to avoid future issues with CUDA graphs

Tyler Michael Smith committed May 22, 2024 at 10:10 UTC 8674f9880e2d8574c2adc759027e0f27dc9b95de
3 files changed +50 -2
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+5 -1
@@ -1,6 +1,8 @@
1 #include <stddef.h>
2 #include <torch/extension.h>
3
4 +#include <ATen/cuda/CUDAContext.h>
5 +
6 // clang-format will break include orders
7 // clang-format off
8 #include "cute/tensor.hpp"
@@ -189,8 +191,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
191 size_t workspace_size = gemm_op.get_workspace_size(args);
192 cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
193
194 + auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
195 +
196 CUTLASS_CHECK(gemm_op.can_implement(args));
193 - cutlass::Status status = gemm_op(args, workspace.get());
197 + cutlass::Status status = gemm_op(args, workspace.get(), stream);
198 CUTLASS_CHECK(status);
199 }
200
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+4 -1
@@ -1,5 +1,7 @@
1 #include <torch/extension.h>
2
3 +#include <ATen/cuda/CUDAContext.h>
4 +
5 #include <iostream>
6 #include <sstream>
7 #include <vector>
@@ -178,7 +180,8 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
180 size_t workspace_size = gemm_op.get_workspace_size(args);
181 TORCH_CHECK(workspace_size == 0);
182
181 - cutlass::Status status = gemm_op.run(args);
183 + auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
184 + cutlass::Status status = gemm_op.run(args, stream);
185 CUTLASS_CHECK(status);
186 }
187 } // namespace
tests/kernels/test_cutlass.py
+41
@@ -190,3 +190,44 @@ def test_cutlass_subset():
190 b.to(dtype=torch.float32)).to(dtype=torch.bfloat16)
191
192 assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
193 +
194 +
195 +# Test to make sure cuda graphs work
196 +class CutlassLayer(torch.nn.Module):
197 +
198 + def __init__(self, b, scale_a, scale_b, out_dtype):
199 + super().__init__()
200 + self.b = b
201 + self.scale_a = scale_a
202 + self.scale_b = scale_b
203 + self.out_dtype = out_dtype
204 +
205 + def forward(self, a):
206 + return ops.cutlass_scaled_mm_dq(a, self.b, self.scale_a, self.scale_b,
207 + self.out_dtype)
208 +
209 +
210 +def test_cutlass_cuda_graph():
211 + m, n, k = 512, 512, 512
212 +
213 + a = to_int8(torch.randn((m, k), device="cuda"))
214 + b = to_int8(torch.randn((n, k), device="cuda").t())
215 +
216 + scale_a = (torch.randn((m, 1), device="cuda", dtype=torch.float32) / 10)
217 + scale_b = (torch.randn((1, n), device="cuda", dtype=torch.float32) / 10)
218 +
219 + # Construct a trivial model with a single layer that calls a CUTLASS kernel
220 + model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16)
221 +
222 + # Run the model with a cuda graph
223 + stream = torch.cuda.Stream()
224 + with torch.cuda.stream(stream):
225 + g = torch.cuda.CUDAGraph()
226 + with torch.cuda.graph(g):
227 + out = model(a)
228 + out.zero_()
229 + g.replay()
230 +
231 + baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
232 + scale_b * b.to(dtype=torch.float32)).to(torch.bfloat16)
233 + assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)