A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Fixup for CUTLASS kernels in CUDA graphs (#4954)
Pass the CUDA stream into the CUTLASS GEMMs, to avoid future issues with CUDA graphs
Tyler Michael Smith committed
May 22, 2024 at 10:10 UTC
8674f9880e2d8574c2adc759027e0f27dc9b95de
3 files changed
+50
-2
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+5
-1
@@ -1,6 +1,8 @@
1
#include <stddef.h>
2
#include <torch/extension.h>
3
4
+#include <ATen/cuda/CUDAContext.h>
5
+
6
// clang-format will break include orders
7
// clang-format off
8
#include "cute/tensor.hpp"
@@ -189,8 +191,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
191
size_t workspace_size = gemm_op.get_workspace_size(args);
192
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
193
194
+ auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
195
+
196
CUTLASS_CHECK(gemm_op.can_implement(args));
193
- cutlass::Status status = gemm_op(args, workspace.get());
197
+ cutlass::Status status = gemm_op(args, workspace.get(), stream);
198
CUTLASS_CHECK(status);
199
}
200
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+4
-1
@@ -1,5 +1,7 @@
1
#include <torch/extension.h>
2
3
+#include <ATen/cuda/CUDAContext.h>
4
+
5
#include <iostream>
6
#include <sstream>
7
#include <vector>
@@ -178,7 +180,8 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
180
size_t workspace_size = gemm_op.get_workspace_size(args);
181
TORCH_CHECK(workspace_size == 0);
182
181
- cutlass::Status status = gemm_op.run(args);
183
+ auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
184
+ cutlass::Status status = gemm_op.run(args, stream);
185
CUTLASS_CHECK(status);
186
}
187
} // namespace
tests/kernels/test_cutlass.py
+41
@@ -190,3 +190,44 @@ def test_cutlass_subset():
190
b.to(dtype=torch.float32)).to(dtype=torch.bfloat16)
191
192
assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
193
+
194
+
195
+# Test to make sure cuda graphs work
196
+class CutlassLayer(torch.nn.Module):
197
+
198
+ def __init__(self, b, scale_a, scale_b, out_dtype):
199
+ super().__init__()
200
+ self.b = b
201
+ self.scale_a = scale_a
202
+ self.scale_b = scale_b
203
+ self.out_dtype = out_dtype
204
+
205
+ def forward(self, a):
206
+ return ops.cutlass_scaled_mm_dq(a, self.b, self.scale_a, self.scale_b,
207
+ self.out_dtype)
208
+
209
+
210
+def test_cutlass_cuda_graph():
211
+ m, n, k = 512, 512, 512
212
+
213
+ a = to_int8(torch.randn((m, k), device="cuda"))
214
+ b = to_int8(torch.randn((n, k), device="cuda").t())
215
+
216
+ scale_a = (torch.randn((m, 1), device="cuda", dtype=torch.float32) / 10)
217
+ scale_b = (torch.randn((1, n), device="cuda", dtype=torch.float32) / 10)
218
+
219
+ # Construct a trivial model with a single layer that calls a CUTLASS kernel
220
+ model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16)
221
+
222
+ # Run the model with a cuda graph
223
+ stream = torch.cuda.Stream()
224
+ with torch.cuda.stream(stream):
225
+ g = torch.cuda.CUDAGraph()
226
+ with torch.cuda.graph(g):
227
+ out = model(a)
228
+ out.zero_()
229
+ g.replay()
230
+
231
+ baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
232
+ scale_b * b.to(dtype=torch.float32)).to(torch.bfloat16)
233
+ assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)