A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Update Cutlass fp8 configs (#5144)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>
Varun Sundar Rabindranath committed
Jun 1, 2024 at 14:16 UTC
f081c3ce4b020fb094e33575d178345c477ab0c6
4 files changed
+480
-15
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
new
+352
@@ -0,0 +1,352 @@
1
+import argparse
2
+import copy
3
+import itertools
4
+import pickle as pkl
5
+import time
6
+from typing import Callable, Iterable, List, Tuple
7
+
8
+import torch
9
+import torch.utils.benchmark as TBenchmark
10
+from torch.utils.benchmark import Measurement as TMeasurement
11
+from weight_shapes import WEIGHT_SHAPES
12
+
13
+from vllm import _custom_ops as ops
14
+
15
+DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())[1:]
16
+DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
17
+DEFAULT_TP_SIZES = [1]
18
+
19
+# helpers
20
+
21
+
22
+def to_fp8(tensor: torch.tensor) -> torch.tensor:
23
+ finfo = torch.finfo(torch.float8_e4m3fn)
24
+ return torch.round(tensor.clamp(
25
+ min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
26
+
27
+
28
+def to_int8(tensor: torch.tensor) -> torch.tensor:
29
+ return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
30
+
31
+
32
+def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
33
+ k: int) -> Tuple[torch.tensor, torch.tensor]:
34
+
35
+ a = torch.randn((m, k), device='cuda') * 5
36
+ b = torch.randn((n, k), device='cuda').t() * 5
37
+
38
+ if dtype == torch.int8:
39
+ return to_int8(a), to_int8(b)
40
+ if dtype == torch.float8_e4m3fn:
41
+ return to_fp8(a), to_fp8(b)
42
+
43
+ raise ValueError("unsupported dtype")
44
+
45
+
46
+# impl
47
+
48
+
49
+def pytorch_i8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
50
+ scale_b: torch.tensor,
51
+ out_dtype: torch.dtype) -> torch.tensor:
52
+ return torch.mm(a, b)
53
+
54
+
55
+def pytorch_fp8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
56
+ scale_b: torch.tensor,
57
+ out_dtype: torch.dtype) -> torch.tensor:
58
+ return torch._scaled_mm(a,
59
+ b,
60
+ scale_a=scale_a,
61
+ scale_b=scale_b,
62
+ out_dtype=out_dtype)
63
+
64
+
65
+def pytorch_fp8_impl_fast_accum(a: torch.tensor, b: torch.tensor,
66
+ scale_a: torch.tensor, scale_b: torch.tensor,
67
+ out_dtype: torch.dtype) -> torch.tensor:
68
+ return torch._scaled_mm(a,
69
+ b,
70
+ scale_a=scale_a,
71
+ scale_b=scale_b,
72
+ out_dtype=out_dtype,
73
+ use_fast_accum=True)
74
+
75
+
76
+def cutlass_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
77
+ scale_b: torch.tensor,
78
+ out_dtype: torch.dtype) -> torch.tensor:
79
+ return ops.cutlass_scaled_mm_dq(a,
80
+ b,
81
+ scale_a,
82
+ scale_b,
83
+ out_dtype=out_dtype)
84
+
85
+
86
+# bench
87
+def bench_fn(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
88
+ scale_b: torch.tensor, out_dtype: torch.dtype, label: str,
89
+ sub_label: str, fn: Callable, description: str) -> TMeasurement:
90
+
91
+ min_run_time = 1
92
+
93
+ globals = {
94
+ "a": a,
95
+ "b": b,
96
+ "scale_a": scale_a,
97
+ "scale_b": scale_b,
98
+ "out_dtype": out_dtype,
99
+ "fn": fn,
100
+ }
101
+ return TBenchmark.Timer(
102
+ stmt="fn(a, b, scale_a, scale_b, out_dtype)",
103
+ globals=globals,
104
+ label=label,
105
+ sub_label=sub_label,
106
+ description=description,
107
+ ).blocked_autorange(min_run_time=min_run_time)
108
+
109
+
110
+def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
111
+ sub_label: str) -> Iterable[TMeasurement]:
112
+ assert dtype == torch.int8
113
+ a, b = make_rand_tensors(torch.int8, m, n, k)
114
+ scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
115
+ scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
116
+
117
+ timers = []
118
+ # pytorch impl
119
+ timers.append(
120
+ bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
121
+ b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
122
+ torch.bfloat16, label, sub_label, pytorch_i8_impl,
123
+ "pytorch_bf16_bf16_bf16_matmul-no-scales"))
124
+
125
+ # cutlass impl
126
+ timers.append(
127
+ bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
128
+ torch.bfloat16, label, sub_label, cutlass_impl,
129
+ "cutlass_i8_i8_bf16_scaled_mm"))
130
+
131
+ return timers
132
+
133
+
134
+def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
135
+ sub_label: str) -> Iterable[TMeasurement]:
136
+ assert dtype == torch.float8_e4m3fn
137
+ a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)
138
+ scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
139
+ scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
140
+
141
+ timers = []
142
+
143
+ # pytorch impl: bf16 output, without fp8 fast accum
144
+ timers.append(
145
+ bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
146
+ pytorch_fp8_impl, "pytorch_fp8_fp8_bf16_scaled_mm"))
147
+
148
+ # pytorch impl: bf16 output, with fp8 fast accum
149
+ timers.append(
150
+ bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
151
+ pytorch_fp8_impl_fast_accum,
152
+ "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum"))
153
+
154
+ # pytorch impl: fp16 output, without fp8 fast accum
155
+ timers.append(
156
+ bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
157
+ pytorch_fp8_impl, "pytorch_fp8_fp8_fp16_scaled_mm"))
158
+
159
+ # pytorch impl: fp16 output, with fp8 fast accum
160
+ timers.append(
161
+ bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
162
+ pytorch_fp8_impl_fast_accum,
163
+ "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum"))
164
+
165
+ # cutlass impl: bf16 output
166
+ timers.append(
167
+ bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
168
+ torch.bfloat16, label, sub_label, cutlass_impl,
169
+ "cutlass_fp8_fp8_bf16_scaled_mm"))
170
+ # cutlass impl: fp16 output
171
+ timers.append(
172
+ bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
173
+ torch.float16, label, sub_label, cutlass_impl,
174
+ "cutlass_fp8_fp8_fp16_scaled_mm"))
175
+ return timers
176
+
177
+
178
+def bench(dtype: torch.dtype, m: int, k: int, n: int, label: str,
179
+ sub_label: str) -> Iterable[TMeasurement]:
180
+ if dtype == torch.int8:
181
+ return bench_int8(dtype, m, k, n, label, sub_label)
182
+ if dtype == torch.float8_e4m3fn:
183
+ return bench_fp8(dtype, m, k, n, label, sub_label)
184
+ raise ValueError("unsupported type")
185
+
186
+
187
+# runner
188
+def print_timers(timers: Iterable[TMeasurement]):
189
+ compare = TBenchmark.Compare(timers)
190
+ compare.print()
191
+
192
+
193
+def run(dtype: torch.dtype,
194
+ MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
195
+
196
+ results = []
197
+ for m, k, n in MKNs:
198
+ timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
199
+ f"MKN=({m}x{k}x{n})")
200
+ print_timers(timers)
201
+ results.extend(timers)
202
+
203
+ return results
204
+
205
+
206
+# output makers
207
+def make_output(data: Iterable[TMeasurement],
208
+ MKNs: Iterable[Tuple[int, int, int]],
209
+ base_description: str,
210
+ timestamp=None):
211
+
212
+ print(f"== All Results {base_description} ====")
213
+ print_timers(data)
214
+
215
+ # pickle all the results
216
+ timestamp = int(time.time()) if timestamp is None else timestamp
217
+ with open(f"{base_description}-{timestamp}.pkl", "wb") as f:
218
+ pkl.dump(data, f)
219
+
220
+
221
+# argparse runners
222
+
223
+
224
+def run_square_bench(args):
225
+ dim_sizes = list(
226
+ range(args.dim_start, args.dim_end + 1, args.dim_increment))
227
+ MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
228
+ data = run(args.dtype, MKNs)
229
+
230
+ make_output(data, MKNs, f"square_bench-{args.dtype}")
231
+
232
+
233
+def run_range_bench(args):
234
+ dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
235
+ n = len(dim_sizes)
236
+ Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
237
+ Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
238
+ Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
239
+ MKNs = list(zip(Ms, Ks, Ns))
240
+ data = run(args.dtype, MKNs)
241
+
242
+ make_output(data, MKNs, f"range_bench-{args.dtype}")
243
+
244
+
245
+def run_model_bench(args):
246
+
247
+ print("Benchmarking models:")
248
+ for i, model in enumerate(args.models):
249
+ print(f"[{i}] {model}")
250
+
251
+ def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:
252
+ KNs = []
253
+ for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):
254
+ KN[tp_split_dim] = KN[tp_split_dim] // tp_size
255
+ KNs.append(KN)
256
+ return KNs
257
+
258
+ model_bench_data = []
259
+ models_tps = list(itertools.product(args.models, args.tp_sizes))
260
+ for model, tp_size in models_tps:
261
+ Ms = args.batch_sizes
262
+ KNs = model_shapes(model, tp_size)
263
+ MKNs = []
264
+ for m in Ms:
265
+ for k, n in KNs:
266
+ MKNs.append((m, k, n))
267
+
268
+ data = run(args.dtype, MKNs)
269
+ model_bench_data.append(data)
270
+
271
+ # Print all results
272
+ for data, model_tp in zip(model_bench_data, models_tps):
273
+ model, tp_size = model_tp
274
+ print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
275
+ print_timers(data)
276
+
277
+ timestamp = int(time.time())
278
+
279
+ all_data = []
280
+ for d in model_bench_data:
281
+ all_data.extend(d)
282
+ # pickle all data
283
+ with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
284
+ pkl.dump(all_data, f)
285
+
286
+
287
+if __name__ == '__main__':
288
+
289
+ def to_torch_dtype(dt):
290
+ if dt == "int8":
291
+ return torch.int8
292
+ if dt == "fp8":
293
+ return torch.float8_e4m3fn
294
+ raise ValueError("unsupported dtype")
295
+
296
+ parser = argparse.ArgumentParser(
297
+ description="""
298
+Benchmark Cutlass GEMM.
299
+
300
+ To run square GEMMs:
301
+ python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 square_bench --dim-start 128 --dim-end 512 --dim-increment 64
302
+
303
+ To run constant N and K and sweep M:
304
+ python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384
305
+
306
+ To run dimensions from a model:
307
+ python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1
308
+
309
+ Output:
310
+ - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs.
311
+ """, # noqa: E501
312
+ formatter_class=argparse.RawTextHelpFormatter)
313
+
314
+ parser.add_argument("--dtype",
315
+ type=to_torch_dtype,
316
+ required=True,
317
+ help="Available options are ['int8', 'fp8']")
318
+ subparsers = parser.add_subparsers(dest="cmd")
319
+
320
+ square_parser = subparsers.add_parser("square_bench")
321
+ square_parser.add_argument("--dim-start", type=int, required=True)
322
+ square_parser.add_argument("--dim-end", type=int, required=True)
323
+ square_parser.add_argument("--dim-increment", type=int, required=True)
324
+ square_parser.set_defaults(func=run_square_bench)
325
+
326
+ range_parser = subparsers.add_parser("range_bench")
327
+ range_parser.add_argument("--dim-start", type=int, required=True)
328
+ range_parser.add_argument("--dim-end", type=int, required=True)
329
+ range_parser.add_argument("--dim-increment", type=int, required=True)
330
+ range_parser.add_argument("--m-constant", type=int, default=None)
331
+ range_parser.add_argument("--n-constant", type=int, default=None)
332
+ range_parser.add_argument("--k-constant", type=int, default=None)
333
+ range_parser.set_defaults(func=run_range_bench)
334
+
335
+ model_parser = subparsers.add_parser("model_bench")
336
+ model_parser.add_argument("--models",
337
+ nargs="+",
338
+ type=str,
339
+ default=DEFAULT_MODELS,
340
+ choices=WEIGHT_SHAPES.keys())
341
+ model_parser.add_argument("--tp-sizes",
342
+ nargs="+",
343
+ type=int,
344
+ default=DEFAULT_TP_SIZES)
345
+ model_parser.add_argument("--batch-sizes",
346
+ nargs="+",
347
+ type=int,
348
+ default=DEFAULT_BATCH_SIZES)
349
+ model_parser.set_defaults(func=run_model_bench)
350
+
351
+ args = parser.parse_args()
352
+ args.func(args)
benchmarks/cutlass_benchmarks/weight_shapes.py
new
+37
@@ -0,0 +1,37 @@
1
+# Weight Shapes are in the format
2
+# ([K, N], TP_SPLIT_DIM)
3
+# Example:
4
+# A shape of ([14336, 4096], 0) indicates the following GEMM shape,
5
+# - TP1 : K = 14336, N = 4096
6
+# - TP2 : K = 7168, N = 4096
7
+# A shape of ([4096, 6144], 1) indicates the following GEMM shape,
8
+# - TP1 : K = 4096, N = 6144
9
+# - TP4 : K = 4096, N = 1536
10
+
11
+# TP1 shapes
12
+WEIGHT_SHAPES = {
13
+ "mistralai/Mistral-7B-v0.1": [
14
+ ([4096, 6144], 1),
15
+ ([4096, 4096], 0),
16
+ ([4096, 28672], 1),
17
+ ([14336, 4096], 0),
18
+ ],
19
+ "meta-llama/Llama-2-7b-hf": [
20
+ ([4096, 12288], 1),
21
+ ([4096, 4096], 0),
22
+ ([4096, 22016], 1),
23
+ ([11008, 4096], 0),
24
+ ],
25
+ "meta-llama/Llama-2-13b-hf": [
26
+ ([5120, 15360], 1),
27
+ ([5120, 5120], 0),
28
+ ([5120, 27648], 1),
29
+ ([13824, 5120], 0),
30
+ ],
31
+ "meta-llama/Llama-2-70b-hf": [
32
+ ([8192, 10240], 1),
33
+ ([8192, 8192], 0),
34
+ ([8192, 57344], 1),
35
+ ([28672, 8192], 0),
36
+ ],
37
+}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+90
-14
@@ -51,6 +51,11 @@ using namespace cute;
51
52
namespace {
53
54
+uint32_t next_pow_2(uint32_t const num) {
55
+ if (num <= 1) return num;
56
+ return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
57
+}
58
+
59
template <typename ElementAB_, typename ElementD_, typename TileShape,
60
typename ClusterShape, typename KernelSchedule,
61
typename EpilogueSchedule>
@@ -188,8 +193,89 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
193
cutlass::Status status = gemm_op.run(args, workspace.get(), stream);
194
CUTLASS_CHECK(status);
195
}
196
+
197
+template <typename InType, typename OutType, int32_t M>
198
+struct sm90_fp8_config {
199
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
200
+ using KernelSchedule =
201
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
202
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
203
+ using TileShape = Shape<_128, _128, _128>;
204
+ using ClusterShape = Shape<_2, _1, _1>;
205
+
206
+ using Cutlass3xGemm =
207
+ cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
208
+ EpilogueSchedule>;
209
+};
210
+
211
+template <typename InType, typename OutType>
212
+struct sm90_fp8_config<InType, OutType, 128> {
213
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
214
+ using KernelSchedule =
215
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
216
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
217
+ using TileShape = Shape<_64, _128, _128>;
218
+ using ClusterShape = Shape<_2, _1, _1>;
219
+
220
+ using Cutlass3xGemm =
221
+ cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
222
+ EpilogueSchedule>;
223
+};
224
+
225
+template <typename InType, typename OutType>
226
+struct sm90_fp8_config<InType, OutType, 64> {
227
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
228
+ using KernelSchedule =
229
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
230
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
231
+ using TileShape = Shape<_64, _64, _128>;
232
+ using ClusterShape = Shape<_1, _8, _1>;
233
+
234
+ using Cutlass3xGemm =
235
+ cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
236
+ EpilogueSchedule>;
237
+};
238
+
239
} // namespace
240
241
+template <typename InType, typename OutType>
242
+void cutlass_scaled_mm_dq_sm90_fp8_dispatch(torch::Tensor& out,
243
+ torch::Tensor const& a,
244
+ torch::Tensor const& b,
245
+ torch::Tensor const& a_scales,
246
+ torch::Tensor const& b_scales) {
247
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
248
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
249
+ TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
250
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
251
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
252
+
253
+ using Cutlass3xGemmDefault =
254
+ typename sm90_fp8_config<InType, OutType, 0>::Cutlass3xGemm;
255
+ using Cutlass3xGemmM64 =
256
+ typename sm90_fp8_config<InType, OutType, 64>::Cutlass3xGemm;
257
+ using Cutlass3xGemmM128 =
258
+ typename sm90_fp8_config<InType, OutType, 128>::Cutlass3xGemm;
259
+
260
+ uint32_t const m = a.size(0);
261
+ uint32_t const mp2 =
262
+ std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
263
+
264
+ if (mp2 <= 64) {
265
+ // m in [1, 64]
266
+ return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmM64>(
267
+ out, a, b, a_scales, b_scales);
268
+ } else if (mp2 <= 128) {
269
+ // m in (64, 128]
270
+ return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmM128>(
271
+ out, a, b, a_scales, b_scales);
272
+ } else {
273
+ // m in (128, inf)
274
+ return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmDefault>(
275
+ out, a, b, a_scales, b_scales);
276
+ }
277
+}
278
+
279
void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
280
torch::Tensor const& b,
281
torch::Tensor const& a_scales,
@@ -223,24 +309,14 @@ void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
309
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
310
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
311
226
- using TileShape = Shape<_128, _128, _128>;
227
- using ClusterShape = Shape<_1, _2, _1>;
228
- using KernelSchedule =
229
- typename cutlass::gemm::KernelCpAsyncWarpSpecializedCooperative;
230
- using EpilogueSchedule =
231
- typename cutlass::epilogue::TmaWarpSpecializedCooperative;
232
-
312
if (out.dtype() == torch::kBFloat16) {
234
- return cutlass_scaled_mm_dq_dispatcher<
235
- cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::bfloat16_t, TileShape,
236
- ClusterShape, KernelSchedule, EpilogueSchedule>>(
313
+ return cutlass_scaled_mm_dq_sm90_fp8_dispatch<cutlass::float_e4m3_t,
314
+ cutlass::bfloat16_t>(
315
out, a, b, a_scales, b_scales);
316
} else {
317
TORCH_CHECK(out.dtype() == torch::kFloat16);
240
-
241
- return cutlass_scaled_mm_dq_dispatcher<
242
- cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::half_t, TileShape,
243
- ClusterShape, KernelSchedule, EpilogueSchedule>>(
318
+ return cutlass_scaled_mm_dq_sm90_fp8_dispatch<cutlass::float_e4m3_t,
319
+ cutlass::half_t>(
320
out, a, b, a_scales, b_scales);
321
}
322
}
tests/kernels/test_cutlass.py
+1
-1
@@ -82,7 +82,7 @@ def cutlass_int8_gemm_helper(m: int,
82
assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
83
84
85
-@pytest.mark.parametrize("m", [512, 222, 33, 1])
85
+@pytest.mark.parametrize("m", [512, 222, 100, 33, 1])
86
@pytest.mark.parametrize("n", [2048, 256, 1024])
87
@pytest.mark.parametrize("k", [128, 496, 1024])
88
@pytest.mark.parametrize("per_act_token", [True, False])