A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling (#11868)
Lucas Wilkinson committed
Jan 30, 2025 at 21:33 UTC
9798b2fb0052092a6420172e41c0c8a307eedfa6
25 files changed
+1924
-346
CMakeLists.txt
+7
-2
@@ -245,7 +245,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
245
FetchContent_Declare(
246
cutlass
247
GIT_REPOSITORY https://github.com/nvidia/cutlass.git
248
- GIT_TAG v3.6.0
248
+ GIT_TAG v3.7.0
249
GIT_PROGRESS TRUE
250
251
# Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
@@ -299,7 +299,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
299
# CUDA 12.0 or later (and only work on Hopper, 9.0a for now).
300
cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0a" "${CUDA_ARCHS}")
301
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
302
- set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
302
+ set(SRCS
303
+ "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
304
+ "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu"
305
+ "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu"
306
+ "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu"
307
+ "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8.cu")
308
set_gencode_flags_for_srcs(
309
SRCS "${SRCS}"
310
CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+142
-148
@@ -3,7 +3,7 @@ import copy
3
import itertools
4
import pickle as pkl
5
import time
6
-from typing import Callable, Iterable, List, Tuple
6
+from typing import Callable, Iterable, List, Optional, Tuple
7
8
import torch
9
import torch.utils.benchmark as TBenchmark
@@ -12,6 +12,8 @@ from utils import make_rand_tensors
12
from weight_shapes import WEIGHT_SHAPES
13
14
from vllm import _custom_ops as ops
15
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
16
+ w8a8_block_fp8_matmul)
17
from vllm.utils import FlexibleArgumentParser
18
19
DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
@@ -38,8 +40,15 @@ def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
40
).blocked_autorange(min_run_time=min_run_time)
41
42
41
-def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
42
- sub_label: str) -> Iterable[TMeasurement]:
43
+def bench_int8(
44
+ dtype: torch.dtype,
45
+ m: int,
46
+ k: int,
47
+ n: int,
48
+ label: str,
49
+ sub_label: str,
50
+ bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
51
+ """Benchmark INT8-based kernels."""
52
assert dtype == torch.int8
53
a, b = make_rand_tensors(torch.int8, m, n, k)
54
scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
@@ -48,155 +57,132 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
57
azp = torch.zeros((m, ), device="cuda", dtype=torch.int32)
58
azp_adj = torch.zeros((n, ), device="cuda", dtype=torch.int32)
59
60
+ bench_fns = {
61
+ "pytorch_bf16_bf16_bf16_matmul-no-scales":
62
+ lambda: torch.mm(a.to(dtype=torch.bfloat16), b.to(dtype=torch.bfloat16)
63
+ ),
64
+ "pytorch_fp16_fp16_fp16_matmul-no-scales":
65
+ lambda: torch.mm(a.to(dtype=torch.float16), b.to(dtype=torch.float16)),
66
+ "cutlass_i8_i8_bf16_scaled_mm":
67
+ lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16),
68
+ "cutlass_i8_i8_bf16_scaled_mm_bias":
69
+ lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16,
70
+ bias),
71
+ "cutlass_i8_i8_bf16_scaled_mm_azp":
72
+ lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
73
+ bfloat16, azp_adj),
74
+ "cutlass_i8_i8_bf16_scaled_mm_azp_bias":
75
+ lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
76
+ bfloat16, azp_adj, None, bias),
77
+ "cutlass_i8_i8_bf16_scaled_mm_azp_pt":
78
+ lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
79
+ bfloat16, azp_adj, azp),
80
+ "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias":
81
+ lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
82
+ bfloat16, azp_adj, azp, bias),
83
+ }
84
+
85
timers = []
52
- # pytorch impl - bfloat16
53
- timers.append(
54
- bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
55
- torch.mm, a.to(dtype=torch.bfloat16),
56
- b.to(dtype=torch.bfloat16)))
57
-
58
- # pytorch impl - float16
59
- timers.append(
60
- bench_fn(label, sub_label,
61
- "pytorch_fp16_fp16_fp16_matmul-no-scales", torch.mm,
62
- a.to(dtype=torch.float16), b.to(dtype=torch.float16)))
63
-
64
- # cutlass impl
65
- timers.append(
66
- bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm",
67
- ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
68
- torch.bfloat16))
69
-
70
- # cutlass with bias
71
- timers.append(
72
- bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_bias",
73
- ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
74
- bias))
75
-
76
- # cutlass with azp per-tensor
77
- timers.append(
78
- bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp",
79
- ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
80
- torch.bfloat16, azp_adj))
81
-
82
- # cutlass with azp per-tensor + bias
83
- timers.append(
84
- bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_bias",
85
- ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
86
- torch.bfloat16, azp_adj, None, bias))
87
-
88
- # cutlass with azp per-token
89
- timers.append(
90
- bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt",
91
- ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
92
- torch.bfloat16, azp_adj, azp))
93
-
94
- # cutlass with azp per-token + bias
95
- timers.append(
96
- bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias",
97
- ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
98
- torch.bfloat16, azp_adj, azp, bias))
86
+ for name, fn in bench_fns.items():
87
+ # If bench_kernels is None, run all. Otherwise, run only exact matches.
88
+ if bench_kernels is None or name in bench_kernels:
89
+ print(f"Running {name}")
90
+ timers.append(bench_fn(label, sub_label, name, fn))
91
92
return timers
93
94
103
-def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
104
- sub_label: str) -> Iterable[TMeasurement]:
95
+def bench_fp8(
96
+ dtype: torch.dtype,
97
+ m: int,
98
+ k: int,
99
+ n: int,
100
+ label: str,
101
+ sub_label: str,
102
+ bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
103
+ """Benchmark FP8-based kernels."""
104
assert dtype == torch.float8_e4m3fn
105
a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)
106
+ a_cont = a.contiguous()
107
scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
108
scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
109
+ block_scale_a = torch.rand((m, k // 128),
110
+ device="cuda",
111
+ dtype=torch.float32)
112
+ block_scale_b = torch.rand((k // 128, n // 128),
113
+ device="cuda",
114
+ dtype=torch.float32)
115
+ block_scale_a_M_major = block_scale_a.t().contiguous().t()
116
+ block_scale_b_K_major = block_scale_b.t().contiguous().t()
117
bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
118
111
- timers = []
119
+ print(m, k, n)
120
+
121
+ bench_fns = {
122
+ "pytorch_bf16_bf16_bf16_matmul-no-scales":
123
+ lambda: torch.mm(a.to(dtype=torch.bfloat16), b.to(dtype=torch.bfloat16)
124
+ ),
125
+ "pytorch_fp16_fp16_fp16_matmul-no-scales":
126
+ lambda: torch.mm(a.to(dtype=torch.float16), b.to(dtype=torch.float16)),
127
+ "pytorch_fp8_fp8_fp16_scaled_mm":
128
+ lambda: torch._scaled_mm(
129
+ a, b, scale_a, scale_b, out_dtype=torch.float16),
130
+ "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum":
131
+ lambda: torch._scaled_mm(a,
132
+ b,
133
+ scale_a,
134
+ scale_b,
135
+ out_dtype=torch.float16,
136
+ use_fast_accum=True),
137
+ "pytorch_fp8_fp8_bf16_scaled_mm":
138
+ lambda: torch._scaled_mm(
139
+ a, b, scale_a, scale_b, out_dtype=torch.bfloat16),
140
+ "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum":
141
+ lambda: torch._scaled_mm(a,
142
+ b,
143
+ scale_a,
144
+ scale_b,
145
+ out_dtype=torch.bfloat16,
146
+ use_fast_accum=True),
147
+ "cutlass_fp8_fp8_bf16_scaled_mm":
148
+ lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16),
149
+ "cutlass_fp8_fp8_fp16_scaled_mm":
150
+ lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.float16),
151
+ "cutlass_fp8_fp8_bf16_scaled_mm_bias":
152
+ lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16,
153
+ bias),
154
+ "cutlass_fp8_fp8_fp16_scaled_mm_bias":
155
+ lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.float16,
156
+ bias.to(dtype=torch.float16)),
157
+ "triton_fp8_fp8_fp16_scaled_mm_blockwise":
158
+ lambda: w8a8_block_fp8_matmul(a_cont, b.t(), block_scale_a,
159
+ block_scale_b.t(), (128, 128)),
160
+ "cutlass_fp8_fp8_fp16_scaled_mm_blockwise":
161
+ lambda: ops.cutlass_scaled_mm(a, b, block_scale_a_M_major,
162
+ block_scale_b_K_major, torch.float16),
163
+ }
164
113
- # pytorch impl w. bf16
114
- timers.append(
115
- bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
116
- torch.mm, a.to(dtype=torch.bfloat16, device="cuda"),
117
- b.to(dtype=torch.bfloat16, device="cuda")))
118
-
119
- # pytorch impl: bf16 output, without fp8 fast accum
120
- timers.append(
121
- bench_fn(label,
122
- sub_label,
123
- "pytorch_fp8_fp8_bf16_scaled_mm",
124
- torch._scaled_mm,
125
- a,
126
- b,
127
- scale_a=scale_a,
128
- scale_b=scale_b,
129
- out_dtype=torch.bfloat16))
130
-
131
- # pytorch impl: bf16 output, with fp8 fast accum
132
- timers.append(
133
- bench_fn(label,
134
- sub_label,
135
- "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
136
- torch._scaled_mm,
137
- a,
138
- b,
139
- scale_a=scale_a,
140
- scale_b=scale_b,
141
- out_dtype=torch.bfloat16,
142
- use_fast_accum=True))
143
-
144
- # pytorch impl: fp16 output, without fp8 fast accum
145
- timers.append(
146
- bench_fn(label,
147
- sub_label,
148
- "pytorch_fp8_fp8_fp16_scaled_mm",
149
- torch._scaled_mm,
150
- a,
151
- b,
152
- scale_a=scale_a,
153
- scale_b=scale_b,
154
- out_dtype=torch.float16))
155
-
156
- # pytorch impl: fp16 output, with fp8 fast accum
157
- timers.append(
158
- bench_fn(label,
159
- sub_label,
160
- "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
161
- torch._scaled_mm,
162
- a,
163
- b,
164
- scale_a=scale_a,
165
- scale_b=scale_b,
166
- out_dtype=torch.float16,
167
- use_fast_accum=True))
168
-
169
- # cutlass impl: bf16 output
170
- timers.append(
171
- bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm",
172
- ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
173
- torch.bfloat16))
174
- # cutlass impl: fp16 output
175
- timers.append(
176
- bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm",
177
- ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16))
178
-
179
- # cutlass impl: bf16 output, with bias
180
- timers.append(
181
- bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm_bias",
182
- ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
183
- bias))
184
-
185
- # cutlass impl: fp16 output, with bias
186
- timers.append(
187
- bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm_bias",
188
- ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16,
189
- bias.to(dtype=torch.float16)))
165
+ timers = []
166
+ for name, fn in bench_fns.items():
167
+ # If bench_kernels is None, run all. Otherwise, run only exact matches.
168
+ if bench_kernels is None or name in bench_kernels:
169
+ print(f"Running {name}")
170
+ timers.append(bench_fn(label, sub_label, name, fn))
171
172
return timers
173
174
194
-def bench(dtype: torch.dtype, m: int, k: int, n: int, label: str,
195
- sub_label: str) -> Iterable[TMeasurement]:
175
+def bench(dtype: torch.dtype,
176
+ m: int,
177
+ k: int,
178
+ n: int,
179
+ label: str,
180
+ sub_label: str,
181
+ bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
182
if dtype == torch.int8:
197
- return bench_int8(dtype, m, k, n, label, sub_label)
183
+ return bench_int8(dtype, m, k, n, label, sub_label, bench_kernels)
184
if dtype == torch.float8_e4m3fn:
199
- return bench_fp8(dtype, m, k, n, label, sub_label)
185
+ return bench_fp8(dtype, m, k, n, label, sub_label, bench_kernels)
186
raise ValueError("unsupported type")
187
188
@@ -207,18 +193,22 @@ def print_timers(timers: Iterable[TMeasurement]):
193
194
195
def run(dtype: torch.dtype,
210
- MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
196
+ MKNs: Iterable[Tuple[int, int, int]],
197
+ bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
198
results = []
199
for m, k, n in MKNs:
213
- timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
214
- f"MKN=({m}x{k}x{n})")
200
+ timers = bench(dtype,
201
+ m,
202
+ k,
203
+ n,
204
+ f"scaled-{dtype}-gemm",
205
+ f"MKN=({m}x{k}x{n})",
206
+ bench_kernels=bench_kernels)
207
print_timers(timers)
208
results.extend(timers)
217
-
209
return results
210
211
221
-# output makers
212
def make_output(data: Iterable[TMeasurement],
213
MKNs: Iterable[Tuple[int, int, int]],
214
base_description: str,
@@ -232,15 +222,11 @@ def make_output(data: Iterable[TMeasurement],
222
pkl.dump(data, f)
223
224
235
-# argparse runners
236
-
237
-
225
def run_square_bench(args):
226
dim_sizes = list(
227
range(args.dim_start, args.dim_end + 1, args.dim_increment))
228
MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
242
- data = run(args.dtype, MKNs)
243
-
229
+ data = run(args.dtype, MKNs, bench_kernels=args.kernels)
230
make_output(data, MKNs, f"square_bench-{args.dtype}")
231
232
@@ -251,8 +237,7 @@ def run_range_bench(args):
237
Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
238
Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
239
MKNs = list(zip(Ms, Ks, Ns))
254
- data = run(args.dtype, MKNs)
255
-
240
+ data = run(args.dtype, MKNs, bench_kernels=args.kernels)
241
make_output(data, MKNs, f"range_bench-{args.dtype}")
242
243
@@ -278,7 +263,7 @@ def run_model_bench(args):
263
for k, n in KNs:
264
MKNs.append((m, k, n))
265
281
- data = run(args.dtype, MKNs)
266
+ data = run(args.dtype, MKNs, bench_kernels=args.kernels)
267
model_bench_data.append(data)
268
269
# Print all results
@@ -328,6 +313,15 @@ Benchmark Cutlass GEMM.
313
type=to_torch_dtype,
314
required=True,
315
help="Available options are ['int8', 'fp8']")
316
+ parser.add_argument(
317
+ "--kernels",
318
+ nargs="+",
319
+ type=str,
320
+ default=None,
321
+ help=
322
+ "Exact names of the kernels to benchmark. If not set, runs all kernels."
323
+ )
324
+
325
subparsers = parser.add_subparsers(dest="cmd")
326
327
square_parser = subparsers.add_parser("square_bench")
@@ -362,4 +356,4 @@ Benchmark Cutlass GEMM.
356
model_parser.set_defaults(func=run_model_bench)
357
358
args = parser.parse_args()
365
- args.func(args)
\ No newline at end of file
359
+ args.func(args)
csrc/core/math.hpp
+8
-1
@@ -1,7 +1,14 @@
1
+#pragma once
2
+
3
#include <climits>
4
#include <iostream>
5
4
-inline uint32_t next_pow_2(uint32_t const num) {
6
+inline constexpr uint32_t next_pow_2(uint32_t const num) {
7
if (num <= 1) return num;
8
return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
9
+}
10
+
11
+template <typename T>
12
+inline constexpr std::enable_if_t<std::is_integral_v<T>, T> ceil_div(T a, T b) {
13
+ return (a + b - 1) / b;
14
}
\ No newline at end of file
csrc/cutlass_extensions/common.hpp
+17
@@ -32,3 +32,20 @@ inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
32
}
33
34
int32_t get_sm_version_num();
35
+
36
+/**
37
+ * A wrapper for a kernel that is used to guard against compilation on
38
+ * architectures that will never use the kernel. The purpose of this is to
39
+ * reduce the size of the compiled binary.
40
+ * __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
41
+ * into code that will be executed on the device where it is defined.
42
+ */
43
+template <typename Kernel>
44
+struct enable_sm90_or_later : Kernel {
45
+ template <typename... Args>
46
+ CUTLASS_DEVICE void operator()(Args&&... args) {
47
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
48
+ Kernel::operator()(std::forward<Args>(args)...);
49
+#endif
50
+ }
51
+};
\ No newline at end of file
csrc/cutlass_extensions/gemm/collective/collective_builder.hpp
new
+123
@@ -0,0 +1,123 @@
1
+// Modified from: cutlass/gemm/collective/builders/sm90_gmma_builder.inl
2
+// clang-format off
3
+#pragma once
4
+
5
+#include "cutlass/gemm/collective/builders/sm90_gmma_builder.inl"
6
+
7
+#include "cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp"
8
+
9
+
10
+/////////////////////////////////////////////////////////////////////////////////////////////////
11
+
12
+namespace cutlass::gemm::collective {
13
+
14
+/////////////////////////////////////////////////////////////////////////////////////////////////
15
+
16
+// GMMA_TMA_WS_SS (BlockScaled Builders)
17
+template <
18
+ class ElementA,
19
+ class GmemLayoutATag,
20
+ int AlignmentA,
21
+ class ElementB,
22
+ class GmemLayoutBTag,
23
+ int AlignmentB,
24
+ class ElementAccumulator,
25
+ class TileShape_MNK,
26
+ class ClusterShape_MNK,
27
+ class StageCountType,
28
+ int ScaleGranularityM
29
+>
30
+struct CollectiveBuilder<
31
+ arch::Sm90,
32
+ arch::OpClassTensorOp,
33
+ ElementA,
34
+ GmemLayoutATag,
35
+ AlignmentA,
36
+ ElementB,
37
+ GmemLayoutBTag,
38
+ AlignmentB,
39
+ ElementAccumulator,
40
+ TileShape_MNK,
41
+ ClusterShape_MNK,
42
+ StageCountType,
43
+ KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<ScaleGranularityM>,
44
+ cute::enable_if_t<
45
+ not detail::is_use_rmem_A<ElementA, GmemLayoutATag, ElementB, GmemLayoutBTag>()>
46
+> {
47
+ using KernelScheduleType = KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<ScaleGranularityM>;
48
+
49
+ static_assert(is_static<TileShape_MNK>::value);
50
+ static_assert(is_static<ClusterShape_MNK>::value);
51
+#ifndef CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED
52
+ static_assert(cutlass::detail::dependent_false<ElementA>, "Unsupported Toolkit for SM90 Collective Builder\n");
53
+#endif
54
+ static_assert(detail::is_aligned<ElementA, AlignmentA, ElementB, AlignmentB, detail::tma_alignment_bytes>(),
55
+ "Should meet TMA alignment requirement\n");
56
+
57
+ static constexpr bool IsArrayOfPointersGemm = (cute::is_any_of_v<KernelScheduleType,
58
+ KernelPtrArrayTmaWarpSpecializedCooperative,
59
+ KernelPtrArrayTmaWarpSpecializedPingpong>);
60
+ static constexpr bool IsFP8Input = detail::is_input_fp8<ElementA, ElementB>();
61
+ static_assert((!IsFP8Input || !IsArrayOfPointersGemm),
62
+ "KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum is only compatible with FP8 Blocked Scaled version right now.");
63
+
64
+ // For fp32 types, map to tf32 MMA value type
65
+ using ElementAMma = cute::conditional_t<cute::is_same_v<ElementA, float>, tfloat32_t, ElementA>;
66
+ using ElementBMma = cute::conditional_t<cute::is_same_v<ElementB, float>, tfloat32_t, ElementB>;
67
+
68
+ static constexpr cute::GMMA::Major GmmaMajorA = detail::gmma_ss_tag_to_major_A<ElementAMma, GmemLayoutATag>();
69
+ static constexpr cute::GMMA::Major GmmaMajorB = detail::gmma_ss_tag_to_major_B<ElementBMma, GmemLayoutBTag>();
70
+
71
+ static constexpr bool IsCooperative = cute::is_any_of_v<KernelScheduleType,
72
+ KernelTmaWarpSpecializedCooperative,
73
+ KernelPtrArrayTmaWarpSpecializedCooperative,
74
+ KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<ScaleGranularityM>>;
75
+ using AtomLayoutMNK = cute::conditional_t<IsCooperative,
76
+ Layout<Shape<_2,_1,_1>>, Layout<Shape<_1,_1,_1>>>;
77
+
78
+ using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::ss_op_selector<
79
+ ElementAMma, ElementBMma, ElementAccumulator, TileShape_MNK, GmmaMajorA, GmmaMajorB>(), AtomLayoutMNK{}));
80
+
81
+ using GmemTiledCopyA = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
82
+ using GmemTiledCopyB = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
83
+
84
+ using SmemLayoutAtomA = decltype(detail::ss_smem_selector<
85
+ GmmaMajorA, ElementAMma, decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
86
+ using SmemLayoutAtomB = decltype(detail::ss_smem_selector<
87
+ GmmaMajorB, ElementBMma, decltype(cute::get<1>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
88
+
89
+ static constexpr size_t TensorMapStorage = IsArrayOfPointersGemm ? sizeof(cute::TmaDescriptor) * 2 /* for A and B */ : 0;
90
+ static constexpr int KernelSmemCarveout = static_cast<int>(TensorMapStorage);
91
+
92
+ static constexpr int PipelineStages = detail::compute_stage_count_or_override<detail::sm90_smem_capacity_bytes - KernelSmemCarveout,
93
+ ElementAMma, ElementBMma, TileShape_MNK>(StageCountType{});
94
+ using DispatchPolicy = MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8<PipelineStages, ClusterShape_MNK, KernelScheduleType, ScaleGranularityM>;
95
+
96
+ using SmemCopyAtomA = void;
97
+ using SmemCopyAtomB = void;
98
+
99
+ using CollectiveOp = CollectiveMma<
100
+ DispatchPolicy,
101
+ TileShape_MNK,
102
+ ElementA,
103
+ TagToStrideA_t<GmemLayoutATag>,
104
+ ElementB,
105
+ TagToStrideB_t<GmemLayoutBTag>,
106
+ TiledMma,
107
+ GmemTiledCopyA,
108
+ SmemLayoutAtomA,
109
+ SmemCopyAtomA,
110
+ cute::identity,
111
+ GmemTiledCopyB,
112
+ SmemLayoutAtomB,
113
+ SmemCopyAtomB,
114
+ cute::identity
115
+ >;
116
+};
117
+
118
+
119
+/////////////////////////////////////////////////////////////////////////////////////////////////
120
+
121
+} // namespace cutlass::gemm::collective
122
+
123
+/////////////////////////////////////////////////////////////////////////////////////////////////
csrc/cutlass_extensions/gemm/collective/fp8_accumulation.hpp
new
+183
@@ -0,0 +1,183 @@
1
+// clang-format off
2
+// adapted from: https://github.com/soundOfDestiny/cutlass/blob/a4208aa6958864923505cade9c63eb2a6daf16e5/include/cutlass/gemm/collective/fp8_accumulation.hpp
3
+
4
+/***************************************************************************************************
5
+ * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6
+ * SPDX-License-Identifier: BSD-3-Clause
7
+ *
8
+ * Redistribution and use in source and binary forms, with or without
9
+ * modification, are permitted provided that the following conditions are met:
10
+ *
11
+ * 1. Redistributions of source code must retain the above copyright notice, this
12
+ * list of conditions and the following disclaimer.
13
+ *
14
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
15
+ * this list of conditions and the following disclaimer in the documentation
16
+ * and/or other materials provided with the distribution.
17
+ *
18
+ * 3. Neither the name of the copyright holder nor the names of its
19
+ * contributors may be used to endorse or promote products derived from
20
+ * this software without specific prior written permission.
21
+ *
22
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
26
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
28
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
+ *
33
+ **************************************************************************************************/
34
+
35
+#pragma once
36
+
37
+#include "cute/algorithm/clear.hpp"
38
+#include "cute/tensor.hpp"
39
+
40
+//////////////////////////////////////////////////////////////////////////////
41
+///////////////////////////////////FP8 Accumulation///////////////////////////
42
+//////////////////////////////////////////////////////////////////////////////
43
+/// This class provides API to promote (add) or scale (multiply_add) the results
44
+/// from the tensor core accumulators to the main accumulators when the number
45
+/// of MMAs reaches the max number of MMA interval specified by user, after that
46
+/// the tensor core accumulators are zeroed.
47
+//////////////////////////////////////////////////////////////////////////////
48
+
49
+namespace cutlass::gemm::collective {
50
+
51
+template <
52
+ class EngineAccum,
53
+ class LayoutAccum>
54
+struct GmmaFP8AccumulationWithScale {
55
+ using TensorAccum = cute::Tensor<EngineAccum, LayoutAccum>;
56
+ using ElementAccumulator = typename EngineAccum::value_type;
57
+
58
+ static_assert(is_static<LayoutAccum>::value, "Accumulator Layout should be static");
59
+ static_assert(is_rmem<TensorAccum>::value , "Accumulator tensor must be rmem resident.");
60
+
61
+private:
62
+ TensorAccum& accum_;
63
+ TensorAccum accum_temp_;
64
+
65
+ uint32_t accum_promotion_interval_; // defines the max num of executed MMAs after which accum should be promoted.
66
+ uint32_t mma_count_per_mainloop_iteration_; // num of MMAs per k_tile of mainloop
67
+ uint32_t mma_count_; // current executed MMAs
68
+ uint32_t reset_accum_flag_; // accum needs to be zeroed or not.
69
+
70
+ // promote or `add` the partial accumulators to main accumulator (FADD).
71
+ CUTLASS_DEVICE
72
+ void promote_core() {
73
+ warpgroup_wait<0>();
74
+ CUTLASS_PRAGMA_UNROLL
75
+ for (int i = 0; i < size(accum_); ++i) {
76
+ accum_(i) += accum_temp_(i);
77
+ }
78
+ }
79
+
80
+ // `multiply` scale the partial accumulators and `add` to main accumulator (FFMA).
81
+ template <
82
+ class EngineScale,
83
+ class LayoutScale>
84
+ CUTLASS_DEVICE
85
+ void scale_core(const cute::Tensor<EngineScale, LayoutScale> &scale) {
86
+ using TensorScale = cute::Tensor<EngineScale, LayoutScale>;
87
+
88
+ static_assert(is_static<LayoutScale>::value, "Scale Layout should be static");
89
+ static_assert(is_rmem<TensorScale>::value , "Scale tensor must be rmem resident.");
90
+
91
+ static_assert(LayoutAccum{}.shape() == LayoutScale{}.shape(), "Accumulator and scale must have same shape.");
92
+
93
+ warpgroup_wait<0>();
94
+ CUTLASS_PRAGMA_UNROLL
95
+ for (int i = 0; i < size(accum_); ++i) {
96
+ accum_(i) += accum_temp_(i) * scale(i);
97
+ }
98
+ }
99
+
100
+public:
101
+ CUTLASS_DEVICE
102
+ GmmaFP8AccumulationWithScale(
103
+ TensorAccum &accum,
104
+ uint32_t accum_promotion_interval,
105
+ uint32_t mma_count_per_mainloop_iteration)
106
+ : accum_(accum),
107
+ accum_promotion_interval_(accum_promotion_interval),
108
+ mma_count_per_mainloop_iteration_(mma_count_per_mainloop_iteration),
109
+ mma_count_(0),
110
+ reset_accum_flag_(0)
111
+ {
112
+ accum_temp_ = cute::make_fragment_like(accum);
113
+ }
114
+
115
+ //
116
+ // Methods (Common)
117
+ //
118
+
119
+ CUTLASS_DEVICE
120
+ TensorAccum& operator()() {
121
+ return accum_temp_;
122
+ }
123
+
124
+ /// prepare the MMA accumulators when initialization or zeroing is required.
125
+ CUTLASS_DEVICE
126
+ bool prepare_if_needed() {
127
+ return reset_accum_flag_;
128
+ }
129
+
130
+ //
131
+ // Methods (for FADD version)
132
+ //
133
+
134
+ /// promote (add) the results from the MMA accumulators to main accumulator if needed.
135
+ CUTLASS_DEVICE
136
+ void promote_if_needed() {
137
+ mma_count_ += mma_count_per_mainloop_iteration_;
138
+ reset_accum_flag_ = __shfl_sync(0xffffffff, mma_count_ == accum_promotion_interval_, 0);
139
+ if (reset_accum_flag_) {
140
+ promote_core();
141
+ mma_count_ = 0;
142
+ }
143
+ }
144
+
145
+ /// promote (add) the residue results from the MMA accumulators to main accumulator if needed.
146
+ CUTLASS_DEVICE
147
+ void promote_residue_if_needed() {
148
+ if (__shfl_sync(0xffffffff, mma_count_ > 0, 0)) {
149
+ promote_core();
150
+ }
151
+ }
152
+
153
+ //
154
+ // Methods (for FFMA version)
155
+ //
156
+
157
+ /// scale (multiply_add) the results from the MMA accumulators to main accumulator if needed.
158
+ template <
159
+ class EngineScale,
160
+ class LayoutScale>
161
+ CUTLASS_DEVICE
162
+ void scale_if_needed(const cute::Tensor<EngineScale, LayoutScale> &scale) {
163
+ mma_count_ += mma_count_per_mainloop_iteration_;
164
+ reset_accum_flag_ = __shfl_sync(0xffffffff, mma_count_ == accum_promotion_interval_, 0);
165
+ if (reset_accum_flag_) {
166
+ scale_core(scale);
167
+ mma_count_ = 0;
168
+ }
169
+ }
170
+
171
+ /// scale (multiply_add) the residue results from the MMA accumulators to main accumulator if needed.
172
+ template <
173
+ class EngineScale,
174
+ class LayoutScale>
175
+ CUTLASS_DEVICE
176
+ void scale_residue_if_needed(const cute::Tensor<EngineScale, LayoutScale> &scale) {
177
+ if (__shfl_sync(0xffffffff, mma_count_ > 0, 0)) {
178
+ scale_core(scale);
179
+ }
180
+ }
181
+};
182
+
183
+} // namespace cutlass::gemm::collective
csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp
new
+730
@@ -0,0 +1,730 @@
1
+// clang-format off
2
+// Adapted (Heavily) from: https://github.com/soundOfDestiny/cutlass/blob/9d997ce0dea4c5fa1a617db6b7ff29aa9235822c/include/cutlass/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp
3
+
4
+/***************************************************************************************************
5
+ * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6
+ * SPDX-License-Identifier: BSD-3-Clause
7
+ *
8
+ * Redistribution and use in source and binary forms, with or without
9
+ * modification, are permitted provided that the following conditions are met:
10
+ *
11
+ * 1. Redistributions of source code must retain the above copyright notice, this
12
+ * list of conditions and the following disclaimer.
13
+ *
14
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
15
+ * this list of conditions and the following disclaimer in the documentation
16
+ * and/or other materials provided with the distribution.
17
+ *
18
+ * 3. Neither the name of the copyright holder nor the names of its
19
+ * contributors may be used to endorse or promote products derived from
20
+ * this software without specific prior written permission.
21
+ *
22
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
26
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
28
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
+ *
33
+ **************************************************************************************************/
34
+
35
+#pragma once
36
+
37
+#include "cutlass/cutlass.h"
38
+#include "cutlass/gemm/dispatch_policy.hpp"
39
+#include "cutlass/trace.h"
40
+#include "cutlass/numeric_types.h"
41
+
42
+#include "cute/arch/cluster_sm90.hpp"
43
+#include "cute/arch/copy_sm80.hpp"
44
+#include "cute/arch/copy_sm90.hpp"
45
+#include "cute/algorithm/functional.hpp"
46
+#include "cute/atom/mma_atom.hpp"
47
+#include "cute/algorithm/gemm.hpp"
48
+#include "cute/tensor_predicate.hpp"
49
+#include "cute/numeric/arithmetic_tuple.hpp"
50
+
51
+#include "cutlass_extensions/gemm/dispatch_policy.hpp"
52
+#include "cutlass_extensions/gemm/collective/fp8_accumulation.hpp"
53
+
54
+/////////////////////////////////////////////////////////////////////////////////////////////////
55
+
56
+namespace cutlass::gemm::collective {
57
+using namespace cute;
58
+
59
+/////////////////////////////////////////////////////////////////////////////////////////////////
60
+
61
+// WarpSpecialized Mainloop
62
+template <
63
+ int Stages,
64
+ class ClusterShape,
65
+ class KernelSchedule,
66
+ int ScaleGranularityM_,
67
+ class TileShape_,
68
+ class ElementA_,
69
+ class StrideA_,
70
+ class ElementB_,
71
+ class StrideB_,
72
+ class TiledMma_,
73
+ class GmemTiledCopyA_,
74
+ class SmemLayoutAtomA_,
75
+ class SmemCopyAtomA_,
76
+ class TransformA_,
77
+ class GmemTiledCopyB_,
78
+ class SmemLayoutAtomB_,
79
+ class SmemCopyAtomB_,
80
+ class TransformB_>
81
+struct CollectiveMma<
82
+ MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8<Stages, ClusterShape, KernelSchedule, ScaleGranularityM_>,
83
+ TileShape_,
84
+ ElementA_,
85
+ StrideA_,
86
+ ElementB_,
87
+ StrideB_,
88
+ TiledMma_,
89
+ GmemTiledCopyA_,
90
+ SmemLayoutAtomA_,
91
+ SmemCopyAtomA_,
92
+ TransformA_,
93
+ GmemTiledCopyB_,
94
+ SmemLayoutAtomB_,
95
+ SmemCopyAtomB_,
96
+ TransformB_>
97
+{
98
+ //
99
+ // Type Aliases
100
+ //
101
+ using DispatchPolicy = MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8<Stages, ClusterShape, KernelSchedule, ScaleGranularityM_>;
102
+ using TileShape = TileShape_;
103
+ using ElementA = ElementA_;
104
+ using StrideA = StrideA_;
105
+ using ElementB = ElementB_;
106
+ using StrideB = StrideB_;
107
+ using TiledMma = TiledMma_;
108
+ using ElementAccumulator = typename TiledMma::ValTypeC;
109
+ using ElementBlockScale = ElementAccumulator;
110
+ using GmemTiledCopyA = GmemTiledCopyA_;
111
+ using GmemTiledCopyB = GmemTiledCopyB_;
112
+ using SmemLayoutAtomA = SmemLayoutAtomA_;
113
+ using SmemLayoutAtomB = SmemLayoutAtomB_;
114
+ using SmemCopyAtomA = SmemCopyAtomA_;
115
+ using SmemCopyAtomB = SmemCopyAtomB_;
116
+ using TransformA = TransformA_;
117
+ using TransformB = TransformB_;
118
+ using ArchTag = typename DispatchPolicy::ArchTag;
119
+
120
+ using CtaShape_MNK = decltype(shape_div(TileShape{}, ClusterShape{}));
121
+ using MainloopPipeline = cutlass::PipelineTmaAsync<DispatchPolicy::Stages>;
122
+ using PipelineState = cutlass::PipelineState<DispatchPolicy::Stages>;
123
+ using PipelineParams = typename MainloopPipeline::Params;
124
+
125
+ // Two threads per CTA are producers (1 for operand tile and 32 for scales)
126
+ static constexpr int NumProducerThreadEvents = 33;
127
+
128
+ static constexpr int ScaleGranularityM = ScaleGranularityM_ == 0 ? size<0>(TileShape{}) : ScaleGranularityM_;
129
+ static constexpr int ScaleMsPerTile = size<0>(TileShape{}) / ScaleGranularityM;
130
+
131
+ static_assert(cute::rank(SmemLayoutAtomA{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)");
132
+ static_assert((size<0>(TileShape{}) % size<0>(SmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
133
+ static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
134
+
135
+ static_assert(cute::rank(SmemLayoutAtomB{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)");
136
+ static_assert((size<1>(TileShape{}) % size<0>(SmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
137
+ static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
138
+
139
+ static_assert((size<0>(TileShape{}) % ScaleGranularityM) == 0, "FP8 scaling granularity must evenly divide tile shape along M.");
140
+
141
+ // Tile along modes in a way that maximizes the TMA box size.
142
+ using SmemLayoutA = decltype(tile_to_shape(
143
+ SmemLayoutAtomA{},
144
+ make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int<DispatchPolicy::Stages>{}),
145
+ cute::conditional_t< ::cutlass::gemm::detail::is_major<0,StrideA>(), Step<_2,_1,_3>, Step<_1,_2,_3>>{}));
146
+ using SmemLayoutB = decltype(tile_to_shape(
147
+ SmemLayoutAtomB{},
148
+ make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int<DispatchPolicy::Stages>{}),
149
+ cute::conditional_t< ::cutlass::gemm::detail::is_major<0,StrideB>(), Step<_2,_1,_3>, Step<_1,_2,_3>>{}));
150
+
151
+ // Block scaling gmem-to-smem copy atom
152
+ using SmemBlockScalingCopyAtomA = Copy_Atom<SM80_CP_ASYNC_CACHEALWAYS<ElementBlockScale>, ElementBlockScale>;
153
+ using SmemBlockScalingCopyAtomB = Copy_Atom<SM80_CP_ASYNC_CACHEALWAYS<ElementBlockScale>, ElementBlockScale>;
154
+
155
+ // Block scaling smem layout
156
+ using SmemLayoutScaleA = Layout<Shape<Int<ScaleMsPerTile>, Int<DispatchPolicy::Stages>>>;
157
+ using SmemLayoutScaleB = Layout<Shape<Int<DispatchPolicy::Stages>>, Stride<_1>>; // `ScaleNsPerTile` is always 1.
158
+
159
+ static_assert(DispatchPolicy::Stages >= 2, "Specialization requires Stages set to value 1 or more.");
160
+ static_assert(cute::is_base_of<cute::GMMA::DescriptorIterator, typename TiledMma::FrgTypeA>::value &&
161
+ cute::is_base_of<cute::GMMA::DescriptorIterator, typename TiledMma::FrgTypeB>::value,
162
+ "MMA atom must source both A and B operand from smem_desc for this mainloop.");
163
+ static_assert(cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD> || cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD_MULTICAST>,
164
+ "GmemTiledCopy - invalid SM90 TMA copy atom specified.");
165
+ static_assert(cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD> || cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD_MULTICAST>,
166
+ "GmemTiledCopy - invalid SM90 TMA copy atom specified.");
167
+ static_assert(cute::is_same_v<ElementAccumulator, ElementBlockScale>,
168
+ "ElementAccumulator and ElementBlockScale should be same datatype");
169
+
170
+ struct SharedStorage
171
+ {
172
+ struct TensorStorage : cute::aligned_struct<128> {
173
+ cute::array_aligned<typename TiledMma::ValTypeA, cute::cosize_v<SmemLayoutA>> smem_A; // mxk
174
+ cute::array_aligned<typename TiledMma::ValTypeB, cute::cosize_v<SmemLayoutB>> smem_B; // nxk
175
+ cute::array_aligned<ElementBlockScale, cute::cosize_v<SmemLayoutScaleA>> smem_scale_A; // ScaleMsPerTile x k
176
+ cute::array_aligned<ElementBlockScale, cute::cosize_v<SmemLayoutScaleB>> smem_scale_B; // 1xk
177
+ } tensors;
178
+
179
+ using PipelineStorage = typename MainloopPipeline::SharedStorage;
180
+ PipelineStorage pipeline;
181
+ };
182
+ using TensorStorage = typename SharedStorage::TensorStorage;
183
+ using PipelineStorage = typename SharedStorage::PipelineStorage;
184
+
185
+ // Host side kernel arguments
186
+ struct Arguments {
187
+ ElementA const* ptr_A;
188
+ StrideA dA;
189
+ ElementB const* ptr_B;
190
+ StrideB dB;
191
+ ElementBlockScale const* ptr_scale_A;
192
+ ElementBlockScale const* ptr_scale_B;
193
+ };
194
+
195
+ // Device side kernel params
196
+ struct Params {
197
+ // Assumption: StrideA is congruent with Problem_MK
198
+ using TMA_A = decltype(make_tma_copy_A_sm90(
199
+ GmemTiledCopyA{},
200
+ make_tensor(static_cast<ElementA const*>(nullptr), repeat_like(StrideA{}, int32_t(0)), StrideA{}),
201
+ SmemLayoutA{}(_,_,0),
202
+ TileShape{},
203
+ ClusterShape{}));
204
+ // Assumption: StrideB is congruent with Problem_NK
205
+ using TMA_B = decltype(make_tma_copy_B_sm90(
206
+ GmemTiledCopyB{},
207
+ make_tensor(static_cast<ElementB const*>(nullptr), repeat_like(StrideB{}, int32_t(0)), StrideB{}),
208
+ SmemLayoutB{}(_,_,0),
209
+ TileShape{},
210
+ ClusterShape{}));
211
+ TMA_A tma_load_a;
212
+ TMA_B tma_load_b;
213
+ uint32_t tma_transaction_bytes = TmaTransactionBytes;
214
+ uint32_t tma_transaction_bytes_mk = TmaTransactionBytesMK;
215
+ uint32_t tma_transaction_bytes_nk = TmaTransactionBytesNK;
216
+ // Block scaling factors for A and B
217
+ ElementBlockScale const* ptr_scale_A;
218
+ ElementBlockScale const* ptr_scale_B;
219
+ };
220
+
221
+ //
222
+ // Methods
223
+ //
224
+
225
+ template <class ProblemShape>
226
+ static constexpr Params
227
+ to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
228
+ (void) workspace;
229
+
230
+ // Optionally append 1s until problem shape is rank-4 (MNKL), in case it is only rank-3 (MNK)
231
+ auto problem_shape_MNKL = append<4>(problem_shape, 1);
232
+ auto [M,N,K,L] = problem_shape_MNKL;
233
+
234
+ auto ptr_A = reinterpret_cast<ElementA const*>(args.ptr_A);
235
+ auto ptr_B = reinterpret_cast<ElementB const*>(args.ptr_B);
236
+
237
+ Tensor tensor_a = make_tensor(ptr_A, make_layout(make_shape(M,K,L), args.dA));
238
+ Tensor tensor_b = make_tensor(ptr_B, make_layout(make_shape(N,K,L), args.dB));
239
+ typename Params::TMA_A tma_load_a = make_tma_copy_A_sm90(
240
+ GmemTiledCopyA{},
241
+ tensor_a,
242
+ SmemLayoutA{}(_,_,cute::Int<0>{}),
243
+ TileShape{},
244
+ ClusterShape{});
245
+ typename Params::TMA_B tma_load_b = make_tma_copy_B_sm90(
246
+ GmemTiledCopyB{},
247
+ tensor_b,
248
+ SmemLayoutB{}(_,_,cute::Int<0>{}),
249
+ TileShape{},
250
+ ClusterShape{});
251
+ uint32_t transaction_bytes_mk = TmaTransactionBytesMK;
252
+ uint32_t transaction_bytes_nk = TmaTransactionBytesNK;
253
+ uint32_t transaction_bytes = transaction_bytes_mk + transaction_bytes_nk;
254
+
255
+ return {
256
+ tma_load_a,
257
+ tma_load_b,
258
+ transaction_bytes,
259
+ transaction_bytes_mk,
260
+ transaction_bytes_nk,
261
+ args.ptr_scale_A,
262
+ args.ptr_scale_B
263
+ };
264
+ }
265
+
266
+ template<class ProblemShape>
267
+ static bool
268
+ can_implement(
269
+ ProblemShape const& problem_shape,
270
+ [[maybe_unused]] Arguments const& args) {
271
+ constexpr int tma_alignment_bits = 128;
272
+ auto problem_shape_MNKL = append<4>(problem_shape, 1);
273
+ auto [M,N,K,L] = problem_shape_MNKL;
274
+
275
+ bool implementable = true;
276
+ constexpr int min_tma_aligned_elements_A = tma_alignment_bits / cutlass::sizeof_bits<ElementA>::value;
277
+ implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_A>(cute::make_shape(M,K,L), StrideA{});
278
+ constexpr int min_tma_aligned_elements_B = tma_alignment_bits / cutlass::sizeof_bits<ElementB>::value;
279
+ implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_B>(cute::make_shape(N,K,L), StrideB{});
280
+
281
+ if (!implementable) {
282
+ CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Problem Size doesn't meet the minimum alignment requirements for TMA.\n");
283
+ }
284
+ return implementable;
285
+ }
286
+
287
+ static constexpr int K_PIPE_MAX = DispatchPolicy::Stages;
288
+ static constexpr int K_PIPE_MMAS = 1;
289
+ static constexpr uint32_t TmaTransactionBytesMK =
290
+ cutlass::bits_to_bytes(size<0>(SmemLayoutA{}) * size<1>(SmemLayoutA{}) * static_cast<uint32_t>(sizeof_bits<ElementA>::value));
291
+ static constexpr uint32_t TmaTransactionBytesNK =
292
+ cutlass::bits_to_bytes(size<0>(SmemLayoutB{}) * size<1>(SmemLayoutB{}) * static_cast<uint32_t>(sizeof_bits<ElementB>::value));
293
+ static constexpr uint32_t TmaTransactionBytes = TmaTransactionBytesMK + TmaTransactionBytesNK;
294
+
295
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
296
+ CUTLASS_DEVICE
297
+ static void prefetch_tma_descriptors(Params const& mainloop_params)
298
+ {
299
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_a.get_tma_descriptor());
300
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_b.get_tma_descriptor());
301
+ }
302
+
303
+ /// Set up the data needed by this collective for load and mma.
304
+ /// Returns a tuple of tensors. The collective and the kernel layer have the contract
305
+ /// Returned tuple must contain at least two elements, with the first two elements being:
306
+ /// gA_mkl - The tma tensor, A after a local tile so it has shape (BLK_M,BLK_K,m,k,l)
307
+ /// gB_nkl - The tma tensor, B after a local tile so it has shape (BLK_N,BLK_K,n,k,l)
308
+ template <class ProblemShape_MNKL>
309
+ CUTLASS_DEVICE auto
310
+ load_init(ProblemShape_MNKL const& problem_shape_MNKL, Params const& mainloop_params) const {
311
+ using X = Underscore;
312
+ // Separate out problem shape for convenience
313
+ auto [M,N,K,L] = problem_shape_MNKL;
314
+
315
+ // TMA requires special handling of strides to deal with coord codomain mapping
316
+ // Represent the full tensors -- get these from TMA
317
+ Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(make_shape(M,K,L)); // (m,k,l)
318
+ Tensor mB_nkl = mainloop_params.tma_load_b.get_tma_tensor(make_shape(N,K,L)); // (n,k,l)
319
+
320
+ // Make tiled views, defer the slice
321
+ Tensor gA_mkl = local_tile(mA_mkl, TileShape{}, make_coord(_,_,_), Step<_1, X,_1>{}); // (BLK_M,BLK_K,m,k,l)
322
+ Tensor gB_nkl = local_tile(mB_nkl, TileShape{}, make_coord(_,_,_), Step< X,_1,_1>{}); // (BLK_N,BLK_K,n,k,l)
323
+
324
+ constexpr auto scales_m = Int<ScaleMsPerTile>{};
325
+ auto tM = get<2>(gA_mkl.shape());
326
+ auto tN = get<2>(gB_nkl.shape());
327
+ auto tK = get<3>(gA_mkl.shape());
328
+
329
+ // Make the tiled views of scale tensors
330
+ auto scaleA_shape = make_shape(M / ScaleGranularityM, tK, L); // (scale_m,k,l)
331
+ auto scaleA_layout = make_ordered_layout(scaleA_shape, Step<_0, _1, _2>{});
332
+ auto scaleB_shape = make_shape(tN, tK, L); // (n,k,l)
333
+ auto scaleB_layout = make_ordered_layout(scaleB_shape, Step<_1, _0, _2>{});
334
+
335
+ // Note that mScaleA_mkl and mScaleB_nkl are already blocked tiled in the `m` host and
336
+ // gScaleA_mkl and gScaleB_nkl in `g` global memory are same as mScaleA_mkl and mScaleB_nkl.
337
+ Tensor mScaleA_mkl = make_tensor(make_gmem_ptr(mainloop_params.ptr_scale_A), scaleA_layout); // (scale_m,k,l)
338
+ Tensor mScaleB_nkl = make_tensor(make_gmem_ptr(mainloop_params.ptr_scale_B), scaleB_layout); // (n,k,l)
339
+
340
+ return cute::make_tuple(gA_mkl, gB_nkl, mScaleA_mkl, mScaleB_nkl);
341
+ }
342
+
343
+ /// Perform a collective-scoped matrix multiply-accumulate
344
+ /// Producer Perspective
345
+ template <
346
+ class TensorA, class TensorB,
347
+ class TensorScaleA, class TensorScaleB,
348
+ class KTileIterator, class BlockCoord
349
+ >
350
+ CUTLASS_DEVICE void
351
+ load(
352
+ Params const& mainloop_params,
353
+ MainloopPipeline pipeline,
354
+ PipelineState smem_pipe_write,
355
+ cute::tuple<TensorA, TensorB, TensorScaleA, TensorScaleB> const& load_inputs,
356
+ BlockCoord const& blk_coord,
357
+ KTileIterator k_tile_iter, int k_tile_count,
358
+ int thread_idx,
359
+ uint32_t block_rank_in_cluster,
360
+ TensorStorage& shared_tensors) {
361
+ int lane_predicate = cute::elect_one_sync();
362
+
363
+ // Blockscaling: Tma loads for load_input and CpAsync for load_scale
364
+ Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.data()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE)
365
+ Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.data()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE)
366
+ Tensor sScaleA = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_A.data()), SmemLayoutScaleA{}); // (ScaleMsPerTile,k)
367
+ Tensor sScaleB = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_B.data()), SmemLayoutScaleB{}); // (k)
368
+
369
+ //
370
+ // Prepare the TMA loads for A and B
371
+ //
372
+
373
+ constexpr uint32_t cluster_shape_x = get<0>(ClusterShape());
374
+ uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x};
375
+
376
+ Tensor gA_mkl = get<0>(load_inputs);
377
+ Tensor gB_nkl = get<1>(load_inputs);
378
+
379
+ auto block_tma_a = mainloop_params.tma_load_a.get_slice(cluster_local_block_id.y);
380
+ auto block_tma_b = mainloop_params.tma_load_b.get_slice(cluster_local_block_id.x);
381
+
382
+ // Partition the inputs based on the current block coordinates.
383
+ auto [m_coord, n_coord, k_coord, l_coord] = blk_coord;
384
+ Tensor gA = gA_mkl(_,_,m_coord,_,l_coord); // (BLK_M,BLK_K,k)
385
+ Tensor gB = gB_nkl(_,_,n_coord,_,l_coord); // (BLK_N,BLK_K,k)
386
+
387
+
388
+ // Block scaling: load_scale has scaling tensors in global memory which are not tiled
389
+ Tensor mScaleA_mkl = get<2>(load_inputs);
390
+ Tensor mScaleB_nkl = get<3>(load_inputs);
391
+ auto scales_m = get<0>(mScaleA_mkl.shape());
392
+
393
+ Tensor cScaleA_mkl = make_identity_tensor(mScaleA_mkl.shape());
394
+
395
+ Tensor gScaleA = local_tile(
396
+ mScaleA_mkl, make_tile(Int<ScaleMsPerTile>{}),
397
+ make_coord(m_coord,_,l_coord)); // (ScaleMsPerTile,k,1)
398
+ Tensor cScaleA = local_tile(
399
+ cScaleA_mkl, make_tile(Int<ScaleMsPerTile>{}),
400
+ make_coord(m_coord,_,l_coord));
401
+ Tensor gScaleB = mScaleB_nkl(n_coord,_,l_coord); // (1,k,1)
402
+
403
+ // TODO: test `scale_copy_a` with `ScaleMsPerTile` < 128
404
+ TiledCopy scale_copy_a = make_tiled_copy(SmemBlockScalingCopyAtomA{},
405
+ Layout<Shape<_32, _1>>{}, Layout<Shape<_4, _1>>{}); // (1,1,1)
406
+ TiledCopy scale_copy_b = make_tiled_copy(SmemBlockScalingCopyAtomB{},
407
+ Layout<Shape<_1>>{}, Layout<Shape<_1>>{}); // (1,1,1)
408
+ ThrCopy thr_scale_copy_a = scale_copy_a.get_slice(threadIdx.x);
409
+ ThrCopy thr_scale_copy_b = scale_copy_b.get_slice(threadIdx.x);
410
+
411
+ Tensor tAgA_ScaleA = thr_scale_copy_a.partition_S(gScaleA);
412
+ Tensor tAcA_ScaleA = thr_scale_copy_a.partition_S(cScaleA);
413
+ Tensor tAsA_ScaleA = thr_scale_copy_a.partition_D(sScaleA);
414
+
415
+ Tensor tBgB_ScaleB = thr_scale_copy_b.partition_S(gScaleB);
416
+ Tensor tBsB_ScaleB = thr_scale_copy_b.partition_D(sScaleB);
417
+
418
+ // Applies the mapping from block_tma_a
419
+ Tensor tAgA = block_tma_a.partition_S(gA); // (TMA,TMA_M,TMA_K,k)
420
+ Tensor tAsA = block_tma_a.partition_D(sA); // (TMA,TMA_M,TMA_K,PIPE)
421
+
422
+ Tensor tBgB = block_tma_b.partition_S(gB); // (TMA,TMA_N,TMA_K,k)
423
+ Tensor tBsB = block_tma_b.partition_D(sB); // (TMA,TMA_N,TMA_K,PIPE)
424
+
425
+ uint16_t mcast_mask_a = 0;
426
+ uint16_t mcast_mask_b = 0;
427
+
428
+ // Issue TmaLoads for GEMM operands A/B and CpAsync for scale tensors
429
+ // Maps the tile -> block, value
430
+ if constexpr (cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD_MULTICAST>) {
431
+ auto block_layout = Layout<typename DispatchPolicy::ClusterShape>{}; // (m,n) -> block_id
432
+ for (int n = 0; n < size<1>(block_layout); ++n) {
433
+ mcast_mask_a |= (uint16_t(1) << block_layout(cluster_local_block_id.x,n,Int<0>{}));
434
+ }
435
+ }
436
+
437
+ if constexpr (cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD_MULTICAST>) {
438
+ auto block_layout = Layout<typename DispatchPolicy::ClusterShape>{}; // (m,n) -> block_id
439
+ for (int m = 0; m < size<0>(block_layout); ++m) {
440
+ mcast_mask_b |= (uint16_t(1) << block_layout(m,cluster_local_block_id.y,Int<0>{}));
441
+ }
442
+ }
443
+
444
+ // Allocate predicate tensors for a_scales (since we can't guarantee that
445
+ // all scales are valid, since we could have a partial tiles along M)
446
+ Tensor tApA_ScaleA = make_tensor<bool>(shape(tAsA_ScaleA(_,_,0)));
447
+ #pragma unroll
448
+ for (int i = 0; i < size(tApA_ScaleA); ++i) {
449
+ tApA_ScaleA(i) = get<0>(tAcA_ScaleA(i)) < scales_m;
450
+ }
451
+
452
+ // Mainloop
453
+ CUTLASS_PRAGMA_NO_UNROLL
454
+ for ( ; k_tile_count > 0; --k_tile_count) {
455
+ // LOCK smem_pipe_write for _writing_
456
+ pipeline.producer_acquire(smem_pipe_write);
457
+
458
+ //
459
+ // Copy gmem to smem for *k_tile_iter
460
+ //
461
+ int write_stage = smem_pipe_write.index();
462
+ using BarrierType = typename MainloopPipeline::ProducerBarrierType;
463
+ BarrierType* tma_barrier = pipeline.producer_get_barrier(smem_pipe_write);
464
+
465
+ // Copy operands A and B from global memory to shared memory
466
+ if (lane_predicate) copy(mainloop_params.tma_load_a.with(*tma_barrier, mcast_mask_a), tAgA(_,_,_,*k_tile_iter), tAsA(_,_,_,write_stage));
467
+ if (lane_predicate) copy(mainloop_params.tma_load_b.with(*tma_barrier, mcast_mask_b), tBgB(_,_,_,*k_tile_iter), tBsB(_,_,_,write_stage));
468
+
469
+ // Copy scale tensors from global memory to shared memory
470
+ copy_if(scale_copy_a, tApA_ScaleA, tAgA_ScaleA(_,_,*k_tile_iter), tAsA_ScaleA(_,_,write_stage));
471
+ copy(scale_copy_b, tBgB_ScaleB(_,*k_tile_iter), tBsB_ScaleB(_,write_stage));
472
+ pipeline.producer_commit(smem_pipe_write, cutlass::arch::cpasync_barrier_arrive_noinc);
473
+
474
+ ++k_tile_iter;
475
+
476
+ // Advance smem_pipe_write
477
+ ++smem_pipe_write;
478
+ }
479
+ }
480
+
481
+ /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster
482
+ CUTLASS_DEVICE void
483
+ load_tail(
484
+ MainloopPipeline pipeline,
485
+ PipelineState smem_pipe_write) {
486
+ int lane_predicate = cute::elect_one_sync();
487
+
488
+ // Issue the epilogue waits
489
+ if (lane_predicate) {
490
+ /* This helps avoid early exit of blocks in Cluster
491
+ * Waits for all stages to either be released (all
492
+ * Consumer UNLOCKs), or if the stage was never used
493
+ * then would just be acquired since the phase was
494
+ * still inverted from make_producer_start_state
495
+ */
496
+ pipeline.producer_tail(smem_pipe_write);
497
+ }
498
+ }
499
+
500
+ /// Perform a collective-scoped matrix multiply-accumulate
501
+ /// Consumer Perspective
502
+ template <
503
+ class FrgTensorC
504
+ >
505
+ CUTLASS_DEVICE void
506
+ mma(MainloopPipeline pipeline,
507
+ PipelineState smem_pipe_read,
508
+ FrgTensorC& accum,
509
+ int k_tile_count,
510
+ int thread_idx,
511
+ TensorStorage& shared_tensors,
512
+ Params const& mainloop_params) {
513
+
514
+
515
+ static_assert(is_rmem<FrgTensorC>::value, "C tensor must be rmem resident.");
516
+ static_assert(cute::rank(SmemLayoutA{}) == 3, "Smem layout must be rank 3.");
517
+ static_assert(cute::rank(SmemLayoutB{}) == 3, "Smem layout must be rank 3.");
518
+ static_assert(cute::is_void_v<SmemCopyAtomA>,
519
+ "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions.");
520
+ static_assert(cute::is_void_v<SmemCopyAtomB>,
521
+ "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions.");
522
+
523
+ Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.data()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE)
524
+ Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.data()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE)
525
+
526
+ // Block scaling
527
+ Tensor sScaleAViewAsC = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_A.data()),
528
+ Layout<
529
+ Shape<Shape<Int<ScaleGranularityM>, Int<ScaleMsPerTile>>, cute::tuple_element_t<1, TileShape>, Int<DispatchPolicy::Stages>>,
530
+ Stride<Stride<_0, _1>, _0, Int<ScaleMsPerTile>>
531
+ >{}); // ((ScaleGranularityM,ScaleMsPerTile),n,k)
532
+ Tensor sScaleB = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_B.data()), SmemLayoutScaleB{}); // (k)
533
+
534
+ //
535
+ // Define C accumulators and A/B partitioning
536
+ //
537
+
538
+ // Layout of warp group to thread mapping
539
+
540
+ static_assert(stride<0>(typename TiledMma::ALayout{}) == 0 and
541
+ stride<0>(typename TiledMma::BLayout{}) == 0 and
542
+ size<0>(typename TiledMma::ALayout{}) == NumThreadsPerWarpGroup and
543
+ size<0>(typename TiledMma::BLayout{}) == NumThreadsPerWarpGroup,
544
+ "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup");
545
+
546
+ constexpr int MmaWarpGroups = size(TiledMma{}) / NumThreadsPerWarpGroup;
547
+ Layout warp_group_thread_layout = make_layout(Int<MmaWarpGroups>{},
548
+ Int<NumThreadsPerWarpGroup>{});
549
+
550
+ int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / NumThreadsPerWarpGroup, 0);
551
+
552
+ TiledMma tiled_mma;
553
+ auto thread_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx));
554
+
555
+ Tensor tCsScaleAViewAsC = tiled_mma.get_slice(thread_idx).partition_C(sScaleAViewAsC); // (MMA,MMA_M,MMA_N,PIPE), `thread_mma` above is correct when partitioning A and B, but it is not correct when partitioning C.
556
+
557
+ Tensor tCsA = thread_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,PIPE)
558
+ Tensor tCsB = thread_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE)
559
+
560
+ // Allocate "fragments/descriptors"
561
+ Tensor tCrA = thread_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K,PIPE)
562
+ Tensor tCrB = thread_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE)
563
+
564
+ CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(accum)); // M
565
+ CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<2>(accum)); // N
566
+ CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCsB)); // K
567
+ CUTE_STATIC_ASSERT_V(size<3>(tCsA) == size<3>(tCsB)); // PIPE
568
+ CUTE_STATIC_ASSERT_V(Int<DispatchPolicy::Stages>{} == size<2>(sA)); // PIPE
569
+ CUTE_STATIC_ASSERT_V(Int<DispatchPolicy::Stages>{} == size<2>(sB)); // PIPE
570
+
571
+ //
572
+ // PIPELINED MAIN LOOP
573
+ //
574
+ static_assert((0 <= K_PIPE_MMAS) && (K_PIPE_MMAS < K_PIPE_MAX),
575
+ "ERROR : Incorrect number of MMAs in flight");
576
+
577
+ // We release buffers to producer warps(dma load) with some mmas in flight
578
+ PipelineState smem_pipe_release = smem_pipe_read;
579
+
580
+ // Per block scale values for operand A and B
581
+
582
+ using RegLayoutScaleAViewAsC = decltype(make_layout_like(tCsScaleAViewAsC(_, _, _, 0).layout())); // `make_layout_like` makes a compact layout.
583
+ using RegLayoutScaleAEssential = decltype(filter_zeros(RegLayoutScaleAViewAsC{}.stride(), RegLayoutScaleAViewAsC{}.shape())); // an interface to traverse the underlying storage for the compact layout mentioned above
584
+
585
+ Tensor tCrScaleAViewAsC = make_tensor<ElementBlockScale>(RegLayoutScaleAViewAsC{}); // (MMA,MMA_M,MMA_N)
586
+ ElementBlockScale scale_b;
587
+
588
+ // Prologue GMMAs
589
+ int prologue_mma_count = min(K_PIPE_MMAS, k_tile_count);
590
+
591
+ tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
592
+
593
+ GmmaFP8AccumulationWithScale accumulation(accum, size<2>(TileShape{}) / size<2>(typename TiledMma::AtomShape_MNK{}), size<2>(tCrA));
594
+ warpgroup_fence_operand(accumulation());
595
+ CUTLASS_PRAGMA_UNROLL
596
+ for (int k_tile_prologue = prologue_mma_count; k_tile_prologue > 0; --k_tile_prologue)
597
+ {
598
+ // WAIT on smem_pipe_read until its data are available (phase bit flips from rdPhaseBit value)
599
+ auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
600
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
601
+
602
+ if (accumulation.prepare_if_needed()) {
603
+ tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
604
+ }
605
+
606
+ int read_stage = smem_pipe_read.index();
607
+
608
+ // Load per block scale values from shared memory to registers.
609
+ scale_b = sScaleB[read_stage];
610
+ CUTLASS_PRAGMA_UNROLL
611
+ for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
612
+ tCrScaleAViewAsC.data()[i] = tCsScaleAViewAsC(_, _, _, read_stage)(idx2crd(i, RegLayoutScaleAEssential{}));
613
+ }
614
+ if constexpr (ScaleMsPerTile == 1) {
615
+ static_assert(size(RegLayoutScaleAEssential{}) == 1);
616
+ tCrScaleAViewAsC.data()[0] = __shfl_sync(0xffffffff, tCrScaleAViewAsC.data()[0] * scale_b, 0); // `tCrScaleAViewAsC.data()[0]` are all same in a warp group when `ScaleMsPerTile == 1`.
617
+ } else {
618
+ CUTLASS_PRAGMA_UNROLL
619
+ for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
620
+ tCrScaleAViewAsC.data()[i] = tCrScaleAViewAsC.data()[i] * scale_b;
621
+ }
622
+ }
623
+
624
+ warpgroup_arrive();
625
+ // Unroll the K mode manually to set scale D to 1
626
+ CUTLASS_PRAGMA_UNROLL
627
+ for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) {
628
+ // (V,M,K) x (V,N,K) => (V,M,N)
629
+ cute::gemm(tiled_mma, tCrA(_,_,k_block,read_stage), tCrB(_,_,k_block,read_stage), accumulation());
630
+ tiled_mma.accumulate_ = GMMA::ScaleOut::One;
631
+ }
632
+ warpgroup_commit_batch();
633
+
634
+ // Block scale the accumulators with reg tensor `tCrScaleAViewAsC`
635
+ accumulation.scale_if_needed(tCrScaleAViewAsC);
636
+
637
+ ++smem_pipe_read;
638
+ }
639
+
640
+ warpgroup_fence_operand(accumulation());
641
+ // Mainloop GMMAs
642
+ k_tile_count -= prologue_mma_count;
643
+
644
+ CUTLASS_PRAGMA_NO_UNROLL
645
+ for ( ; k_tile_count > 0; --k_tile_count)
646
+ {
647
+ // WAIT on smem_pipe_read until its data are available (phase bit flips from rdPhaseBit value)
648
+ auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
649
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
650
+
651
+ //
652
+ // Compute on k_tile
653
+ //
654
+
655
+ int read_stage = smem_pipe_read.index();
656
+
657
+ // Load per block scale values from shared memory to registers (at most twice per block along M and exactly once per block along N)
658
+ scale_b = sScaleB[read_stage];
659
+ CUTLASS_PRAGMA_UNROLL
660
+ for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
661
+ tCrScaleAViewAsC.data()[i] = tCsScaleAViewAsC(_, _, _, read_stage)(idx2crd(i, RegLayoutScaleAEssential{}));
662
+ }
663
+ if constexpr (ScaleMsPerTile == 1) {
664
+ static_assert(size(RegLayoutScaleAEssential{}) == 1);
665
+ tCrScaleAViewAsC.data()[0] = __shfl_sync(0xffffffff, tCrScaleAViewAsC.data()[0] * scale_b, 0); // `tCrScaleAViewAsC.data()[0]` are all same in a warp group when `ScaleMsPerTile == 1`.
666
+ } else {
667
+ CUTLASS_PRAGMA_UNROLL
668
+ for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
669
+ tCrScaleAViewAsC.data()[i] = tCrScaleAViewAsC.data()[i] * scale_b;
670
+ }
671
+ }
672
+
673
+ if (accumulation.prepare_if_needed()) {
674
+ tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
675
+ }
676
+
677
+ warpgroup_fence_operand(accumulation());
678
+ warpgroup_arrive();
679
+ // Unroll the K mode manually to set scale D to 1
680
+ CUTLASS_PRAGMA_UNROLL
681
+ for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) {
682
+ // (V,M,K) x (V,N,K) => (V,M,N)
683
+ cute::gemm(tiled_mma, tCrA(_,_,k_block,read_stage), tCrB(_,_,k_block,read_stage), accumulation());
684
+ tiled_mma.accumulate_ = GMMA::ScaleOut::One;
685
+ }
686
+ warpgroup_commit_batch();
687
+
688
+ /// Wait on the GMMA barrier for K_PIPE_MMAS (or fewer) outstanding to ensure smem_pipe_write is consumed
689
+ warpgroup_wait<K_PIPE_MMAS>();
690
+ warpgroup_fence_operand(accumulation());
691
+
692
+ // Block scale the accumulators with reg tensor `tCrScaleAViewAsC`
693
+ accumulation.scale_if_needed(tCrScaleAViewAsC);
694
+
695
+ pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it
696
+
697
+ // Advance smem_pipe_read and smem_pipe_release
698
+ ++smem_pipe_read;
699
+ ++smem_pipe_release;
700
+ }
701
+
702
+ accumulation.scale_residue_if_needed(tCrScaleAViewAsC);
703
+
704
+ warpgroup_fence_operand(accumulation());
705
+ }
706
+
707
+ /// Perform a Consumer Epilogue to release all buffers
708
+ CUTLASS_DEVICE void
709
+ mma_tail(MainloopPipeline pipeline, PipelineState smem_pipe_release, int k_tile_count) {
710
+ // Prologue GMMAs
711
+ int prologue_mma_count = min(K_PIPE_MMAS, k_tile_count);
712
+ k_tile_count -= prologue_mma_count;
713
+
714
+ smem_pipe_release.advance(k_tile_count);
715
+
716
+ // Wait on all GMMAs to complete
717
+ warpgroup_wait<0>();
718
+
719
+ for (int count = 0; count < prologue_mma_count; ++count) {
720
+ pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it
721
+ ++smem_pipe_release;
722
+ }
723
+ }
724
+};
725
+
726
+/////////////////////////////////////////////////////////////////////////////////////////////////
727
+
728
+} // namespace cutlass::gemm::collective
729
+
730
+/////////////////////////////////////////////////////////////////////////////////////////////////
csrc/cutlass_extensions/gemm/dispatch_policy.hpp
new
+39
@@ -0,0 +1,39 @@
1
+#pragma once
2
+
3
+#include "cutlass/gemm/dispatch_policy.hpp"
4
+
5
+namespace cutlass::gemm {
6
+
7
+//////////////////////////////////////////////////////////////////////////////
8
+
9
+// FP8 related policies (including Blocked Scaled Accumulation)
10
+// `ScaleGranularityM` specifies scaling granularity along M, while zero-value
11
+// `ScaleGranularityM` indicates that scaling granularity is
12
+// `size<0>(TileShape_MNK{})` along M.
13
+template <int ScaleGranularityM = 0>
14
+struct KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum
15
+ : KernelTmaWarpSpecializedCooperative {};
16
+
17
+// n-buffer in smem (Hopper TMA), pipelined with Hopper GMMA and TMA, Warp
18
+// specialized dynamic schedule For FP8 kernels with Block Scaling
19
+template <int Stages_, class ClusterShape_ = Shape<_1, _1, _1>,
20
+ class KernelSchedule = KernelTmaWarpSpecialized,
21
+ int ScaleGranularityM =
22
+ 0 // `ScaleGranularityM` specifies scaling granularity along M,
23
+ // while zero-value `ScaleGranularityM` indicates that scaling
24
+ // granularity is `size<0>(TileShape_MNK{})` along M.
25
+ >
26
+struct MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8
27
+ : MainloopSm90TmaGmmaWarpSpecialized<Stages_, ClusterShape_,
28
+ KernelSchedule> {
29
+ static_assert(
30
+ cute::is_same_v<
31
+ KernelSchedule,
32
+ KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<
33
+ ScaleGranularityM>>,
34
+ "KernelSchedule must be one of the warp specialized policies");
35
+};
36
+
37
+//////////////////////////////////////////////////////////////////////////////
38
+
39
+} // namespace cutlass::gemm
\ No newline at end of file
csrc/cutlass_extensions/vllm_collective_builder.cuh
+1
-1
@@ -1,6 +1,6 @@
1
#pragma once
2
3
-#include "cutlass/gemm/collective/collective_builder.hpp"
3
+#include "cutlass_extensions/gemm/collective/collective_builder.hpp"
4
5
namespace cutlass::gemm::collective {
6
using namespace cute;
csrc/quantization/cutlass_w8a8/c3x/cutlass_gemm_caller.cuh
new
+93
@@ -0,0 +1,93 @@
1
+#pragma once
2
+
3
+// clang-format will break include orders
4
+// clang-format off
5
+#include <torch/all.h>
6
+
7
+#include <ATen/cuda/CUDAContext.h>
8
+
9
+#include "cutlass/cutlass.h"
10
+
11
+#include "cute/tensor.hpp"
12
+#include "cute/atom/mma_atom.hpp"
13
+#include "cutlass/numeric_types.h"
14
+
15
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
16
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
17
+#include "cutlass/epilogue/collective/collective_builder.hpp"
18
+#include "cutlass/gemm/collective/collective_builder.hpp"
19
+
20
+#include "core/math.hpp"
21
+#include "cutlass_extensions/common.hpp"
22
+// clang-format on
23
+
24
+namespace vllm::c3x {
25
+
26
+static inline cute::Shape<int, int, int, int> get_problem_shape(
27
+ torch::Tensor const& a, torch::Tensor const& b) {
28
+ int32_t m = a.size(0), n = b.size(1), k = a.size(1);
29
+ return {m, n, k, 1};
30
+}
31
+
32
+template <typename GemmKernel>
33
+void cutlass_gemm_caller(torch::Device device,
34
+ cute::Shape<int, int, int, int> prob_shape,
35
+ typename GemmKernel::MainloopArguments mainloop_args,
36
+ typename GemmKernel::EpilogueArguments epilogue_args) {
37
+ typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
38
+ prob_shape, mainloop_args, epilogue_args};
39
+
40
+ // Launch the CUTLASS GEMM kernel.
41
+ using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
42
+ GemmOp gemm_op;
43
+ CUTLASS_CHECK(gemm_op.can_implement(args));
44
+
45
+ size_t workspace_size = gemm_op.get_workspace_size(args);
46
+ auto const workspace_options =
47
+ torch::TensorOptions().dtype(torch::kUInt8).device(device);
48
+ auto workspace = torch::empty(workspace_size, workspace_options);
49
+
50
+ auto stream = at::cuda::getCurrentCUDAStream(device.index());
51
+
52
+ cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
53
+ CUTLASS_CHECK(status);
54
+}
55
+
56
+template <typename Gemm, typename... EpilogueArgs>
57
+void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
58
+ torch::Tensor const& b,
59
+ EpilogueArgs&&... epilogue_params) {
60
+ using ElementAB = typename Gemm::ElementAB;
61
+ using ElementD = typename Gemm::ElementD;
62
+ using GemmKernel = typename Gemm::GemmKernel;
63
+
64
+ int64_t lda = a.stride(0);
65
+ int64_t ldb = b.stride(1);
66
+ int64_t ldc = out.stride(0);
67
+
68
+ using StrideA = cute::Stride<int64_t, cute::Int<1>, int64_t>;
69
+ using StrideB = cute::Stride<int64_t, cute::Int<1>, int64_t>;
70
+ using StrideC = typename Gemm::StrideC;
71
+
72
+ StrideA a_stride{lda, cute::Int<1>{}, 0};
73
+ StrideB b_stride{ldb, cute::Int<1>{}, 0};
74
+ StrideC c_stride{ldc, cute::Int<1>{}, cute::Int<0>{}};
75
+
76
+ typename GemmKernel::ProblemShape prob_shape = get_problem_shape(a, b);
77
+
78
+ auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
79
+ auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
80
+ typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
81
+ b_stride};
82
+
83
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
84
+ typename GemmKernel::EpilogueArguments epilogue_args{
85
+ Gemm::Epilogue::prepare_args(
86
+ std::forward<EpilogueArgs>(epilogue_params)...),
87
+ c_ptr, c_stride, c_ptr, c_stride};
88
+
89
+ cutlass_gemm_caller<GemmKernel>(a.device(), prob_shape, mainloop_args,
90
+ epilogue_args);
91
+}
92
+
93
+} // namespace vllm::c3x
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh
renamed
-74
@@ -2,9 +2,6 @@
2
3
// clang-format will break include orders
4
// clang-format off
5
-#include <torch/all.h>
6
-
7
-#include <ATen/cuda/CUDAContext.h>
5
6
#include "cutlass/cutlass.h"
7
@@ -32,21 +29,6 @@ using namespace cute;
29
30
namespace vllm {
31
35
-// A wrapper for the GEMM kernel that is used to guard against compilation on
36
-// architectures that will never use the kernel. The purpose of this is to
37
-// reduce the size of the compiled binary.
38
-// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
39
-// into code that will be executed on the device where it is defined.
40
-template <typename Kernel>
41
-struct enable_sm90_or_later : Kernel {
42
- template <typename... Args>
43
- CUTLASS_DEVICE void operator()(Args&&... args) {
44
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
45
- Kernel::operator()(std::forward<Args>(args)...);
46
-#endif
47
- }
48
-};
49
-
32
template <typename ElementAB_, typename ElementD_,
33
template <typename, typename, typename> typename Epilogue_,
34
typename TileShape, typename ClusterShape, typename KernelSchedule,
@@ -101,60 +83,4 @@ struct cutlass_3x_gemm {
83
struct GemmKernel : public KernelType {};
84
};
85
104
-template <typename Gemm, typename... EpilogueArgs>
105
-void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
106
- torch::Tensor const& b,
107
- EpilogueArgs&&... epilogue_params) {
108
- using ElementAB = typename Gemm::ElementAB;
109
- using ElementD = typename Gemm::ElementD;
110
-
111
- int32_t m = a.size(0);
112
- int32_t n = b.size(1);
113
- int32_t k = a.size(1);
114
-
115
- int64_t lda = a.stride(0);
116
- int64_t ldb = b.stride(1);
117
- int64_t ldc = out.stride(0);
118
-
119
- using StrideA = Stride<int64_t, Int<1>, int64_t>;
120
- using StrideB = Stride<int64_t, Int<1>, int64_t>;
121
- using StrideC = typename Gemm::StrideC;
122
-
123
- StrideA a_stride{lda, Int<1>{}, 0};
124
- StrideB b_stride{ldb, Int<1>{}, 0};
125
- StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
126
-
127
- using GemmKernel = typename Gemm::GemmKernel;
128
- typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
129
-
130
- auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
131
- auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
132
- typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
133
- b_stride};
134
-
135
- auto c_ptr = static_cast<ElementD*>(out.data_ptr());
136
- typename GemmKernel::EpilogueArguments epilogue_args{
137
- Gemm::Epilogue::prepare_args(
138
- std::forward<EpilogueArgs>(epilogue_params)...),
139
- c_ptr, c_stride, c_ptr, c_stride};
140
-
141
- typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
142
- prob_shape, mainloop_args, epilogue_args};
143
-
144
- // Launch the CUTLASS GEMM kernel.
145
- using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
146
- GemmOp gemm_op;
147
- CUTLASS_CHECK(gemm_op.can_implement(args));
148
-
149
- size_t workspace_size = gemm_op.get_workspace_size(args);
150
- auto const workspace_options =
151
- torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
152
- auto workspace = torch::empty(workspace_size, workspace_options);
153
-
154
- auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
155
-
156
- cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
157
- CUTLASS_CHECK(status);
158
-}
159
-
86
} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu
new
+24
@@ -0,0 +1,24 @@
1
+#include "scaled_mm_kernels.hpp"
2
+#include "scaled_mm_sm90_int8_dispatch.cuh"
3
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
4
+
5
+namespace vllm {
6
+
7
+void cutlass_scaled_mm_azp_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
8
+ torch::Tensor const& b,
9
+ torch::Tensor const& a_scales,
10
+ torch::Tensor const& b_scales,
11
+ torch::Tensor const& azp_adj,
12
+ std::optional<torch::Tensor> const& azp,
13
+ std::optional<torch::Tensor> const& bias) {
14
+ if (azp) {
15
+ return cutlass_scaled_mm_sm90_int8_epilogue<
16
+ c3x::ScaledEpilogueBiasAzpToken>(out, a, b, a_scales, b_scales, azp_adj,
17
+ *azp, bias);
18
+ } else {
19
+ return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogueBiasAzp>(
20
+ out, a, b, a_scales, b_scales, azp_adj, bias);
21
+ }
22
+}
23
+
24
+} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8.cu
new
+24
@@ -0,0 +1,24 @@
1
+
2
+#include "scaled_mm_kernels.hpp"
3
+#include "scaled_mm_blockwise_sm90_fp8_dispatch.cuh"
4
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
5
+
6
+namespace vllm {
7
+
8
+void cutlass_scaled_mm_blockwise_sm90_fp8(torch::Tensor& out,
9
+ torch::Tensor const& a,
10
+ torch::Tensor const& b,
11
+ torch::Tensor const& a_scales,
12
+ torch::Tensor const& b_scales) {
13
+ if (out.dtype() == torch::kBFloat16) {
14
+ cutlass_gemm_blockwise_sm90_fp8_dispatch<cutlass::bfloat16_t>(
15
+ out, a, b, a_scales, b_scales);
16
+
17
+ } else {
18
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
19
+ cutlass_gemm_blockwise_sm90_fp8_dispatch<cutlass::half_t>(
20
+ out, a, b, a_scales, b_scales);
21
+ }
22
+}
23
+
24
+} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh
new
+168
@@ -0,0 +1,168 @@
1
+#pragma once
2
+
3
+#include "cutlass/cutlass.h"
4
+#include "cutlass/numeric_types.h"
5
+
6
+#include "cute/tensor.hpp"
7
+#include "cutlass/tensor_ref.h"
8
+#include "cutlass/gemm/dispatch_policy.hpp"
9
+#include "cutlass/gemm/collective/collective_builder.hpp"
10
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
11
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
12
+#include "cutlass/gemm/kernel/tile_scheduler_params.h"
13
+#include "cutlass/epilogue/dispatch_policy.hpp"
14
+#include "cutlass/epilogue/collective/collective_builder.hpp"
15
+
16
+#include "cutlass_extensions/gemm/dispatch_policy.hpp"
17
+#include "cutlass_extensions/gemm/collective/collective_builder.hpp"
18
+
19
+#include "cutlass_gemm_caller.cuh"
20
+
21
+namespace vllm {
22
+
23
+using namespace cute;
24
+
25
+template <typename OutType, int GroupSizeM_, int GroupSizeN_, int GroupSizeK_,
26
+ int TileSizeM_ = 128, class ClusterShape = Shape<_1, _2, _1>>
27
+struct cutlass_3x_gemm_fp8_blockwise {
28
+ using GroupSizeM = Int<GroupSizeM_>;
29
+ using GroupSizeN = Int<GroupSizeN_>;
30
+ using GroupSizeK = Int<GroupSizeK_>;
31
+ using TileSizeM = Int<TileSizeM_>;
32
+
33
+ static_assert(TileSizeM_ % GroupSizeM_ == 0,
34
+ "TileSizeM must be a multiple of GroupSizeM");
35
+
36
+ using ElementAB = cutlass::float_e4m3_t;
37
+
38
+ using ElementA = ElementAB;
39
+ using LayoutA = cutlass::layout::RowMajor;
40
+ static constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
41
+
42
+ using ElementB = ElementAB;
43
+ using LayoutB = cutlass::layout::ColumnMajor;
44
+ static constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
45
+
46
+ using ElementD = OutType;
47
+ using StrideD = Stride<int64_t, Int<1>, Int<0>>;
48
+ static constexpr int AlignmentD = 128 / cutlass::sizeof_bits<ElementD>::value;
49
+
50
+ using ElementC = void;
51
+ using StrideC = StrideD;
52
+ static constexpr int AlignmentC = AlignmentD;
53
+
54
+ using ElementAccumulator = float;
55
+ using ElementBlockScale = float;
56
+ using ElementCompute = float;
57
+ using ArchTag = cutlass::arch::Sm90;
58
+ using OperatorClass = cutlass::arch::OpClassTensorOp;
59
+ using TileShape = Shape<TileSizeM, GroupSizeN, GroupSizeK>;
60
+
61
+ using KernelSchedule = cutlass::gemm::
62
+ KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<
63
+ GroupSizeM_>;
64
+ using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
65
+ using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
66
+
67
+ using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT<
68
+ cutlass::epilogue::fusion::Sm90AccFetch>;
69
+
70
+ using CollectiveEpilogue =
71
+ typename cutlass::epilogue::collective::CollectiveBuilder<
72
+ ArchTag, OperatorClass, TileShape, ClusterShape, EpilogueTileType,
73
+ ElementAccumulator, ElementCompute, ElementC, StrideC, AlignmentC,
74
+ ElementD, StrideD, AlignmentD, EpilogueSchedule,
75
+ StoreEpilogueCompute>::CollectiveOp;
76
+
77
+ using CollectiveMainloop =
78
+ typename cutlass::gemm::collective::CollectiveBuilder<
79
+ ArchTag, OperatorClass, ElementA, LayoutA, AlignmentA, ElementB,
80
+ LayoutB, AlignmentB, ElementAccumulator, TileShape, ClusterShape,
81
+ cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
82
+ sizeof(typename CollectiveEpilogue::SharedStorage))>,
83
+ KernelSchedule>::CollectiveOp;
84
+
85
+ using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
86
+ Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
87
+ cutlass::gemm::PersistentScheduler>>;
88
+
89
+ struct GemmKernel : public KernelType {};
90
+
91
+ using StrideA = typename GemmKernel::StrideA;
92
+ using StrideB = typename GemmKernel::StrideB;
93
+};
94
+
95
+template <typename Gemm>
96
+void cutlass_gemm_caller_blockwise(torch::Tensor& out, torch::Tensor const& a,
97
+ torch::Tensor const& b,
98
+ torch::Tensor const& a_scales,
99
+ torch::Tensor const& b_scales) {
100
+ using GemmKernel = typename Gemm::GemmKernel;
101
+
102
+ using ElementAB = typename Gemm::ElementAB;
103
+ using ElementD = typename Gemm::ElementD;
104
+
105
+ auto prob_shape = c3x::get_problem_shape(a, b);
106
+ int32_t m = get<0>(prob_shape), n = get<1>(prob_shape),
107
+ k = get<2>(prob_shape);
108
+
109
+ int64_t lda = a.stride(0);
110
+ int64_t ldb = b.stride(1);
111
+ int64_t ldc = out.stride(0);
112
+
113
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
114
+ using StrideB = Stride<int64_t, Int<1>, int64_t>;
115
+ using StrideC = typename Gemm::StrideC;
116
+
117
+ StrideA a_stride{lda, Int<1>{}, 0};
118
+ StrideB b_stride{ldb, Int<1>{}, 0};
119
+ StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
120
+
121
+ auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
122
+ auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
123
+ auto a_scales_ptr = static_cast<float*>(a_scales.data_ptr());
124
+ auto b_scales_ptr = static_cast<float*>(b_scales.data_ptr());
125
+
126
+ // Check is the t is contiguous and is 1D or 2D with one of the dimensions
127
+ // being 1 (i.e. a row or column vector)
128
+ auto is_contiguous_vector = [](const torch::Tensor& t) {
129
+ auto t_sizes = t.sizes();
130
+ return t.is_contiguous() &&
131
+ (t.dim() == 1 ||
132
+ (t.dim() == 2 &&
133
+ *std::min_element(t_sizes.begin(), t_sizes.end()) == 1));
134
+ };
135
+
136
+ // TODO(lucas): lets clean-up the kernel so that we pass in Strides so
137
+ // we don't have to deal with enforcing implicit layouts
138
+ TORCH_CHECK(a_scales.size(0) == m / Gemm::GroupSizeM::value);
139
+ TORCH_CHECK(a_scales.size(1) == k / Gemm::GroupSizeK::value);
140
+ TORCH_CHECK(a_scales.stride(0) == 1 || is_contiguous_vector(a_scales),
141
+ "a_scales must be M major");
142
+ TORCH_CHECK(b_scales.size(0) == k / Gemm::GroupSizeK::value);
143
+ TORCH_CHECK(b_scales.size(1) == n / Gemm::GroupSizeN::value);
144
+ TORCH_CHECK(b_scales.stride(0) == 1 || is_contiguous_vector(b_scales),
145
+ "b_scales must be K major");
146
+ typename GemmKernel::MainloopArguments mainloop_args{
147
+ a_ptr, a_stride, b_ptr, b_stride, a_scales_ptr, b_scales_ptr};
148
+
149
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
150
+ typename GemmKernel::EpilogueArguments epilogue_args{
151
+ {}, c_ptr, c_stride, c_ptr, c_stride};
152
+
153
+ c3x::cutlass_gemm_caller<GemmKernel>(a.device(), prob_shape, mainloop_args,
154
+ epilogue_args);
155
+}
156
+
157
+template <typename OutType>
158
+void cutlass_gemm_blockwise_sm90_fp8_dispatch(torch::Tensor& out,
159
+ torch::Tensor const& a,
160
+ torch::Tensor const& b,
161
+ torch::Tensor const& a_scales,
162
+ torch::Tensor const& b_scales) {
163
+ cutlass_gemm_caller_blockwise<
164
+ cutlass_3x_gemm_fp8_blockwise<OutType, 1, 128, 128>>(out, a, b, a_scales,
165
+ b_scales);
166
+}
167
+
168
+} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_kernels.hpp
new
+33
@@ -0,0 +1,33 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+
5
+namespace vllm {
6
+
7
+void cutlass_scaled_mm_sm90_fp8(torch::Tensor& out, torch::Tensor const& a,
8
+ torch::Tensor const& b,
9
+ torch::Tensor const& a_scales,
10
+ torch::Tensor const& b_scales,
11
+ std::optional<torch::Tensor> const& bias);
12
+
13
+void cutlass_scaled_mm_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
14
+ torch::Tensor const& b,
15
+ torch::Tensor const& a_scales,
16
+ torch::Tensor const& b_scales,
17
+ std::optional<torch::Tensor> const& bias);
18
+
19
+void cutlass_scaled_mm_azp_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
20
+ torch::Tensor const& b,
21
+ torch::Tensor const& a_scales,
22
+ torch::Tensor const& b_scales,
23
+ torch::Tensor const& azp_adj,
24
+ std::optional<torch::Tensor> const& azp,
25
+ std::optional<torch::Tensor> const& bias);
26
+
27
+void cutlass_scaled_mm_blockwise_sm90_fp8(torch::Tensor& out,
28
+ torch::Tensor const& a,
29
+ torch::Tensor const& b,
30
+ torch::Tensor const& a_scales,
31
+ torch::Tensor const& b_scales);
32
+
33
+} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu
new
+24
@@ -0,0 +1,24 @@
1
+#include "scaled_mm_kernels.hpp"
2
+#include "scaled_mm_sm90_fp8_dispatch.cuh"
3
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
4
+
5
+namespace vllm {
6
+
7
+void cutlass_scaled_mm_sm90_fp8(torch::Tensor& out, torch::Tensor const& a,
8
+ torch::Tensor const& b,
9
+ torch::Tensor const& a_scales,
10
+ torch::Tensor const& b_scales,
11
+ std::optional<torch::Tensor> const& bias) {
12
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
13
+ if (bias) {
14
+ TORCH_CHECK(bias->dtype() == out.dtype(),
15
+ "currently bias dtype must match output dtype ", out.dtype());
16
+ return cutlass_scaled_mm_sm90_fp8_epilogue<c3x::ScaledEpilogueBias>(
17
+ out, a, b, a_scales, b_scales, *bias);
18
+ } else {
19
+ return cutlass_scaled_mm_sm90_fp8_epilogue<c3x::ScaledEpilogue>(
20
+ out, a, b, a_scales, b_scales);
21
+ }
22
+}
23
+
24
+} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8_dispatch.cuh
renamed
+25
-1
@@ -1,6 +1,7 @@
1
#pragma once
2
3
-#include "scaled_mm_c3x.cuh"
3
+#include "scaled_mm.cuh"
4
+#include "cutlass_gemm_caller.cuh"
5
6
/**
7
* This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm
@@ -9,6 +10,8 @@
10
11
namespace vllm {
12
13
+using c3x::cutlass_gemm_caller;
14
+
15
template <typename InType, typename OutType,
16
template <typename, typename, typename> typename Epilogue>
17
struct sm90_fp8_config_default {
@@ -93,4 +96,25 @@ inline void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out,
96
}
97
}
98
99
+template <template <typename, typename, typename> typename Epilogue,
100
+ typename... EpilogueArgs>
101
+void cutlass_scaled_mm_sm90_fp8_epilogue(torch::Tensor& out,
102
+ torch::Tensor const& a,
103
+ torch::Tensor const& b,
104
+ EpilogueArgs&&... epilogue_args) {
105
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
106
+ TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
107
+
108
+ if (out.dtype() == torch::kBFloat16) {
109
+ return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
110
+ cutlass::bfloat16_t, Epilogue>(
111
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
112
+ } else {
113
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
114
+ return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
115
+ cutlass::half_t, Epilogue>(
116
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
117
+ }
118
+}
119
+
120
} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu
new
+24
@@ -0,0 +1,24 @@
1
+#include "scaled_mm_kernels.hpp"
2
+#include "scaled_mm_sm90_int8_dispatch.cuh"
3
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
4
+
5
+namespace vllm {
6
+
7
+void cutlass_scaled_mm_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
8
+ torch::Tensor const& b,
9
+ torch::Tensor const& a_scales,
10
+ torch::Tensor const& b_scales,
11
+ std::optional<torch::Tensor> const& bias) {
12
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
13
+ if (bias) {
14
+ TORCH_CHECK(bias->dtype() == out.dtype(),
15
+ "currently bias dtype must match output dtype ", out.dtype());
16
+ return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogueBias>(
17
+ out, a, b, a_scales, b_scales, *bias);
18
+ } else {
19
+ return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogue>(
20
+ out, a, b, a_scales, b_scales);
21
+ }
22
+}
23
+
24
+} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8_dispatch.cuh
renamed
+24
-1
@@ -1,6 +1,7 @@
1
#pragma once
2
3
-#include "scaled_mm_c3x.cuh"
3
+#include "scaled_mm.cuh"
4
+#include "cutlass_gemm_caller.cuh"
5
6
/**
7
* This file defines Gemm kernel configurations for SM90 (int8) based on the
@@ -9,6 +10,8 @@
10
11
namespace vllm {
12
13
+using c3x::cutlass_gemm_caller;
14
+
15
template <typename InType, typename OutType,
16
template <typename, typename, typename> typename Epilogue>
17
struct sm90_int8_config_default {
@@ -137,4 +140,24 @@ inline void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out,
140
}
141
}
142
143
+template <template <typename, typename, typename> typename Epilogue,
144
+ typename... EpilogueArgs>
145
+void cutlass_scaled_mm_sm90_int8_epilogue(torch::Tensor& out,
146
+ torch::Tensor const& a,
147
+ torch::Tensor const& b,
148
+ EpilogueArgs&&... epilogue_args) {
149
+ TORCH_CHECK(a.dtype() == torch::kInt8);
150
+ TORCH_CHECK(b.dtype() == torch::kInt8);
151
+
152
+ if (out.dtype() == torch::kBFloat16) {
153
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
154
+ Epilogue>(
155
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
156
+ } else {
157
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
158
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
159
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
160
+ }
161
+}
162
+
163
} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+47
-57
@@ -1,52 +1,13 @@
1
#include <cudaTypedefs.h>
2
+#include "c3x/scaled_mm_kernels.hpp"
3
3
-#if defined CUDA_VERSION && CUDA_VERSION >= 12000
4
-
5
- #include "scaled_mm_c3x_sm90_fp8_dispatch.cuh"
6
- #include "scaled_mm_c3x_sm90_int8_dispatch.cuh"
7
-
8
- #include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
9
-using namespace vllm;
4
+#include "core/math.hpp"
5
6
/*
7
This file defines quantized GEMM operations using the CUTLASS 3.x API, for
8
NVIDIA GPUs with sm90a (Hopper) or later.
9
*/
10
16
-template <template <typename, typename, typename> typename Epilogue,
17
- typename... EpilogueArgs>
18
-void cutlass_scaled_mm_sm90_epilogue(torch::Tensor& out, torch::Tensor const& a,
19
- torch::Tensor const& b,
20
- EpilogueArgs&&... epilogue_args) {
21
- if (a.dtype() == torch::kInt8) {
22
- TORCH_CHECK(b.dtype() == torch::kInt8);
23
-
24
- if (out.dtype() == torch::kBFloat16) {
25
- return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
26
- Epilogue>(
27
- out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
28
- } else {
29
- TORCH_CHECK(out.dtype() == torch::kFloat16);
30
- return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
31
- out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
32
- }
33
- } else {
34
- TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
35
- TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
36
-
37
- if (out.dtype() == torch::kBFloat16) {
38
- return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
39
- cutlass::bfloat16_t, Epilogue>(
40
- out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
41
- } else {
42
- TORCH_CHECK(out.dtype() == torch::kFloat16);
43
- return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
44
- cutlass::half_t, Epilogue>(
45
- out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
46
- }
47
- }
48
-}
49
-
11
void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
12
torch::Tensor const& b,
13
torch::Tensor const& a_scales,
@@ -54,14 +15,50 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
15
std::optional<torch::Tensor> const& bias) {
16
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
17
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
57
- if (bias) {
58
- TORCH_CHECK(bias->dtype() == c.dtype(),
59
- "currently bias dtype must match output dtype ", c.dtype());
60
- return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
61
- c, a, b, a_scales, b_scales, *bias);
18
+
19
+ using GroupShape = std::array<int64_t, 2>;
20
+
21
+ int M = a.size(0), N = b.size(1), K = a.size(1);
22
+
23
+ GroupShape a_scale_group_shape = [&, &s = a_scales]() -> GroupShape {
24
+ if (s.numel() == 1) return {M, K}; // tensor-wise
25
+ if (s.dim() == 2)
26
+ return {ceil_div(a.size(0), s.size(0)), ceil_div(a.size(1), s.size(1))};
27
+ TORCH_CHECK(false, "Unsupported scale shape for scale_a");
28
+ }();
29
+
30
+ GroupShape b_scale_group_shape = [&, &s = b_scales]() -> GroupShape {
31
+ if (s.numel() == 1) return {K, N}; // tensor-wise
32
+ if (s.dim() == 2)
33
+ return {ceil_div(b.size(0), s.size(0)), ceil_div(b.size(1), s.size(1))};
34
+ TORCH_CHECK(false, "Unsupported scale shape for scale_b");
35
+ }();
36
+
37
+ if ((a_scale_group_shape == GroupShape{M, K} ||
38
+ a_scale_group_shape == GroupShape{1, K}) &&
39
+ (b_scale_group_shape == GroupShape{K, N} ||
40
+ b_scale_group_shape == GroupShape{K, 1})) {
41
+ // "standard per-tensor/per-token/per-channel" scaling
42
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
43
+ if (a.dtype() == torch::kFloat8_e4m3fn) {
44
+ vllm::cutlass_scaled_mm_sm90_fp8(c, a, b, a_scales, b_scales, bias);
45
+ } else {
46
+ TORCH_CHECK(a.dtype() == torch::kInt8);
47
+ vllm::cutlass_scaled_mm_sm90_int8(c, a, b, a_scales, b_scales, bias);
48
+ }
49
+ } else if (a_scale_group_shape == GroupShape{1, 128} &&
50
+ b_scale_group_shape == GroupShape{128, 128}) {
51
+ // 1x128 per-token group scales for activations
52
+ // 128x128 blockwise scales for weights
53
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn &&
54
+ b.dtype() == torch::kFloat8_e4m3fn,
55
+ "Currently only FP8 is supported for A group shape 1x128 and "
56
+ "B group shape 128x128");
57
+ TORCH_CHECK(!bias, "Bias not yet supported blockwise scaled_mm");
58
+
59
+ vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
60
} else {
63
- return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogue>(
64
- c, a, b, a_scales, b_scales);
61
+ TORCH_CHECK(false, "Unsupported scale group shapes for CUTLASS 3.x GEMM");
62
}
63
}
64
@@ -75,13 +72,6 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
72
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
73
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
74
78
- if (azp) {
79
- return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzpToken>(
80
- out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
81
- } else {
82
- return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzp>(
83
- out, a, b, a_scales, b_scales, azp_adj, bias);
84
- }
75
+ vllm::cutlass_scaled_mm_azp_sm90_int8(out, a, b, a_scales, b_scales, azp_adj,
76
+ azp, bias);
77
}
86
-
87
-#endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
-3
@@ -89,15 +89,12 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
89
TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
90
TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
91
b.size(1) == c.size(1));
92
- TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
93
- TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
92
93
// Check for strides and alignment
94
TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
95
TORCH_CHECK(b.stride(0) == 1); // Column-major
96
TORCH_CHECK(c.stride(0) % 16 == 0 &&
97
b.stride(1) % 16 == 0); // 16 Byte Alignment
100
- TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
98
99
if (bias) {
100
TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous() &&
csrc/quantization/machete/machete_mainloop.cuh
+4
@@ -272,6 +272,10 @@ struct MacheteCollectiveMma {
272
using PipelineState = cutlass::PipelineState<DispatchPolicy::Stages>;
273
274
using PipelineParams = typename MainloopPipeline::Params;
275
+
276
+ // One threads per CTA are producers (1 for operand tile)
277
+ static constexpr int NumProducerThreadEvents = 1;
278
+
279
using ScaleTileShape = decltype(make_shape(shape<0>(TileShape{}),
280
shape<1>(SmemLayoutAtomScale{})));
281
tests/kernels/test_cutlass.py
+132
-56
@@ -10,6 +10,7 @@ import torch
10
from tests.kernels.utils import opcheck
11
from vllm import _custom_ops as ops
12
from vllm.platforms import current_platform
13
+from vllm.utils import cdiv
14
15
from .utils import baseline_scaled_mm, to_fp8, to_int8
16
@@ -39,6 +40,11 @@ CUDA_DEVICES = [
40
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
41
]
42
43
+# -1 means full extent in that dimension
44
+TENSORWISE_GROUP_SHAPE = (-1, -1)
45
+PER_TOKEN_GROUP_SHAPE = (1, -1)
46
+PER_OUT_CH_GROUP_SHAPE = (-1, 1)
47
+
48
capability = current_platform.get_device_capability()
49
capability = capability[0] * 10 + capability[1]
50
@@ -47,11 +53,22 @@ def rand_int8(shape: tuple, device: str = "cuda"):
53
return to_int8(torch.rand(shape, device=device) * 255 - 128)
54
55
56
+def group_scale_helper(shape, group_shape):
57
+ return [shape[i] if s < 0 else s for i, s in enumerate(group_shape)]
58
+
59
+
60
+def scale_shape(shape, group_shape):
61
+ assert len(shape) == len(group_shape)
62
+ group_shape = group_scale_helper(shape, group_shape)
63
+ return tuple(
64
+ cdiv(shape[i], group_shape[i]) for i in range(len(group_shape)))
65
+
66
+
67
def cutlass_fp8_gemm_helper(m: int,
68
n: int,
69
k: int,
53
- per_token_act_quant: bool,
54
- per_out_channel_weight_quant: bool,
70
+ a_scale_group_shape: tuple,
71
+ b_scale_group_shape: tuple,
72
use_bias: bool,
73
out_dtype: Type[torch.dtype] = torch.bfloat16,
74
device: str = "cuda"):
@@ -60,13 +77,17 @@ def cutlass_fp8_gemm_helper(m: int,
77
a = to_fp8(torch.randn((m, k), device=device))
78
b = to_fp8(torch.randn((n, k), device=device).t())
79
63
- m_a_scales = m if per_token_act_quant else 1
64
- n_b_scales = n if per_out_channel_weight_quant else 1
80
+ a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
81
+ b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
82
+
83
+ scale_a = (torch.randn(a_scales_shape, device=device, dtype=torch.float32))
84
+ scale_b = (torch.randn(b_scales_shape, device=device, dtype=torch.float32))
85
+
86
+ # make scales M-major for blockwise quant, doesn't affect 1D scales
87
+ scale_a = scale_a.t().contiguous().t()
88
+ # make scales K-major for blockwise quant, doesn't affect 1D scales
89
+ scale_b = scale_b.t().contiguous().t()
90
66
- scale_a = (torch.randn((m_a_scales, 1), device=device,
67
- dtype=torch.float32))
68
- scale_b = (torch.randn((1, n_b_scales), device=device,
69
- dtype=torch.float32))
91
if use_bias:
92
bias = torch.rand((n, ), device=device, dtype=out_dtype) * 10
93
else:
@@ -84,8 +105,8 @@ def cutlass_fp8_gemm_helper(m: int,
105
def cutlass_int8_gemm_helper(m: int,
106
n: int,
107
k: int,
87
- per_token_act_quant: bool,
88
- per_out_channel_weight_quant: bool,
108
+ a_scale_group_shape: tuple,
109
+ b_scale_group_shape: tuple,
110
use_bias: bool,
111
out_dtype: Type[torch.dtype] = torch.bfloat16,
112
device: str = "cuda"):
@@ -94,13 +115,11 @@ def cutlass_int8_gemm_helper(m: int,
115
a = to_int8(torch.randn((m, k), device=device) * 5)
116
b = to_int8(torch.randn((n, k), device=device).t() * 5)
117
97
- m_a_scales = m if per_token_act_quant else 1
98
- n_b_scales = n if per_out_channel_weight_quant else 1
118
+ a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
119
+ b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
120
100
- scale_a = (torch.randn((m_a_scales, 1), device=device,
101
- dtype=torch.float32))
102
- scale_b = (torch.randn((1, n_b_scales), device=device,
103
- dtype=torch.float32))
121
+ scale_a = (torch.randn(a_scales_shape, device=device, dtype=torch.float32))
122
+ scale_b = (torch.randn(b_scales_shape, device=device, dtype=torch.float32))
123
124
if use_bias:
125
bias = torch.rand((n, ), device=device, dtype=out_dtype) * 10
@@ -117,82 +136,135 @@ def cutlass_int8_gemm_helper(m: int,
136
137
138
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
120
-@pytest.mark.parametrize("per_act_token", [True, False])
121
-@pytest.mark.parametrize("per_out_ch", [True, False])
139
+@pytest.mark.parametrize("a_scale_group_shape",
140
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
141
+@pytest.mark.parametrize("b_scale_group_shape",
142
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
143
@pytest.mark.parametrize("use_bias", [True, False])
144
@pytest.mark.skipif(not current_platform.has_device_capability(89),
145
reason="FP8 is not supported on this GPU type.")
125
-def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
126
- per_out_ch: bool, use_bias: bool):
127
- cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
146
+def test_cutlass_fp8_gemm(m: int, n: int, k: int, a_scale_group_shape,
147
+ b_scale_group_shape, use_bias: bool):
148
+ cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape,
149
+ use_bias)
150
151
152
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
131
-@pytest.mark.parametrize("per_act_token", [True, False])
132
-@pytest.mark.parametrize("per_out_ch", [True, False])
153
+@pytest.mark.parametrize("a_scale_group_shape,b_scale_group_shape",
154
+ [((1, 128), (128, 128))])
155
+@pytest.mark.parametrize("use_bias", [False])
156
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
157
+ reason="FP8 blockwise is not supported on this GPU type.")
158
+def test_cutlass_fp8_blockwise_scale_gemm(m: int, n: int, k: int,
159
+ a_scale_group_shape,
160
+ b_scale_group_shape, use_bias: bool):
161
+ if k % b_scale_group_shape[0] != 0 or n % b_scale_group_shape[1] != 0:
162
+ return
163
+ if m % a_scale_group_shape[0] != 0 or k % a_scale_group_shape[1] != 0:
164
+ return
165
+ cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape,
166
+ use_bias)
167
+
168
+
169
+@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
170
+@pytest.mark.parametrize("a_scale_group_shape",
171
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
172
+@pytest.mark.parametrize("b_scale_group_shape",
173
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
174
@pytest.mark.parametrize("use_bias", [True, False])
134
-def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
135
- per_out_ch: bool, use_bias: bool):
136
- cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
175
+def test_cutlass_int8_gemm(m: int, n: int, k: int, a_scale_group_shape,
176
+ b_scale_group_shape, use_bias: bool):
177
+ cutlass_int8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape,
178
+ use_bias)
179
180
139
-@pytest.mark.parametrize("per_act_token", [True, False])
140
-@pytest.mark.parametrize("per_out_ch", [True, False])
181
+@pytest.mark.parametrize("a_scale_group_shape",
182
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
183
+@pytest.mark.parametrize("b_scale_group_shape",
184
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
185
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
186
@pytest.mark.parametrize("use_bias", [True, False])
143
-def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
187
+def test_cutlass_int8_gemm_output_dtype(a_scale_group_shape,
188
+ b_scale_group_shape,
189
out_dtype: Type[torch.dtype],
190
use_bias: bool):
191
cutlass_int8_gemm_helper(512,
192
512,
193
512,
149
- per_act_token,
150
- per_out_ch,
194
+ a_scale_group_shape,
195
+ b_scale_group_shape,
196
use_bias,
197
out_dtype=out_dtype)
198
199
155
-@pytest.mark.parametrize("per_act_token", [True, False])
156
-@pytest.mark.parametrize("per_out_ch", [True, False])
200
+@pytest.mark.parametrize("a_scale_group_shape",
201
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
202
+@pytest.mark.parametrize("b_scale_group_shape",
203
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
204
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
205
@pytest.mark.parametrize("use_bias", [True, False])
206
@pytest.mark.skipif(not current_platform.has_device_capability(89),
207
reason="FP8 is not supported on this GPU type.")
161
-def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
208
+def test_cutlass_fp8_gemm_output_dtype(a_scale_group_shape,
209
+ b_scale_group_shape,
210
out_dtype: Type[torch.dtype],
211
use_bias: bool):
212
cutlass_fp8_gemm_helper(512,
213
512,
214
512,
167
- per_act_token,
168
- per_out_ch,
215
+ a_scale_group_shape,
216
+ b_scale_group_shape,
217
use_bias,
218
out_dtype=out_dtype)
219
220
173
-@pytest.mark.parametrize("per_act_token", [True, False])
174
-@pytest.mark.parametrize("per_out_ch", [True, False])
221
+@pytest.mark.parametrize("a_scale_group_shape,b_scale_group_shape",
222
+ [((1, 128), (128, 128))])
223
+@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
224
+@pytest.mark.parametrize("use_bias", [False])
225
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
226
+ reason="FP8 blockwise is not supported on this GPU type.")
227
+def test_cutlass_fp8_blockwise_scale_gemm_dtype(a_scale_group_shape,
228
+ b_scale_group_shape,
229
+ out_dtype: Type[torch.dtype],
230
+ use_bias: bool):
231
+ cutlass_fp8_gemm_helper(512,
232
+ 512,
233
+ 512,
234
+ a_scale_group_shape,
235
+ b_scale_group_shape,
236
+ use_bias,
237
+ out_dtype=out_dtype)
238
+
239
+
240
+@pytest.mark.parametrize("a_scale_group_shape",
241
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
242
+@pytest.mark.parametrize("b_scale_group_shape",
243
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
244
@pytest.mark.parametrize("use_bias", [True, False])
245
@pytest.mark.parametrize("device", CUDA_DEVICES)
246
@pytest.mark.skipif(not current_platform.has_device_capability(89),
247
reason="FP8 is not supported on this GPU type.")
179
-def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
248
+def test_cutlass_fp8_gemm_devices(a_scale_group_shape, b_scale_group_shape,
249
use_bias: bool, device: str):
181
- cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch, use_bias,
182
- torch.bfloat16, device)
250
+ cutlass_fp8_gemm_helper(512, 512, 512, a_scale_group_shape,
251
+ b_scale_group_shape, use_bias, torch.bfloat16,
252
+ device)
253
254
185
-@pytest.mark.parametrize("per_act_token", [True, False])
186
-@pytest.mark.parametrize("per_out_ch", [True, False])
255
+@pytest.mark.parametrize("a_scale_group_shape",
256
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
257
+@pytest.mark.parametrize("b_scale_group_shape",
258
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
259
@pytest.mark.parametrize("use_bias", [True, False])
260
@pytest.mark.parametrize("device", CUDA_DEVICES)
189
-def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
261
+def test_cutlass_int8_gemm_devices(a_scale_group_shape, b_scale_group_shape,
262
use_bias: bool, device: str):
263
cutlass_int8_gemm_helper(512,
264
512,
265
512,
194
- per_act_token,
195
- per_out_ch,
266
+ a_scale_group_shape,
267
+ b_scale_group_shape,
268
use_bias,
269
out_dtype=torch.bfloat16,
270
device=device)
@@ -203,28 +275,32 @@ def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
275
# of a large power of two. In any case, the kernel will have a naive fallback
276
# when N and K are not divisible by 16. But M is the number of tokens and the
277
# kernel must handle any M thrown at it.
206
-@pytest.mark.parametrize("per_act_token", [True, False])
207
-@pytest.mark.parametrize("per_out_ch", [True, False])
278
+@pytest.mark.parametrize("a_scale_group_shape",
279
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
280
+@pytest.mark.parametrize("b_scale_group_shape",
281
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
282
@pytest.mark.parametrize("use_bias", [True, False])
283
@pytest.mark.skipif(not current_platform.has_device_capability(89),
284
reason="FP8 is not supported on this GPU type.")
211
-def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
285
+def test_cutlass_fp8_gemm_m_sweep(a_scale_group_shape, b_scale_group_shape,
286
use_bias: bool):
287
for nk in range(32, 128, 32):
288
for m in range(1, 128):
215
- cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
216
- use_bias)
289
+ cutlass_fp8_gemm_helper(m, nk, nk, a_scale_group_shape,
290
+ b_scale_group_shape, use_bias)
291
292
219
-@pytest.mark.parametrize("per_act_token", [True, False])
220
-@pytest.mark.parametrize("per_out_ch", [True, False])
293
+@pytest.mark.parametrize("a_scale_group_shape",
294
+ [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
295
+@pytest.mark.parametrize("b_scale_group_shape",
296
+ [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
297
@pytest.mark.parametrize("use_bias", [True, False])
222
-def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
298
+def test_cutlass_int8_gemm_m_sweep(a_scale_group_shape, b_scale_group_shape,
299
use_bias: bool):
300
for nk in range(32, 128, 32):
301
for m in range(1, 128):
226
- cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
227
- use_bias)
302
+ cutlass_int8_gemm_helper(m, nk, nk, a_scale_group_shape,
303
+ b_scale_group_shape, use_bias)
304
305
306
@pytest.mark.parametrize("m", [32, 64, 128])
tests/kernels/utils.py
+30
-2
@@ -1119,8 +1119,36 @@ def baseline_scaled_mm(a: torch.Tensor,
1119
scale_b: torch.Tensor,
1120
out_dtype: Type[torch.dtype],
1121
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
1122
- output = (scale_a * (scale_b * (torch.mm(
1123
- a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
1122
+
1123
+ # We treat N-dimensional group scaling as extended numpy-style broadcasting
1124
+ # in numpy simply stretches dimensions with an extent of 1 to match the
1125
+ # the target shape by repeating the data along that dimension (broadcasting)
1126
+ # , we extend these semantics to say if the extent of a dimension in the
1127
+ # source shape is not 1 and does not match the target shape we repeat each
1128
+ # element along that dimension src_shape[dim] // target_shape[dim] times
1129
+ # example if we have:
1130
+ # a = [[1, 2], and target_shape = (2, 4)
1131
+ # [3, 4]]
1132
+ # then we would expand a to:
1133
+ # a = [[1, 1, 2, 2],
1134
+ # [3, 3, 4, 4]]
1135
+ # NOTE this function this function does not explicitly broadcast dimensions
1136
+ # with an extent of 1, since this can be done implicitly by pytorch
1137
+ def group_broadcast(t, shape):
1138
+ for i, s in enumerate(shape):
1139
+ if t.shape[i] != s and t.shape[i] != 1:
1140
+ assert s % t.shape[i] == 0
1141
+ t = t.unsqueeze(i + 1)\
1142
+ .expand(*t.shape[:i+1], s // t.shape[i], *t.shape[i+1:])\
1143
+ .flatten(i, i + 1)
1144
+ return t
1145
+
1146
+ scale_a = group_broadcast(scale_a, a.shape)
1147
+ scale_b = group_broadcast(scale_b, b.shape)
1148
+
1149
+ output = torch.mm((scale_a * a.to(dtype=torch.float32)),
1150
+ (scale_b * b.to(dtype=torch.float32))).to(out_dtype)
1151
+
1152
if bias is not None:
1153
output = output + bias
1154
vllm/_custom_ops.py
+22
@@ -441,6 +441,28 @@ def cutlass_scaled_mm(a: torch.Tensor,
441
scale_b: torch.Tensor,
442
out_dtype: torch.dtype,
443
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
444
+ """
445
+ `cutlass_scaled_mm` implements a fused version of
446
+ `output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)`
447
+ where scale_a * a and scale_b * b are implemented using numpy-style
448
+ broadcasting.
449
+
450
+ In order to support blockwise scaling like found in DeepSeek V3 we also
451
+ support extended "group" broadcast rules. We extend the numpy-style
452
+ broadcasting rules with the following rule:
453
+ "if the extent of a dimension in the source shape is between 1 and
454
+ corresponding extent in the target shape we repeat each element along
455
+ that dimension src_shape[dim] // target_shape[dim] times consecutively"
456
+ example if we have:
457
+ a = [[1, 2], and target_shape = (2, 4)
458
+ [3, 4]]
459
+ then we would expand a to:
460
+ a = [[1, 1, 2, 2],
461
+ [3, 3, 4, 4]]
462
+ currently we only support the case:
463
+ scale_a.shape * [1, 128] == a.shape
464
+ scale_b.shape * [128, 128] == b.shape
465
+ """
466
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
467
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
468
assert bias is None or bias.shape[0] == b.shape[