A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Add per-tensor and per-token AZP epilogues (#5941)
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Luka Govedič committed
Aug 6, 2024 at 14:17 UTC
8d59dbb00044a588cab96bcdc028006ed922eb06
11 files changed
+1175
-153
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+107
-78
@@ -32,7 +32,6 @@ def to_int8(tensor: torch.Tensor) -> torch.Tensor:
32
33
def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
34
k: int) -> Tuple[torch.Tensor, torch.Tensor]:
35
-
35
a = torch.randn((m, k), device='cuda') * 5
36
b = torch.randn((n, k), device='cuda').t() * 5
37
@@ -44,59 +43,18 @@ def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
43
raise ValueError("unsupported dtype")
44
45
47
-# impl
48
-
49
-
50
-def pytorch_mm_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
51
- scale_b: torch.Tensor,
52
- out_dtype: torch.dtype) -> torch.Tensor:
53
- return torch.mm(a, b)
54
-
55
-
56
-def pytorch_fp8_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
57
- scale_b: torch.Tensor,
58
- out_dtype: torch.dtype) -> torch.Tensor:
59
- return torch._scaled_mm(a,
60
- b,
61
- scale_a=scale_a,
62
- scale_b=scale_b,
63
- out_dtype=out_dtype)
64
-
65
-
66
-def pytorch_fp8_impl_fast_accum(a: torch.Tensor, b: torch.Tensor,
67
- scale_a: torch.Tensor, scale_b: torch.Tensor,
68
- out_dtype: torch.dtype) -> torch.Tensor:
69
- return torch._scaled_mm(a,
70
- b,
71
- scale_a=scale_a,
72
- scale_b=scale_b,
73
- out_dtype=out_dtype,
74
- use_fast_accum=True)
75
-
76
-
77
-def cutlass_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
78
- scale_b: torch.Tensor,
79
- out_dtype: torch.dtype) -> torch.Tensor:
80
- return ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=out_dtype)
81
-
82
-
46
# bench
84
-def bench_fn(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
85
- scale_b: torch.Tensor, out_dtype: torch.dtype, label: str,
86
- sub_label: str, fn: Callable, description: str) -> TMeasurement:
87
-
47
+def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
48
+ **kwargs) -> TMeasurement:
49
min_run_time = 1
50
51
globals = {
91
- "a": a,
92
- "b": b,
93
- "scale_a": scale_a,
94
- "scale_b": scale_b,
95
- "out_dtype": out_dtype,
52
+ "args": args,
53
+ "kwargs": kwargs,
54
"fn": fn,
55
}
56
return TBenchmark.Timer(
99
- stmt="fn(a, b, scale_a, scale_b, out_dtype)",
57
+ stmt="fn(*args, **kwargs)",
58
globals=globals,
59
label=label,
60
sub_label=sub_label,
@@ -110,26 +68,58 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
68
a, b = make_rand_tensors(torch.int8, m, n, k)
69
scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
70
scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
71
+ bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
72
+ azp = torch.zeros((m, ), device="cuda", dtype=torch.int32)
73
+ azp_adj = torch.zeros((n, ), device="cuda", dtype=torch.int32)
74
75
timers = []
76
# pytorch impl - bfloat16
77
timers.append(
117
- bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
118
- b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
119
- torch.bfloat16, label, sub_label, pytorch_mm_impl,
120
- "pytorch_bf16_bf16_bf16_matmul-no-scales"))
78
+ bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
79
+ torch.mm, a.to(dtype=torch.bfloat16),
80
+ b.to(dtype=torch.bfloat16)))
81
82
# pytorch impl - float16
83
timers.append(
124
- bench_fn(a.to(dtype=torch.float16, device="cuda"),
125
- b.to(dtype=torch.float16, device="cuda"), scale_a, scale_b,
126
- torch.float16, label, sub_label, pytorch_mm_impl,
127
- "pytorch_fp16_fp16_fp16_matmul-no-scales"))
84
+ bench_fn(label, sub_label,
85
+ "pytorch_fp16_fp16_fp16_matmul-no-scales", torch.mm,
86
+ a.to(dtype=torch.float16), b.to(dtype=torch.float16)))
87
88
# cutlass impl
89
timers.append(
131
- bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
132
- cutlass_impl, "cutlass_i8_i8_bf16_scaled_mm"))
90
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm",
91
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
92
+ torch.bfloat16))
93
+
94
+ # cutlass with bias
95
+ timers.append(
96
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_bias",
97
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
98
+ bias))
99
+
100
+ # cutlass with azp per-tensor
101
+ timers.append(
102
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp",
103
+ ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
104
+ torch.bfloat16, azp_adj))
105
+
106
+ # cutlass with azp per-tensor + bias
107
+ timers.append(
108
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_bias",
109
+ ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
110
+ torch.bfloat16, azp_adj, None, bias))
111
+
112
+ # cutlass with azp per-token
113
+ timers.append(
114
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt",
115
+ ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
116
+ torch.bfloat16, azp_adj, azp))
117
+
118
+ # cutlass with azp per-token + bias
119
+ timers.append(
120
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias",
121
+ ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
122
+ torch.bfloat16, azp_adj, azp, bias))
123
124
return timers
125
@@ -140,46 +130,88 @@ def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
130
a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)
131
scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
132
scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
133
+ bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
134
135
timers = []
136
137
# pytorch impl w. bf16
138
timers.append(
148
- bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
149
- b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
150
- torch.bfloat16, label, sub_label, pytorch_mm_impl,
151
- "pytorch_bf16_bf16_bf16_matmul-no-scales"))
139
+ bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
140
+ torch.mm, a.to(dtype=torch.bfloat16, device="cuda"),
141
+ b.to(dtype=torch.bfloat16, device="cuda")))
142
143
# pytorch impl: bf16 output, without fp8 fast accum
144
timers.append(
155
- bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
156
- pytorch_fp8_impl, "pytorch_fp8_fp8_bf16_scaled_mm"))
145
+ bench_fn(label,
146
+ sub_label,
147
+ "pytorch_fp8_fp8_bf16_scaled_mm",
148
+ torch._scaled_mm,
149
+ a,
150
+ b,
151
+ scale_a=scale_a,
152
+ scale_b=scale_b,
153
+ out_dtype=torch.bfloat16))
154
155
# pytorch impl: bf16 output, with fp8 fast accum
156
timers.append(
160
- bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
161
- pytorch_fp8_impl_fast_accum,
162
- "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum"))
157
+ bench_fn(label,
158
+ sub_label,
159
+ "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
160
+ torch._scaled_mm,
161
+ a,
162
+ b,
163
+ scale_a=scale_a,
164
+ scale_b=scale_b,
165
+ out_dtype=torch.bfloat16,
166
+ use_fast_accum=True))
167
168
# pytorch impl: fp16 output, without fp8 fast accum
169
timers.append(
166
- bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
167
- pytorch_fp8_impl, "pytorch_fp8_fp8_fp16_scaled_mm"))
170
+ bench_fn(label,
171
+ sub_label,
172
+ "pytorch_fp8_fp8_fp16_scaled_mm",
173
+ torch._scaled_mm,
174
+ a,
175
+ b,
176
+ scale_a=scale_a,
177
+ scale_b=scale_b,
178
+ out_dtype=torch.float16))
179
180
# pytorch impl: fp16 output, with fp8 fast accum
181
timers.append(
171
- bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
172
- pytorch_fp8_impl_fast_accum,
173
- "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum"))
182
+ bench_fn(label,
183
+ sub_label,
184
+ "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
185
+ torch._scaled_mm,
186
+ a,
187
+ b,
188
+ scale_a=scale_a,
189
+ scale_b=scale_b,
190
+ out_dtype=torch.float16,
191
+ use_fast_accum=True))
192
193
# cutlass impl: bf16 output
194
timers.append(
177
- bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
178
- cutlass_impl, "cutlass_fp8_fp8_bf16_scaled_mm"))
195
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm",
196
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
197
+ torch.bfloat16))
198
# cutlass impl: fp16 output
199
timers.append(
181
- bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
182
- cutlass_impl, "cutlass_fp8_fp8_fp16_scaled_mm"))
200
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm",
201
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16))
202
+
203
+ # cutlass impl: bf16 output, with bias
204
+ timers.append(
205
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm_bias",
206
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
207
+ bias))
208
+
209
+ # cutlass impl: fp16 output, with bias
210
+ timers.append(
211
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm_bias",
212
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16,
213
+ bias.to(dtype=torch.float16)))
214
+
215
return timers
216
217
@@ -200,7 +232,6 @@ def print_timers(timers: Iterable[TMeasurement]):
232
233
def run(dtype: torch.dtype,
234
MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
203
-
235
results = []
236
for m, k, n in MKNs:
237
timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
@@ -216,7 +247,6 @@ def make_output(data: Iterable[TMeasurement],
247
MKNs: Iterable[Tuple[int, int, int]],
248
base_description: str,
249
timestamp=None):
219
-
250
print(f"== All Results {base_description} ====")
251
print_timers(data)
252
@@ -251,7 +281,6 @@ def run_range_bench(args):
281
282
283
def run_model_bench(args):
254
-
284
print("Benchmarking models:")
285
for i, model in enumerate(args.models):
286
print(f"[{i}] {model}")
csrc/ops.h
+8
@@ -128,6 +128,14 @@ void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
128
torch::Tensor const& b_scales,
129
c10::optional<torch::Tensor> const& bias);
130
131
+void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
132
+ torch::Tensor const& b,
133
+ torch::Tensor const& a_scales,
134
+ torch::Tensor const& b_scales,
135
+ torch::Tensor const& azp_adj,
136
+ c10::optional<torch::Tensor> const& azp,
137
+ c10::optional<torch::Tensor> const& bias);
138
+
139
torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
140
torch::Tensor const& b_q_weight,
141
torch::Tensor const& s_tok,
csrc/quantization/cutlass_w8a8/Epilogues.md
new
+147
@@ -0,0 +1,147 @@
1
+# CUTLASS Epilogues
2
+
3
+## Introduction
4
+This document describes the various CUTLASS epilogues implemented for fusing de-quantization operations onto GEMMs.
5
+
6
+Currently, we only support symmetric quantization for weights,
7
+and symmetric and asymmetric quantization for activations.
8
+Both can be quantized per-tensor or per-channel (weights) / per-token (activations).
9
+
10
+There are 4 epilogues:
11
+1. ScaledEpilogue: symmetric quantization for activations, no bias.
12
+1. ScaledEpilogueBias: symmetric quantization for activations, supports bias.
13
+1. ScaledEpilogueAzp: asymmetric per-tensor quantization for activations, supports bias.
14
+1. ScaledEpilogueAzpPerToken: asymmetric per-token quantization for activations, supports bias.
15
+
16
+We do not have epilogues for asymmetric quantization of activations without bias in order to reduce final binary size.
17
+Instead, if no bias is passed, the epilogue will use 0 as the bias.
18
+That induces a redundant addition operation (and runtime check), but the performance impact is minor.
19
+
20
+## Underlying Linear Algebra
21
+
22
+More details available in the [Activation Quantization RFC](https://github.com/vllm-project/vllm/issues/3975).
23
+
24
+If $` \widehat X `$ is the quantized $` X `$, our matrices become the following
25
+
26
+```math
27
+A = s_a (\widehat A - J_a z_a)
28
+```
29
+```math
30
+B = s_b \widehat B
31
+```
32
+```math
33
+D = A B + C
34
+```
35
+```math
36
+D = s_a s_b \widehat D + C
37
+```
38
+
39
+Here, D is the output of the GEMM, and C is the bias.
40
+A is the activations and supports asymmetric quantization,
41
+and B is the weights and only supports symmetric quantization.
42
+$ s_a $ and $s_b$ are the scales for activations and weights, respectively.
43
+$ z_a $ is the zero-point for activations, and $ J_a $ is the matrix of all ones with dimensions of A.
44
+Additional epilogues would be required to support asymmetric quantization for weights.
45
+
46
+Expanding further, we can calculate $` \widehat D `$ as follows:
47
+
48
+```math
49
+A B = s_a ( \widehat A - J_a z_a ) s_b \widehat B
50
+```
51
+```math
52
+A B = s_a s_b \left( \widehat A \widehat B - J_a z_a \widehat B \right)
53
+```
54
+```math
55
+\widehat D = \widehat A \widehat B - z_a J_a \widehat B
56
+```
57
+
58
+Note that $` \widehat A \widehat B `$ is the raw output of the GEMM,
59
+and $` J_a \widehat B `$ is known ahead of time.
60
+Each row of it is equal to $` \mathbf 1 \widehat B `$, which is a row-vector of column sums of $` \widehat B `$.
61
+
62
+## Epilogues
63
+
64
+### ScaledEpilogue
65
+This epilogue computes the symmetric quantization for activations without bias, meaning $` C = 0 `$ and $` z_a = 0 `$.
66
+The output of the GEMM is:
67
+
68
+```math
69
+\widehat D = \widehat A \widehat B
70
+```
71
+```math
72
+D = s_a s_b \widehat D
73
+```
74
+```math
75
+D = s_a s_b \widehat A \widehat B
76
+```
77
+
78
+Epilogue parameters:
79
+- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
80
+- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
81
+
82
+### ScaledEpilogueBias
83
+This epilogue computes the symmetric quantization for activations with bias, meaning $` z_a = 0 `$.
84
+The output of the GEMM is:
85
+
86
+```math
87
+\widehat D = \widehat A \widehat B
88
+```
89
+```math
90
+D = s_a s_b \widehat D + C
91
+```
92
+```math
93
+D = s_a s_b \widehat A \widehat B + C
94
+```
95
+
96
+
97
+Epilogue parameters:
98
+- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
99
+- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
100
+- `bias` is the bias, is always per-channel (row-vector).
101
+
102
+### ScaledEpilogueAzp
103
+This epilogue computes the asymmetric per-tensor quantization for activations with bias.
104
+The output of the GEMM is:
105
+
106
+```math
107
+\widehat D = \widehat A \widehat B - z_a J_a \widehat B
108
+```
109
+```math
110
+D = s_a s_b \widehat D + C
111
+```
112
+```math
113
+D = s_a s_b \left( \widehat A \widehat B - z_a J_a \widehat B \right) + C
114
+```
115
+
116
+Because $` z_a `$ is a scalar, the zero-point term $` z_a J_a \widehat B `$ has every row equal to $` z_a \mathbf 1 B `$.
117
+That is precomputed and stored in `azp_with_adj` as a row-vector.
118
+
119
+Epilogue parameters:
120
+- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
121
+ - Generally this will be per-tensor as the zero-points are per-tensor.
122
+- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
123
+- `azp_with_adj` is the precomputed zero-point term ($` z_a J_a \widehat B `$), is per-channel (row-vector).
124
+- `bias` is the bias, is always per-channel (row-vector).
125
+
126
+To use these kernels efficiently, users must precompute the `azp_with_adj` term offline and pass it to the kernel.
127
+
128
+### ScaledEpilogueAzpPerToken
129
+This epilogue computes the asymmetric per-token quantization for activations with bias.
130
+
131
+The output of the GEMM is the same as above, but the $` z_a `$ is a column-vector.
132
+That means the zero-point term $` z_a J_a \widehat B `$ becomes an outer product of $` z_a `$ and $` \mathbf 1 \widehat B `$.
133
+
134
+Epilogue parameters:
135
+- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
136
+ - Generally this will be per-token as the zero-points are per-token.
137
+- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
138
+- `azp_adj` is the precomputed zero-point adjustment term ($` \mathbf 1 \widehat B `$), is per-channel (row-vector).
139
+- `azp` is the zero-point (`z_a`), is per-token (column-vector).
140
+- `bias` is the bias, is always per-channel (row-vector).
141
+
142
+To use these kernels efficiently, users must precompute the `azp_adj` term offline and pass it to the kernel.
143
+
144
+The epilogue performs the following computation (where `Dq` is the raw quantized output of the GEMM):
145
+```
146
+out = scale_a * scale_b * (Dq - azp_adj * azp) + bias
147
+```
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c2x.hpp
+151
-1
@@ -207,6 +207,156 @@ struct VisitorRowOrScalarBroadcast {
207
208
};
209
210
+/////////////////////////////////////////////////////////////////////////////////////////////////
211
+
212
+// This is a modified RowBroadcast that will broadcast 0 if ptr_row is null
213
+template<
214
+ class ThreadMap,
215
+ class Element,
216
+ class StrideMNL
217
+>
218
+struct VisitorRowOrZeroBroadcast {
219
+
220
+ // This struct has been modified to remove null_default (because it's always 0)
221
+ struct Arguments {
222
+ Element const* ptr_row = nullptr;
223
+ StrideMNL dRow = {};
224
+ };
225
+
226
+ using Params = Arguments;
227
+
228
+ template <class ProblemShape>
229
+ static constexpr Params
230
+ to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
231
+ return args;
232
+ }
233
+
234
+ template <class ProblemShape>
235
+ static size_t
236
+ get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
237
+ return 0;
238
+ }
239
+
240
+ struct SharedStorage {};
241
+
242
+ // Global load type
243
+ static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
244
+ using VecType = uint_bit_t<cute::min(128, vec_bits)>;
245
+ static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
246
+
247
+ CUTLASS_HOST_DEVICE
248
+ VisitorRowOrZeroBroadcast() { }
249
+
250
+ CUTLASS_HOST_DEVICE
251
+ VisitorRowOrZeroBroadcast(Params const& params, SharedStorage const& shared_storage)
252
+ : params_ptr(¶ms) { }
253
+
254
+ Params const* params_ptr;
255
+
256
+ template <class GTensor, class RTensor, class CTensor, class ProblemShape>
257
+ struct Callbacks : EmptyCallbacks {
258
+ CUTLASS_DEVICE
259
+ Callbacks(
260
+ GTensor&& tC_gRow,
261
+ RTensor&& tC_rRow,
262
+ CTensor&& tC_cRow,
263
+ ProblemShape problem_shape,
264
+ Params const* params_ptr
265
+ ):
266
+ tC_gRow(cute::forward<GTensor>(tC_gRow)),
267
+ tC_rRow(cute::forward<RTensor>(tC_rRow)),
268
+ tC_cRow(cute::forward<CTensor>(tC_cRow)),
269
+ n(get<1>(problem_shape)),
270
+ params_ptr(params_ptr) { }
271
+
272
+ GTensor tC_gRow;
273
+ RTensor tC_rRow;
274
+ CTensor tC_cRow;
275
+ Params const* params_ptr;
276
+ int n;
277
+
278
+ // This function is modified from VisitorRowBroadcast
279
+ CUTLASS_DEVICE void
280
+ begin_epilogue() {
281
+ clear(tC_rRow);
282
+ auto src_v = filter(tC_gRow);
283
+ auto coord_v = filter(tC_cRow);
284
+ auto dst_v = filter(tC_rRow);
285
+
286
+ if (params_ptr->ptr_row != nullptr) {
287
+ // In this case we are loading from a row vector and broadcasting
288
+ CUTLASS_PRAGMA_UNROLL
289
+ for (int i = 0; i < size(src_v); ++i) {
290
+ bool guard = get<1>(coord_v(i)) < n;
291
+ cutlass::arch::global_load<VecType, sizeof(VecType)>(
292
+ dst_v(i), (void const*)&src_v(i), guard);
293
+ }
294
+ } else {
295
+ // In this case we are broadcasting 0
296
+ VecType filled_vec;
297
+ CUTLASS_PRAGMA_UNROLL
298
+ for (int i = 0; i < VecLength; i++) {
299
+ reinterpret_cast<Element*>(&filled_vec)[i] = Element{0};
300
+ }
301
+
302
+ CUTLASS_PRAGMA_UNROLL
303
+ for (int i = 0; i < size(src_v); ++i) {
304
+ if (get<1>(coord_v(i)) < n) {
305
+ dst_v(i) = filled_vec;
306
+ }
307
+ }
308
+ }
309
+ }
310
+
311
+ template <class ElementAccumulator, int FragmentSize>
312
+ CUTLASS_DEVICE auto // returns an Array
313
+ visit(int iter_idx, int row_idx, int column_idx, int frg_idx,
314
+ Array<ElementAccumulator, FragmentSize> const& frg_acc) {
315
+ Tensor rRow_frg = recast<Array<Element, FragmentSize>>(coalesce(tC_rRow));
316
+ return rRow_frg(column_idx);
317
+ }
318
+ };
319
+
320
+ template <class ProblemShape>
321
+ CUTLASS_DEVICE auto
322
+ get_callbacks(
323
+ gemm::GemmCoord threadblock_tile_offset,
324
+ int thread_idx,
325
+ ProblemShape problem_shape
326
+ ) {
327
+ Tensor mRow = make_tensor(
328
+ make_gmem_ptr(params_ptr->ptr_row),
329
+ problem_shape,
330
+ params_ptr->dRow);
331
+
332
+ // VECTOR, FRAGMENT_COLUMN
333
+ Tensor tC_gRow = recast<VecType>(
334
+ ThreadMap::partition(mRow, thread_idx, threadblock_tile_offset)
335
+ )(_,_,_0{},_0{},_0{},_0{});
336
+ Tensor tC_rRow = make_tensor_like(tC_gRow);
337
+
338
+ // Generate the pred tensor
339
+ Tensor cRow = make_identity_tensor(mRow.shape());
340
+ Tensor tC_cRow = outer_partition(
341
+ ThreadMap::partition(cRow, thread_idx, threadblock_tile_offset)(_,_,_0{},_0{},_0{},_0{}),
342
+ Shape<Int<VecLength>>{},
343
+ (_0{})
344
+ );
345
+
346
+ return Callbacks<
347
+ decltype(tC_gRow), decltype(tC_rRow),
348
+ decltype(tC_cRow), ProblemShape>(
349
+ cute::move(tC_gRow),
350
+ cute::move(tC_rRow),
351
+ cute::move(tC_cRow),
352
+ problem_shape,
353
+ params_ptr
354
+ );
355
+ }
356
+
357
+};
358
+
359
+
360
/////////////////////////////////////////////////////////////////////////////////////////////////
361
362
// Column vector broadcast
@@ -217,7 +367,7 @@ template<
367
>
368
struct VisitorColOrScalarBroadcast {
369
220
- // This struct has been modified to have a bool indicating that ptr_col is a
370
+ // This struct has been modified to have a bool indicating that ptr_col is a
371
// scalar that must be broadcast.
372
struct Arguments {
373
Element const* ptr_col = nullptr;
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+57
@@ -50,6 +50,25 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
50
}
51
}
52
53
+void cutlass_scaled_mm_azp_sm75(torch::Tensor& out, torch::Tensor const& a,
54
+ torch::Tensor const& b,
55
+ torch::Tensor const& a_scales,
56
+ torch::Tensor const& b_scales,
57
+ torch::Tensor const& azp_adj,
58
+ c10::optional<torch::Tensor> const& azp,
59
+ c10::optional<torch::Tensor> const& bias) {
60
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
61
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
62
+
63
+ if (azp) {
64
+ return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
65
+ out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
66
+ } else {
67
+ return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzp>(
68
+ out, a, b, a_scales, b_scales, azp_adj, bias);
69
+ }
70
+}
71
+
72
template <template <typename, typename> typename Epilogue,
73
typename... EpilogueArgs>
74
void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
@@ -87,6 +106,25 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
106
}
107
}
108
109
+void cutlass_scaled_mm_azp_sm80(torch::Tensor& out, torch::Tensor const& a,
110
+ torch::Tensor const& b,
111
+ torch::Tensor const& a_scales,
112
+ torch::Tensor const& b_scales,
113
+ torch::Tensor const& azp_adj,
114
+ c10::optional<torch::Tensor> const& azp,
115
+ c10::optional<torch::Tensor> const& bias) {
116
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
117
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
118
+
119
+ if (azp) {
120
+ return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
121
+ out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
122
+ } else {
123
+ return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzp>(
124
+ out, a, b, a_scales, b_scales, azp_adj, bias);
125
+ }
126
+}
127
+
128
template <template <typename, typename> typename Epilogue,
129
typename... EpilogueArgs>
130
void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
@@ -139,3 +177,22 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
177
out, a, b, a_scales, b_scales);
178
}
179
}
180
+
181
+void cutlass_scaled_mm_azp_sm89(torch::Tensor& out, torch::Tensor const& a,
182
+ torch::Tensor const& b,
183
+ torch::Tensor const& a_scales,
184
+ torch::Tensor const& b_scales,
185
+ torch::Tensor const& azp_adj,
186
+ c10::optional<torch::Tensor> const& azp,
187
+ c10::optional<torch::Tensor> const& bias) {
188
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
189
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
190
+
191
+ if (azp) {
192
+ return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
193
+ out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
194
+ } else {
195
+ return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzp>(
196
+ out, a, b, a_scales, b_scales, azp_adj, bias);
197
+ }
198
+}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+217
-36
@@ -73,19 +73,63 @@ struct enable_sm89_to_sm90 : Kernel {
73
};
74
75
/*
76
- * This class provides the common ScaleA and ScaleB descriptors for the
77
- * ScaledEpilogue and ScaledEpilogueBias classes.
76
+ * This class provides the common load descriptors for the
77
+ * ScaledEpilogue[...] classes
78
*/
79
template <typename ElementD, typename OutputTileThreadMap>
80
struct ScaledEpilogueBase {
81
protected:
82
using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
83
84
- using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
85
- OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
84
+ template <typename T>
85
+ using ColOrScalarLoad =
86
+ cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
87
+ OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
88
+
89
+ template <typename T>
90
+ using RowOrScalarLoad =
91
+ cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92
+ OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
93
+
94
+ template <typename T>
95
+ using ColLoad = cutlass::epilogue::threadblock::VisitorColBroadcast<
96
+ OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
97
+
98
+ template <typename T>
99
+ using RowLoad = cutlass::epilogue::threadblock::VisitorRowBroadcast<
100
+ OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
101
+
102
+ template <typename T>
103
+ using RowOrZeroLoad =
104
+ cutlass::epilogue::threadblock::VisitorRowOrZeroBroadcast<
105
+ OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
106
+
107
+ // This utility function constructs the arguments for the load descriptors
108
+ // from a tensor. It can handle both row and column, as well as row/column or
109
+ // scalar cases.
110
+ template <typename Descriptor, typename T>
111
+ static auto args_from_tensor(torch::Tensor const& tensor) {
112
+ using Arguments = typename Descriptor::Arguments;
113
+ auto* data_ptr = static_cast<T*>(tensor.data_ptr());
114
+ if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
115
+ std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
116
+ return Arguments{data_ptr, tensor.numel() != 1};
117
+ } else {
118
+ // it would technically work but no use case as data_ptr is never nullptr
119
+ static_assert(!std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
120
+ return Arguments{data_ptr};
121
+ }
122
+ }
123
87
- using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
88
- OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
124
+ // This overload handles the case where there might not be a tensor, in which
125
+ // case a nullptr is passed and a constant (0) is used.
126
+ template <typename Descriptor, typename T>
127
+ static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
128
+ static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
129
+ using Arguments = typename Descriptor::Arguments;
130
+ auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
131
+ return Arguments{data_ptr};
132
+ }
133
};
134
135
/*
@@ -110,8 +154,8 @@ struct ScaledEpilogue
154
private:
155
using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156
using Accum = typename SUPER::Accum;
113
- using ScaleA = typename SUPER::ScaleA;
114
- using ScaleB = typename SUPER::ScaleB;
157
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
158
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
159
160
using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161
cutlass::multiplies, float, float,
@@ -131,28 +175,32 @@ struct ScaledEpilogue
175
176
static ArgumentType prepare_args(torch::Tensor const& a_scales,
177
torch::Tensor const& b_scales) {
134
- using ScaleAArgs = typename ScaleA::Arguments;
135
- using ScaleBArgs = typename ScaleB::Arguments;
136
-
137
- ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
138
- ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
178
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
179
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
180
140
- typename EVTCompute0::Arguments evt0_compute_args{b_args};
141
-
142
- typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
143
- return evt_compute_args;
181
+ typename EVTCompute0::Arguments evt0_args{b_args};
182
+ return ArgumentType{a_args, evt0_args};
183
}
184
};
185
186
+/*
187
+ * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
188
+ * This bias can also be used in the per-tensor azp case, where the activation
189
+ * zero point (azp) is used to compute an azp correction term,
190
+ * which is folded into the bias.
191
+ *
192
+ * The bias tensor must be per-output channel.
193
+ * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
194
+ */
195
template <typename ElementD, typename OutputTileThreadMap>
196
struct ScaledEpilogueBias
149
- : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
150
- private:
197
+ : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
198
+ protected:
199
using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
200
using Accum = typename SUPER::Accum;
153
- using ScaleA = typename SUPER::ScaleA;
154
- using ScaleB = typename SUPER::ScaleB;
155
-
201
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
202
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
203
+ using Bias = typename SUPER::template RowLoad<ElementD>;
204
using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
205
cutlass::multiplies, float, float,
206
cutlass::FloatRoundStyle::round_to_nearest>;
@@ -164,30 +212,163 @@ struct ScaledEpilogueBias
212
cutlass::multiply_add, ElementD, float,
213
cutlass::FloatRoundStyle::round_to_nearest>;
214
167
- using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
168
- OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
169
-
215
public:
216
using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
217
EVTCompute0, Bias>;
218
using ArgumentType = typename EVTCompute::Arguments;
174
-
219
static ArgumentType prepare_args(torch::Tensor const& a_scales,
220
torch::Tensor const& b_scales,
221
torch::Tensor const& bias) {
178
- using ScaleAArgs = typename ScaleA::Arguments;
179
- using ScaleBArgs = typename ScaleB::Arguments;
180
- using BiasArgs = typename Bias::Arguments;
222
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
223
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
224
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
225
+
226
+ typename EVTCompute0::Arguments evt0_args{b_args};
227
+ return ArgumentType{a_args, evt0_args, bias_args};
228
+ }
229
+};
230
+
231
+/*
232
+ * This epilogue directly supports per-tensor azp in int32 form.
233
+ * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
234
+ * term, which should already be multiplied with the scalar azp.
235
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
236
+ *
237
+ * This epilogue also supports bias, which remains per-channel.
238
+ */
239
+template <typename ElementD, typename OutputTileThreadMap>
240
+struct ScaledEpilogueBiasAzp
241
+ : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
242
+ private:
243
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
244
+ using Accum = typename SUPER::Accum;
245
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
246
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
247
+ using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
248
+
249
+ // This is the full AZP term, azp * J @ B, shape (1,n)
250
+ using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
251
+
252
+ // Compute float(accum - azp_adj), both operands are int32_t
253
+ using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
254
+ cutlass::minus, float, int32_t,
255
+ cutlass::FloatRoundStyle::round_to_nearest>;
256
+
257
+ using EVTComputeAzp =
258
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Accum, AzpWithAdj>;
259
+
260
+ using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
261
+ cutlass::multiplies, float, float,
262
+ cutlass::FloatRoundStyle::round_to_nearest>;
263
+
264
+ using EVTComputeScaleB =
265
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
266
+ EVTComputeAzp>;
267
+
268
+ using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
269
+ cutlass::multiply_add, ElementD, float,
270
+ cutlass::FloatRoundStyle::round_to_nearest>;
271
+
272
+ public:
273
+ using EVTCompute =
274
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
275
+ EVTComputeScaleB, Bias>;
276
+
277
+ using ArgumentType = typename EVTCompute::Arguments;
278
+
279
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
280
+ torch::Tensor const& b_scales,
281
+ torch::Tensor const& azp_adj,
282
+ c10::optional<torch::Tensor> const& bias) {
283
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
284
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
285
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
286
+ auto azp_adj_args =
287
+ SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
288
+
289
+ typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
290
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
291
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
292
+ }
293
+};
294
+
295
+/*
296
+ * This epilogue supports per-token azp by computing and applying
297
+ * the correction term using a rank-1 update. If the term were materialized,
298
+ * it would require O(m*n) space, and this way it only requires O(m+n) space.
299
+ * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
300
+ * point for each row of A.
301
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
302
+ *
303
+ * This epilogue also supports bias, which remains per-channel.
304
+ */
305
+template <typename ElementD, typename OutputTileThreadMap>
306
+struct ScaledEpilogueBiasAzpToken
307
+ : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
308
+ private:
309
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
310
+ using Accum = typename SUPER::Accum;
311
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
312
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
313
+ using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
314
+
315
+ // Per-token azp term, shape (m,1)
316
+ using Azp = typename SUPER::template ColLoad<int32_t>;
317
182
- ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
183
- ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
184
- BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
318
+ // This is the AZP adjustment term, J @ B, shape (1,n)
319
+ using AzpAdj = typename SUPER::template RowLoad<int32_t>;
320
+
321
+ // Compute azp * azp_adj
322
+ using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
323
+ cutlass::multiplies, int32_t, int32_t,
324
+ cutlass::FloatRoundStyle::round_to_nearest>;
325
186
- typename EVTCompute0::Arguments evt0_compute_args{b_args};
326
+ using EVTComputeAzp =
327
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Azp, AzpAdj>;
328
188
- typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
189
- bias_args};
190
- return evt_compute_args;
329
+ // Compute float(accum - azp*azp_adj), all operands are int32_t
330
+ using ComputeAcc = cutlass::epilogue::threadblock::VisitorCompute<
331
+ cutlass::minus, float, int32_t,
332
+ cutlass::FloatRoundStyle::round_to_nearest>;
333
+
334
+ using EVTComputeAcc =
335
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeAcc, Accum, EVTComputeAzp>;
336
+
337
+ using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
338
+ cutlass::multiplies, float, float,
339
+ cutlass::FloatRoundStyle::round_to_nearest>;
340
+
341
+ using EVTComputeScaleB =
342
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
343
+ EVTComputeAcc>;
344
+
345
+ using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
346
+ cutlass::multiply_add, ElementD, float,
347
+ cutlass::FloatRoundStyle::round_to_nearest>;
348
+
349
+ public:
350
+ using EVTCompute =
351
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
352
+ EVTComputeScaleB, Bias>;
353
+
354
+ using ArgumentType = typename EVTCompute::Arguments;
355
+
356
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
357
+ torch::Tensor const& b_scales,
358
+ torch::Tensor const& azp_adj,
359
+ torch::Tensor const& azp,
360
+ c10::optional<torch::Tensor> const& bias) {
361
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
362
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
363
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
364
+ auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
365
+ auto azp_adj_args =
366
+ SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
367
+
368
+ typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
369
+ typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
370
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
371
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
372
}
373
};
374
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+230
-28
@@ -58,21 +58,63 @@ struct enable_sm90_or_later : Kernel {
58
};
59
60
/*
61
- * This class provides the common ScaleA and ScaleB descriptors for the
62
- * ScaledEpilogue and ScaledEpilogueBias classes.
61
+ * This class provides the common load descriptors for the
62
+ * ScaledEpilogue[...] classes
63
*/
64
template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
65
struct ScaledEpilogueBase {
66
protected:
67
using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
68
69
- using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
70
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
69
+ template <typename T>
70
+ using ColOrScalarLoad = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
71
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
72
Stride<Int<1>, Int<0>, Int<0>>>;
73
73
- using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
74
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
74
+ template <typename T>
75
+ using RowOrScalarLoad = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
76
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
77
Stride<Int<0>, Int<1>, Int<0>>>;
78
+
79
+ // Don't want to support nullptr by default
80
+ template <typename T, bool EnableNullPtr = false>
81
+ using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
82
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
83
+ Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
84
+
85
+ // Don't want to support nullptr by default
86
+ template <typename T, bool EnableNullPtr = false>
87
+ using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
88
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
89
+ Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
90
+
91
+ // This utility function constructs the arguments for the load descriptors
92
+ // from a tensor. It can handle both row and column, as well as row/column or
93
+ // scalar cases.
94
+ template <typename Descriptor, typename T>
95
+ static auto args_from_tensor(torch::Tensor const& tensor) {
96
+ using Arguments = typename Descriptor::Arguments;
97
+ auto* data_ptr = static_cast<T*>(tensor.data_ptr());
98
+ if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
99
+ std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
100
+ return Arguments{data_ptr, tensor.numel() != 1};
101
+ } else {
102
+ static_assert(!std::is_same_v<Descriptor, ColLoad<T, true>> &&
103
+ !std::is_same_v<Descriptor, RowLoad<T, true>>);
104
+ return Arguments{data_ptr};
105
+ }
106
+ }
107
+
108
+ // This overload handles the case where there might not be a tensor, in which
109
+ // case a nullptr is passed and a constant (0) is used.
110
+ template <typename Descriptor, typename T>
111
+ static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
112
+ using Arguments = typename Descriptor::Arguments;
113
+ auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
114
+ static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
115
+ std::is_same_v<Descriptor, RowLoad<T, true>>);
116
+ return Arguments{data_ptr};
117
+ }
118
};
119
120
/*
@@ -97,8 +139,8 @@ struct ScaledEpilogue
139
private:
140
using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
141
using Accum = typename SUPER::Accum;
100
- using ScaleA = typename SUPER::ScaleA;
101
- using ScaleB = typename SUPER::ScaleB;
142
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
143
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
144
145
using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
146
cutlass::multiplies, float, float,
@@ -118,24 +160,32 @@ struct ScaledEpilogue
160
161
static ArgumentType prepare_args(torch::Tensor const& a_scales,
162
torch::Tensor const& b_scales) {
121
- using ScaleA_Args = typename ScaleA::Arguments;
122
- using ScaleB_Args = typename ScaleB::Arguments;
123
-
124
- ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
125
- ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
163
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
164
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
165
127
- return ArgumentType{a_args, {b_args}};
166
+ typename EVTCompute0::Arguments evt0_args{b_args};
167
+ return ArgumentType{a_args, evt0_args};
168
}
169
};
170
171
+/*
172
+ * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
173
+ * This bias can also be used in the per-tensor azp case, where the activation
174
+ * zero point (azp) is used to compute an azp correction term,
175
+ * which is folded into the bias.
176
+ *
177
+ * The bias tensor must be per-output channel.
178
+ * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
179
+ */
180
template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
181
struct ScaledEpilogueBias
182
: private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
183
private:
184
using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
185
using Accum = typename SUPER::Accum;
137
- using ScaleA = typename SUPER::ScaleA;
138
- using ScaleB = typename SUPER::ScaleB;
186
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
187
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
188
+ using Bias = typename SUPER::template RowLoad<ElementD>;
189
190
using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
191
cutlass::multiplies, float, float,
@@ -148,27 +198,160 @@ struct ScaledEpilogueBias
198
cutlass::multiply_add, ElementD, float,
199
cutlass::FloatRoundStyle::round_to_nearest>;
200
151
- using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast<
152
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, ElementD,
153
- Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<ElementD>, false>;
154
-
201
public:
202
using EVTCompute =
203
cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
158
- using ArgumentType = typename EVTCompute::Arguments;
204
205
+ using ArgumentType = typename EVTCompute::Arguments;
206
static ArgumentType prepare_args(torch::Tensor const& a_scales,
207
torch::Tensor const& b_scales,
208
torch::Tensor const& bias) {
163
- using ScaleA_Args = typename ScaleA::Arguments;
164
- using ScaleB_Args = typename ScaleB::Arguments;
165
- using Bias_Args = typename Bias::Arguments;
209
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
210
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
211
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
212
+
213
+ typename EVTCompute0::Arguments evt0_args{b_args};
214
+ return ArgumentType{a_args, evt0_args, bias_args};
215
+ }
216
+};
217
+
218
+/*
219
+ * This epilogue directly supports per-tensor azp in int32 form.
220
+ * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
221
+ * term, which should already be multiplied with the scalar azp.
222
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
223
+ *
224
+ * This epilogue also supports bias, which remains per-channel.
225
+ */
226
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
227
+struct ScaledEpilogueBiasAzp
228
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
229
+ private:
230
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
231
+ using Accum = typename SUPER::Accum;
232
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
233
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
234
+ using Bias = typename SUPER::template RowLoad<ElementD, true>;
235
+
236
+ // This is the full AZP term, azp * J @ B, shape (1,n)
237
+ using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
238
+
239
+ // Compute float(accum - azp_adj), both operands are int32_t
240
+ using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
241
+ cutlass::minus, float, int32_t,
242
+ cutlass::FloatRoundStyle::round_to_nearest>;
243
+
244
+ using EVTComputeAzp =
245
+ cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Accum, AzpWithAdj>;
246
+
247
+ using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
248
+ cutlass::multiplies, float, float,
249
+ cutlass::FloatRoundStyle::round_to_nearest>;
250
+
251
+ using EVTComputeScaleB =
252
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAzp>;
253
+
254
+ using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
255
+ cutlass::multiply_add, ElementD, float,
256
+ cutlass::FloatRoundStyle::round_to_nearest>;
257
+
258
+ public:
259
+ using EVTCompute =
260
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
261
+ EVTComputeScaleB, Bias>;
262
+ using ArgumentType = typename EVTCompute::Arguments;
263
+
264
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
265
+ torch::Tensor const& b_scales,
266
+ torch::Tensor const& azp_adj,
267
+ c10::optional<torch::Tensor> const& bias) {
268
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
269
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
270
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
271
+ auto azp_adj_args =
272
+ SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
273
+
274
+ typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
275
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
276
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
277
+ }
278
+};
279
+
280
+/*
281
+ * This epilogue supports per-token azp by computing and applying
282
+ * the correction term using a rank-1 update. If the term were materialized,
283
+ * it would require O(m*n) space, and this way it only requires O(m+n) space.
284
+ * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
285
+ * point for each row of A.
286
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
287
+ *
288
+ * This epilogue also supports bias, which remains per-channel.
289
+ */
290
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
291
+struct ScaledEpilogueBiasAzpToken
292
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
293
+ private:
294
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
295
+ using Accum = typename SUPER::Accum;
296
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
297
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
298
+ using Bias = typename SUPER::template RowLoad<ElementD, true>;
299
+
300
+ // Per-token azp term, shape (m,1)
301
+ using Azp = typename SUPER::template ColLoad<int32_t>;
302
+
303
+ // This is the AZP adjustment term, J @ B, shape (1,n)
304
+ using AzpAdj = typename SUPER::template RowLoad<int32_t>;
305
+
306
+ // Compute azp * azp_adj
307
+ using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
308
+ cutlass::multiplies, int32_t, int32_t,
309
+ cutlass::FloatRoundStyle::round_to_nearest>;
310
+
311
+ using EVTComputeAzp =
312
+ cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Azp, AzpAdj>;
313
167
- ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
168
- ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
169
- Bias_Args bias_args{static_cast<ElementD*>(bias.data_ptr())};
314
+ // Compute float(accum - azp*azp_adj), all operands are int32_t
315
+ using ComputeAcc = cutlass::epilogue::fusion::Sm90Compute<
316
+ cutlass::minus, float, int32_t,
317
+ cutlass::FloatRoundStyle::round_to_nearest>;
318
171
- return ArgumentType{a_args, {b_args}, bias_args};
319
+ using EVTComputeAcc =
320
+ cutlass::epilogue::fusion::Sm90EVT<ComputeAcc, Accum, EVTComputeAzp>;
321
+
322
+ using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
323
+ cutlass::multiplies, float, float,
324
+ cutlass::FloatRoundStyle::round_to_nearest>;
325
+
326
+ using EVTComputeScaleB =
327
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAcc>;
328
+
329
+ using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
330
+ cutlass::multiply_add, ElementD, float,
331
+ cutlass::FloatRoundStyle::round_to_nearest>;
332
+
333
+ public:
334
+ using EVTCompute =
335
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
336
+ EVTComputeScaleB, Bias>;
337
+ using ArgumentType = typename EVTCompute::Arguments;
338
+
339
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
340
+ torch::Tensor const& b_scales,
341
+ torch::Tensor const& azp_adj,
342
+ torch::Tensor const& azp,
343
+ c10::optional<torch::Tensor> const& bias) {
344
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
345
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
346
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
347
+ auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
348
+ auto azp_adj_args =
349
+ SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
350
+
351
+ typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
352
+ typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
353
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
354
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
355
}
356
};
357
@@ -546,4 +729,23 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
729
}
730
}
731
732
+void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
733
+ torch::Tensor const& b,
734
+ torch::Tensor const& a_scales,
735
+ torch::Tensor const& b_scales,
736
+ torch::Tensor const& azp_adj,
737
+ c10::optional<torch::Tensor> const& azp,
738
+ c10::optional<torch::Tensor> const& bias) {
739
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
740
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
741
+
742
+ if (azp) {
743
+ return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzpToken>(
744
+ out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
745
+ } else {
746
+ return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzp>(
747
+ out, a, b, a_scales, b_scales, azp_adj, bias);
748
+ }
749
+}
750
+
751
#endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+104
-7
@@ -29,6 +29,40 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
29
c10::optional<torch::Tensor> const& bias);
30
#endif
31
32
+void cutlass_scaled_mm_azp_sm75(torch::Tensor& c, torch::Tensor const& a,
33
+ torch::Tensor const& b,
34
+ torch::Tensor const& a_scales,
35
+ torch::Tensor const& b_scales,
36
+ torch::Tensor const& azp_adj,
37
+ c10::optional<torch::Tensor> const& azp,
38
+ c10::optional<torch::Tensor> const& bias);
39
+
40
+void cutlass_scaled_mm_azp_sm80(torch::Tensor& c, torch::Tensor const& a,
41
+ torch::Tensor const& b,
42
+ torch::Tensor const& a_scales,
43
+ torch::Tensor const& b_scales,
44
+ torch::Tensor const& azp_adj,
45
+ c10::optional<torch::Tensor> const& azp,
46
+ c10::optional<torch::Tensor> const& bias);
47
+
48
+void cutlass_scaled_mm_azp_sm89(torch::Tensor& c, torch::Tensor const& a,
49
+ torch::Tensor const& b,
50
+ torch::Tensor const& a_scales,
51
+ torch::Tensor const& b_scales,
52
+ torch::Tensor const& azp_adj,
53
+ c10::optional<torch::Tensor> const& azp,
54
+ c10::optional<torch::Tensor> const& bias);
55
+
56
+#if defined CUDA_VERSION && CUDA_VERSION >= 12000
57
+void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
58
+ torch::Tensor const& b,
59
+ torch::Tensor const& a_scales,
60
+ torch::Tensor const& b_scales,
61
+ torch::Tensor const& azp_adj,
62
+ c10::optional<torch::Tensor> const& azp,
63
+ c10::optional<torch::Tensor> const& bias);
64
+#endif
65
+
66
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
67
// CUTLASS FP8 kernels need at least
68
// CUDA 12.0 on SM90 systems (Hopper)
@@ -45,18 +79,20 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
79
return false;
80
}
81
48
-void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
49
- torch::Tensor const& b, torch::Tensor const& a_scales,
50
- torch::Tensor const& b_scales,
51
- c10::optional<torch::Tensor> const& bias) {
52
- int32_t major_capability;
53
- int32_t minor_capability;
82
+int32_t get_sm_version_num() {
83
+ int32_t major_capability, minor_capability;
84
cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
85
0);
86
cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
87
0);
88
int32_t version_num = major_capability * 10 + minor_capability;
89
+ return version_num;
90
+}
91
92
+void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
93
+ torch::Tensor const& b, torch::Tensor const& a_scales,
94
+ torch::Tensor const& b_scales,
95
+ c10::optional<torch::Tensor> const& bias) {
96
// Checks for conformality
97
TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
98
TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
@@ -77,7 +113,7 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
113
}
114
115
at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
80
-
116
+ int32_t version_num = get_sm_version_num();
117
if (version_num >= 90) {
118
// Hopper
119
@@ -99,3 +135,64 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
135
cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
136
}
137
}
138
+
139
+void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
140
+ torch::Tensor const& b,
141
+ torch::Tensor const& a_scales,
142
+ torch::Tensor const& b_scales,
143
+ torch::Tensor const& azp_adj,
144
+ c10::optional<torch::Tensor> const& azp,
145
+ c10::optional<torch::Tensor> const& bias) {
146
+ // Checks for conformality
147
+ TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
148
+ TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
149
+ b.size(1) == c.size(1));
150
+ TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
151
+ TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
152
+
153
+ // Check for strides and alignment
154
+ TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
155
+ TORCH_CHECK(b.stride(0) == 1); // Column-major
156
+ TORCH_CHECK(c.stride(0) % 16 == 0 &&
157
+ b.stride(1) % 16 == 0); // 16 Byte Alignment
158
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
159
+
160
+ // bias, azp, azp_adj are all 1d
161
+ // bias and azp_adj have n elements, azp has m elements
162
+ if (bias) {
163
+ TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous());
164
+ }
165
+ if (azp) {
166
+ TORCH_CHECK(azp->numel() == a.size(0) && azp->is_contiguous());
167
+ }
168
+ TORCH_CHECK(azp_adj.numel() == b.size(1) && azp_adj.is_contiguous());
169
+
170
+ // azp & bias types
171
+ TORCH_CHECK(azp_adj.dtype() == torch::kInt32);
172
+ TORCH_CHECK(!azp || azp->dtype() == torch::kInt32);
173
+ TORCH_CHECK(!bias || bias->dtype() == c.dtype(),
174
+ "currently bias dtype must match output dtype ", c.dtype());
175
+
176
+ at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
177
+ int32_t version_num = get_sm_version_num();
178
+ if (version_num >= 90) {
179
+ // Hopper
180
+
181
+ // Guard against compilation issues for sm90 kernels
182
+#if defined CUDA_VERSION && CUDA_VERSION >= 12000
183
+ cutlass_scaled_mm_azp_sm90(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
184
+#else
185
+ cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
186
+#endif
187
+ } else if (version_num == 89) {
188
+ // Ada Lovelace
189
+ cutlass_scaled_mm_azp_sm89(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
190
+ } else if (version_num >= 80) {
191
+ // Ampere
192
+ cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
193
+ } else {
194
+ // Turing
195
+ TORCH_CHECK(version_num >= 75);
196
+ cutlass_scaled_mm_azp_sm75(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
197
+ }
198
+}
\ No newline at end of file
csrc/torch_bindings.cpp
+10
-1
@@ -166,13 +166,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
166
ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
167
168
// CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
169
- // quantization.
169
+ // quantization, as well as bias
170
ops.def(
171
"cutlass_scaled_mm(Tensor! out, Tensor a,"
172
" Tensor b, Tensor a_scales,"
173
" Tensor b_scales, Tensor? bias) -> ()");
174
ops.impl("cutlass_scaled_mm", torch::kCUDA, &cutlass_scaled_mm);
175
176
+ // CUTLASS w8a8 GEMM, supporting asymmetric per-tensor or per-row/column
177
+ // quantization.
178
+ ops.def(
179
+ "cutlass_scaled_mm_azp(Tensor! out, Tensor a,"
180
+ " Tensor b, Tensor a_scales,"
181
+ " Tensor b_scales, Tensor azp_adj,"
182
+ " Tensor? azp, Tensor? bias) -> ()");
183
+ ops.impl("cutlass_scaled_mm_azp", torch::kCUDA, &cutlass_scaled_mm_azp);
184
+
185
// Check if cutlass scaled_mm is supported for CUDA devices of the given
186
// capability
187
ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
tests/kernels/test_cutlass.py
+119
-1
@@ -28,13 +28,16 @@ def to_int8(tensor: torch.Tensor):
28
return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
29
30
31
+def rand_int8(shape: tuple, device: str = "cuda"):
32
+ return to_int8(torch.rand(shape, device=device) * 255 - 128)
33
+
34
+
35
def baseline_scaled_mm(a: torch.Tensor,
36
b: torch.Tensor,
37
scale_a: torch.Tensor,
38
scale_b: torch.Tensor,
39
out_dtype: Type[torch.dtype],
40
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
37
-
41
output = (scale_a * (scale_b * (torch.mm(
42
a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
43
if bias is not None:
@@ -221,6 +224,121 @@ def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
224
use_bias)
225
226
227
+@pytest.mark.parametrize("m", [32, 64, 128])
228
+@pytest.mark.parametrize("n", [16, 32, 64])
229
+@pytest.mark.parametrize("k", [64, 128, 256])
230
+@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
231
+@pytest.mark.skip
232
+def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int,
233
+ out_dtype: torch.dtype):
234
+ # Currently, the test is failing because folding azp into
235
+ # 16-bit bias loses too much precision
236
+ scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
237
+ scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
238
+
239
+ aq_i8 = rand_int8((m, k))
240
+ bq_i8 = rand_int8((n, k)).t()
241
+
242
+ aq_i32 = aq_i8.to(dtype=torch.int32)
243
+ bq_i32 = bq_i8.to(dtype=torch.int32)
244
+
245
+ aq_f32 = aq_i8.to(dtype=torch.float32)
246
+ bq_f32 = bq_i8.to(dtype=torch.float32)
247
+
248
+ b_dq = scale_b * bq_f32
249
+
250
+ azp_a = torch.rand((1, ), device="cuda", dtype=torch.float32) * 10 + 1.5
251
+ azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
252
+ azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
253
+
254
+ a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32)
255
+ assert torch.allclose(a_dq, scale_a * aq_f32 + azp_a)
256
+
257
+ baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype)
258
+
259
+ J = torch.ones((1, k), device="cuda", dtype=torch.float32)
260
+ azp_bias = (azp_a * scale_b * (J @ bq_f32)).to(out_dtype)
261
+ assert azp_bias.shape == (1, n)
262
+ assert azp_bias[0, :].shape == (n, )
263
+
264
+ baseline_q = (scale_a.to(device='cpu') * scale_b.to(device='cpu') * (
265
+ (aq_i32 + azp_aq_i8).to(device='cpu') @ bq_i32.to(device='cpu'))).to(
266
+ dtype=out_dtype, device='cuda')
267
+
268
+ out = ops.cutlass_scaled_mm(aq_i8,
269
+ bq_i8,
270
+ scale_a,
271
+ scale_b,
272
+ out_dtype=out_dtype,
273
+ bias=azp_bias[0, :])
274
+ assert torch.allclose(out, baseline_dq, rtol=1e-2, atol=1e0)
275
+ assert torch.allclose(out, baseline_q, rtol=1e-2, atol=1e0)
276
+
277
+
278
+@pytest.mark.parametrize("m", [32, 64, 128])
279
+@pytest.mark.parametrize("n", [16, 32, 64])
280
+@pytest.mark.parametrize("k", [64, 128, 256])
281
+@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
282
+@pytest.mark.parametrize("use_bias", [True, False])
283
+@pytest.mark.parametrize("azp_per_token", [True, False])
284
+def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
285
+ use_bias: bool, azp_per_token: bool):
286
+ m_azp = m if azp_per_token else 1
287
+ scale_a = torch.randn((m_azp, 1), device="cuda", dtype=torch.float32) / 10
288
+ scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
289
+
290
+ aq_i8 = rand_int8((m, k))
291
+ aq_i32 = aq_i8.to(dtype=torch.int32)
292
+ aq_f32 = aq_i8.to(dtype=torch.float32)
293
+
294
+ bq_i8 = rand_int8((n, k)).t()
295
+ bq_i32 = bq_i8.to(dtype=torch.int32)
296
+ bq_f32 = bq_i8.to(dtype=torch.float32)
297
+ b_dq = scale_b * bq_f32
298
+
299
+ azp_a = torch.rand(
300
+ (m_azp, 1), device="cuda", dtype=torch.float32) * 10 + 1.5
301
+ azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
302
+ azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
303
+
304
+ a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32)
305
+ assert torch.allclose(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3)
306
+
307
+ if use_bias:
308
+ bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5
309
+ else:
310
+ bias = torch.zeros((1, n), device="cuda", dtype=out_dtype)
311
+
312
+ baseline_dq = (torch.mm(a_dq, b_dq) + bias).to(out_dtype)
313
+
314
+ # int32 mm not supported on CUDA
315
+ a_noazp_i32_cpu = (aq_i32 - azp_aq_i8).to(device='cpu')
316
+ cq = (a_noazp_i32_cpu @ bq_i32.to(device='cpu')).to(device='cuda')
317
+ baseline_q = (scale_a * scale_b * cq + bias).to(dtype=out_dtype)
318
+
319
+ # Hadamard is just the sum of the cols
320
+ azp_adj_i32 = bq_i32.sum(dim=0, keepdim=True, dtype=torch.int32)
321
+ azp_i32 = azp_aq_i8.to(dtype=torch.int32)
322
+ func_bias = bias if use_bias else None
323
+
324
+ if azp_per_token:
325
+ out = ops.cutlass_scaled_mm_azp(aq_i8, bq_i8, scale_a, scale_b,
326
+ out_dtype, azp_adj_i32, azp_i32,
327
+ func_bias)
328
+ else:
329
+ azp_with_adj_i32 = azp_i32 * azp_adj_i32
330
+ out = ops.cutlass_scaled_mm_azp(aq_i8, bq_i8, scale_a, scale_b,
331
+ out_dtype, azp_with_adj_i32, None,
332
+ func_bias)
333
+
334
+ # bfloat16 precision is 7-bit mantissa -> 2^-8 ~ 0.4%
335
+ # float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05%
336
+ rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3
337
+ atol = 1e-3
338
+ assert torch.allclose(out, baseline_dq, rtol=rtol, atol=atol)
339
+ assert torch.allclose(out, baseline_q, rtol=rtol, atol=atol)
340
+
341
+
342
# Test working with a subset of A and B
343
def test_cutlass_subset():
344
big_m, big_n, big_k = 1024, 1024, 1024
vllm/_custom_ops.py
+25
-1
@@ -241,6 +241,8 @@ def cutlass_scaled_mm(a: torch.Tensor,
241
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
242
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
243
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
244
+ assert bias is None or bias.shape[0] == b.shape[
245
+ 1] and bias.dtype == out_dtype
246
247
m = a.shape[0]
248
n = b.shape[1]
@@ -251,6 +253,28 @@ def cutlass_scaled_mm(a: torch.Tensor,
253
return out
254
255
256
+def cutlass_scaled_mm_azp(a: torch.Tensor,
257
+ b: torch.Tensor,
258
+ scale_a: torch.Tensor,
259
+ scale_b: torch.Tensor,
260
+ out_dtype: torch.dtype,
261
+ azp_adj: torch.Tensor,
262
+ azp: Optional[torch.Tensor] = None,
263
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
264
+ assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
265
+ assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
266
+ assert bias is None or bias.numel(
267
+ ) == b.shape[1] and bias.dtype == out_dtype
268
+
269
+ m = a.shape[0]
270
+ n = b.shape[1]
271
+ out = torch.empty((m, n), dtype=out_dtype, device=a.device)
272
+
273
+ torch.ops._C.cutlass_scaled_mm_azp(out, a, b, scale_a, scale_b, azp_adj,
274
+ azp, bias)
275
+ return out
276
+
277
+
278
# aqlm
279
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
280
codebooks: torch.Tensor, scales: torch.Tensor,
@@ -572,7 +596,7 @@ for k, v in names_and_values.items():
596
if isinstance(v, fn_type) \
597
and v.__code__.co_filename == __file__ \
598
and any(arg is torch.Tensor or arg == "torch.Tensor"
575
- for arg in v.__annotations__.values()):
599
+ for arg in v.__annotations__.values()):
600
names_and_values_to_update[k] = hint_on_error(v)
601
602
names_and_values.update(names_and_values_to_update)