@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d59dbb000
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Add per-tensor and per-token AZP epilogues (#5941)

Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>

Luka Govedič committed Aug 6, 2024 at 14:17 UTC 8d59dbb00044a588cab96bcdc028006ed922eb06
11 files changed +1175 -153
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+107 -78
@@ -32,7 +32,6 @@ def to_int8(tensor: torch.Tensor) -> torch.Tensor:
32
33 def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
34 k: int) -> Tuple[torch.Tensor, torch.Tensor]:
35 -
35 a = torch.randn((m, k), device='cuda') * 5
36 b = torch.randn((n, k), device='cuda').t() * 5
37
@@ -44,59 +43,18 @@ def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
43 raise ValueError("unsupported dtype")
44
45
47 -# impl
48 -
49 -
50 -def pytorch_mm_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
51 - scale_b: torch.Tensor,
52 - out_dtype: torch.dtype) -> torch.Tensor:
53 - return torch.mm(a, b)
54 -
55 -
56 -def pytorch_fp8_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
57 - scale_b: torch.Tensor,
58 - out_dtype: torch.dtype) -> torch.Tensor:
59 - return torch._scaled_mm(a,
60 - b,
61 - scale_a=scale_a,
62 - scale_b=scale_b,
63 - out_dtype=out_dtype)
64 -
65 -
66 -def pytorch_fp8_impl_fast_accum(a: torch.Tensor, b: torch.Tensor,
67 - scale_a: torch.Tensor, scale_b: torch.Tensor,
68 - out_dtype: torch.dtype) -> torch.Tensor:
69 - return torch._scaled_mm(a,
70 - b,
71 - scale_a=scale_a,
72 - scale_b=scale_b,
73 - out_dtype=out_dtype,
74 - use_fast_accum=True)
75 -
76 -
77 -def cutlass_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
78 - scale_b: torch.Tensor,
79 - out_dtype: torch.dtype) -> torch.Tensor:
80 - return ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=out_dtype)
81 -
82 -
46 # bench
84 -def bench_fn(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
85 - scale_b: torch.Tensor, out_dtype: torch.dtype, label: str,
86 - sub_label: str, fn: Callable, description: str) -> TMeasurement:
87 -
47 +def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
48 + **kwargs) -> TMeasurement:
49 min_run_time = 1
50
51 globals = {
91 - "a": a,
92 - "b": b,
93 - "scale_a": scale_a,
94 - "scale_b": scale_b,
95 - "out_dtype": out_dtype,
52 + "args": args,
53 + "kwargs": kwargs,
54 "fn": fn,
55 }
56 return TBenchmark.Timer(
99 - stmt="fn(a, b, scale_a, scale_b, out_dtype)",
57 + stmt="fn(*args, **kwargs)",
58 globals=globals,
59 label=label,
60 sub_label=sub_label,
@@ -110,26 +68,58 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
68 a, b = make_rand_tensors(torch.int8, m, n, k)
69 scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
70 scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
71 + bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
72 + azp = torch.zeros((m, ), device="cuda", dtype=torch.int32)
73 + azp_adj = torch.zeros((n, ), device="cuda", dtype=torch.int32)
74
75 timers = []
76 # pytorch impl - bfloat16
77 timers.append(
117 - bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
118 - b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
119 - torch.bfloat16, label, sub_label, pytorch_mm_impl,
120 - "pytorch_bf16_bf16_bf16_matmul-no-scales"))
78 + bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
79 + torch.mm, a.to(dtype=torch.bfloat16),
80 + b.to(dtype=torch.bfloat16)))
81
82 # pytorch impl - float16
83 timers.append(
124 - bench_fn(a.to(dtype=torch.float16, device="cuda"),
125 - b.to(dtype=torch.float16, device="cuda"), scale_a, scale_b,
126 - torch.float16, label, sub_label, pytorch_mm_impl,
127 - "pytorch_fp16_fp16_fp16_matmul-no-scales"))
84 + bench_fn(label, sub_label,
85 + "pytorch_fp16_fp16_fp16_matmul-no-scales", torch.mm,
86 + a.to(dtype=torch.float16), b.to(dtype=torch.float16)))
87
88 # cutlass impl
89 timers.append(
131 - bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
132 - cutlass_impl, "cutlass_i8_i8_bf16_scaled_mm"))
90 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm",
91 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
92 + torch.bfloat16))
93 +
94 + # cutlass with bias
95 + timers.append(
96 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_bias",
97 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
98 + bias))
99 +
100 + # cutlass with azp per-tensor
101 + timers.append(
102 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp",
103 + ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
104 + torch.bfloat16, azp_adj))
105 +
106 + # cutlass with azp per-tensor + bias
107 + timers.append(
108 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_bias",
109 + ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
110 + torch.bfloat16, azp_adj, None, bias))
111 +
112 + # cutlass with azp per-token
113 + timers.append(
114 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt",
115 + ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
116 + torch.bfloat16, azp_adj, azp))
117 +
118 + # cutlass with azp per-token + bias
119 + timers.append(
120 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias",
121 + ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
122 + torch.bfloat16, azp_adj, azp, bias))
123
124 return timers
125
@@ -140,46 +130,88 @@ def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
130 a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)
131 scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
132 scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
133 + bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
134
135 timers = []
136
137 # pytorch impl w. bf16
138 timers.append(
148 - bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
149 - b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
150 - torch.bfloat16, label, sub_label, pytorch_mm_impl,
151 - "pytorch_bf16_bf16_bf16_matmul-no-scales"))
139 + bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
140 + torch.mm, a.to(dtype=torch.bfloat16, device="cuda"),
141 + b.to(dtype=torch.bfloat16, device="cuda")))
142
143 # pytorch impl: bf16 output, without fp8 fast accum
144 timers.append(
155 - bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
156 - pytorch_fp8_impl, "pytorch_fp8_fp8_bf16_scaled_mm"))
145 + bench_fn(label,
146 + sub_label,
147 + "pytorch_fp8_fp8_bf16_scaled_mm",
148 + torch._scaled_mm,
149 + a,
150 + b,
151 + scale_a=scale_a,
152 + scale_b=scale_b,
153 + out_dtype=torch.bfloat16))
154
155 # pytorch impl: bf16 output, with fp8 fast accum
156 timers.append(
160 - bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
161 - pytorch_fp8_impl_fast_accum,
162 - "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum"))
157 + bench_fn(label,
158 + sub_label,
159 + "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
160 + torch._scaled_mm,
161 + a,
162 + b,
163 + scale_a=scale_a,
164 + scale_b=scale_b,
165 + out_dtype=torch.bfloat16,
166 + use_fast_accum=True))
167
168 # pytorch impl: fp16 output, without fp8 fast accum
169 timers.append(
166 - bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
167 - pytorch_fp8_impl, "pytorch_fp8_fp8_fp16_scaled_mm"))
170 + bench_fn(label,
171 + sub_label,
172 + "pytorch_fp8_fp8_fp16_scaled_mm",
173 + torch._scaled_mm,
174 + a,
175 + b,
176 + scale_a=scale_a,
177 + scale_b=scale_b,
178 + out_dtype=torch.float16))
179
180 # pytorch impl: fp16 output, with fp8 fast accum
181 timers.append(
171 - bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
172 - pytorch_fp8_impl_fast_accum,
173 - "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum"))
182 + bench_fn(label,
183 + sub_label,
184 + "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
185 + torch._scaled_mm,
186 + a,
187 + b,
188 + scale_a=scale_a,
189 + scale_b=scale_b,
190 + out_dtype=torch.float16,
191 + use_fast_accum=True))
192
193 # cutlass impl: bf16 output
194 timers.append(
177 - bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
178 - cutlass_impl, "cutlass_fp8_fp8_bf16_scaled_mm"))
195 + bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm",
196 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
197 + torch.bfloat16))
198 # cutlass impl: fp16 output
199 timers.append(
181 - bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
182 - cutlass_impl, "cutlass_fp8_fp8_fp16_scaled_mm"))
200 + bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm",
201 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16))
202 +
203 + # cutlass impl: bf16 output, with bias
204 + timers.append(
205 + bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm_bias",
206 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
207 + bias))
208 +
209 + # cutlass impl: fp16 output, with bias
210 + timers.append(
211 + bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm_bias",
212 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16,
213 + bias.to(dtype=torch.float16)))
214 +
215 return timers
216
217
@@ -200,7 +232,6 @@ def print_timers(timers: Iterable[TMeasurement]):
232
233 def run(dtype: torch.dtype,
234 MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
203 -
235 results = []
236 for m, k, n in MKNs:
237 timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
@@ -216,7 +247,6 @@ def make_output(data: Iterable[TMeasurement],
247 MKNs: Iterable[Tuple[int, int, int]],
248 base_description: str,
249 timestamp=None):
219 -
250 print(f"== All Results {base_description} ====")
251 print_timers(data)
252
@@ -251,7 +281,6 @@ def run_range_bench(args):
281
282
283 def run_model_bench(args):
254 -
284 print("Benchmarking models:")
285 for i, model in enumerate(args.models):
286 print(f"[{i}] {model}")
csrc/ops.h
+8
@@ -128,6 +128,14 @@ void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
128 torch::Tensor const& b_scales,
129 c10::optional<torch::Tensor> const& bias);
130
131 +void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
132 + torch::Tensor const& b,
133 + torch::Tensor const& a_scales,
134 + torch::Tensor const& b_scales,
135 + torch::Tensor const& azp_adj,
136 + c10::optional<torch::Tensor> const& azp,
137 + c10::optional<torch::Tensor> const& bias);
138 +
139 torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
140 torch::Tensor const& b_q_weight,
141 torch::Tensor const& s_tok,
csrc/quantization/cutlass_w8a8/Epilogues.md new
+147
@@ -0,0 +1,147 @@
1 +# CUTLASS Epilogues
2 +
3 +## Introduction
4 +This document describes the various CUTLASS epilogues implemented for fusing de-quantization operations onto GEMMs.
5 +
6 +Currently, we only support symmetric quantization for weights,
7 +and symmetric and asymmetric quantization for activations.
8 +Both can be quantized per-tensor or per-channel (weights) / per-token (activations).
9 +
10 +There are 4 epilogues:
11 +1. ScaledEpilogue: symmetric quantization for activations, no bias.
12 +1. ScaledEpilogueBias: symmetric quantization for activations, supports bias.
13 +1. ScaledEpilogueAzp: asymmetric per-tensor quantization for activations, supports bias.
14 +1. ScaledEpilogueAzpPerToken: asymmetric per-token quantization for activations, supports bias.
15 +
16 +We do not have epilogues for asymmetric quantization of activations without bias in order to reduce final binary size.
17 +Instead, if no bias is passed, the epilogue will use 0 as the bias.
18 +That induces a redundant addition operation (and runtime check), but the performance impact is minor.
19 +
20 +## Underlying Linear Algebra
21 +
22 +More details available in the [Activation Quantization RFC](https://github.com/vllm-project/vllm/issues/3975).
23 +
24 +If $` \widehat X `$ is the quantized $` X `$, our matrices become the following
25 +
26 +```math
27 +A = s_a (\widehat A - J_a z_a)
28 +```
29 +```math
30 +B = s_b \widehat B
31 +```
32 +```math
33 +D = A B + C
34 +```
35 +```math
36 +D = s_a s_b \widehat D + C
37 +```
38 +
39 +Here, D is the output of the GEMM, and C is the bias.
40 +A is the activations and supports asymmetric quantization,
41 +and B is the weights and only supports symmetric quantization.
42 +$ s_a $ and $s_b$ are the scales for activations and weights, respectively.
43 +$ z_a $ is the zero-point for activations, and $ J_a $ is the matrix of all ones with dimensions of A.
44 +Additional epilogues would be required to support asymmetric quantization for weights.
45 +
46 +Expanding further, we can calculate $` \widehat D `$ as follows:
47 +
48 +```math
49 +A B = s_a ( \widehat A - J_a z_a ) s_b \widehat B
50 +```
51 +```math
52 +A B = s_a s_b \left( \widehat A \widehat B - J_a z_a \widehat B \right)
53 +```
54 +```math
55 +\widehat D = \widehat A \widehat B - z_a J_a \widehat B
56 +```
57 +
58 +Note that $` \widehat A \widehat B `$ is the raw output of the GEMM,
59 +and $` J_a \widehat B `$ is known ahead of time.
60 +Each row of it is equal to $` \mathbf 1 \widehat B `$, which is a row-vector of column sums of $` \widehat B `$.
61 +
62 +## Epilogues
63 +
64 +### ScaledEpilogue
65 +This epilogue computes the symmetric quantization for activations without bias, meaning $` C = 0 `$ and $` z_a = 0 `$.
66 +The output of the GEMM is:
67 +
68 +```math
69 +\widehat D = \widehat A \widehat B
70 +```
71 +```math
72 +D = s_a s_b \widehat D
73 +```
74 +```math
75 +D = s_a s_b \widehat A \widehat B
76 +```
77 +
78 +Epilogue parameters:
79 +- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
80 +- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
81 +
82 +### ScaledEpilogueBias
83 +This epilogue computes the symmetric quantization for activations with bias, meaning $` z_a = 0 `$.
84 +The output of the GEMM is:
85 +
86 +```math
87 +\widehat D = \widehat A \widehat B
88 +```
89 +```math
90 +D = s_a s_b \widehat D + C
91 +```
92 +```math
93 +D = s_a s_b \widehat A \widehat B + C
94 +```
95 +
96 +
97 +Epilogue parameters:
98 +- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
99 +- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
100 +- `bias` is the bias, is always per-channel (row-vector).
101 +
102 +### ScaledEpilogueAzp
103 +This epilogue computes the asymmetric per-tensor quantization for activations with bias.
104 +The output of the GEMM is:
105 +
106 +```math
107 +\widehat D = \widehat A \widehat B - z_a J_a \widehat B
108 +```
109 +```math
110 +D = s_a s_b \widehat D + C
111 +```
112 +```math
113 +D = s_a s_b \left( \widehat A \widehat B - z_a J_a \widehat B \right) + C
114 +```
115 +
116 +Because $` z_a `$ is a scalar, the zero-point term $` z_a J_a \widehat B `$ has every row equal to $` z_a \mathbf 1 B `$.
117 +That is precomputed and stored in `azp_with_adj` as a row-vector.
118 +
119 +Epilogue parameters:
120 +- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
121 + - Generally this will be per-tensor as the zero-points are per-tensor.
122 +- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
123 +- `azp_with_adj` is the precomputed zero-point term ($` z_a J_a \widehat B `$), is per-channel (row-vector).
124 +- `bias` is the bias, is always per-channel (row-vector).
125 +
126 +To use these kernels efficiently, users must precompute the `azp_with_adj` term offline and pass it to the kernel.
127 +
128 +### ScaledEpilogueAzpPerToken
129 +This epilogue computes the asymmetric per-token quantization for activations with bias.
130 +
131 +The output of the GEMM is the same as above, but the $` z_a `$ is a column-vector.
132 +That means the zero-point term $` z_a J_a \widehat B `$ becomes an outer product of $` z_a `$ and $` \mathbf 1 \widehat B `$.
133 +
134 +Epilogue parameters:
135 +- `scale_a` is the scale for activations, can be per-tensor (scalar) or per-token (column-vector).
136 + - Generally this will be per-token as the zero-points are per-token.
137 +- `scale_b` is the scale for weights, can be per-tensor (scalar) or per-channel (row-vector).
138 +- `azp_adj` is the precomputed zero-point adjustment term ($` \mathbf 1 \widehat B `$), is per-channel (row-vector).
139 +- `azp` is the zero-point (`z_a`), is per-token (column-vector).
140 +- `bias` is the bias, is always per-channel (row-vector).
141 +
142 +To use these kernels efficiently, users must precompute the `azp_adj` term offline and pass it to the kernel.
143 +
144 +The epilogue performs the following computation (where `Dq` is the raw quantized output of the GEMM):
145 +```
146 +out = scale_a * scale_b * (Dq - azp_adj * azp) + bias
147 +```
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c2x.hpp
+151 -1
@@ -207,6 +207,156 @@ struct VisitorRowOrScalarBroadcast {
207
208 };
209
210 +/////////////////////////////////////////////////////////////////////////////////////////////////
211 +
212 +// This is a modified RowBroadcast that will broadcast 0 if ptr_row is null
213 +template<
214 + class ThreadMap,
215 + class Element,
216 + class StrideMNL
217 +>
218 +struct VisitorRowOrZeroBroadcast {
219 +
220 + // This struct has been modified to remove null_default (because it's always 0)
221 + struct Arguments {
222 + Element const* ptr_row = nullptr;
223 + StrideMNL dRow = {};
224 + };
225 +
226 + using Params = Arguments;
227 +
228 + template <class ProblemShape>
229 + static constexpr Params
230 + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
231 + return args;
232 + }
233 +
234 + template <class ProblemShape>
235 + static size_t
236 + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
237 + return 0;
238 + }
239 +
240 + struct SharedStorage {};
241 +
242 + // Global load type
243 + static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
244 + using VecType = uint_bit_t<cute::min(128, vec_bits)>;
245 + static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
246 +
247 + CUTLASS_HOST_DEVICE
248 + VisitorRowOrZeroBroadcast() { }
249 +
250 + CUTLASS_HOST_DEVICE
251 + VisitorRowOrZeroBroadcast(Params const& params, SharedStorage const& shared_storage)
252 + : params_ptr(&params) { }
253 +
254 + Params const* params_ptr;
255 +
256 + template <class GTensor, class RTensor, class CTensor, class ProblemShape>
257 + struct Callbacks : EmptyCallbacks {
258 + CUTLASS_DEVICE
259 + Callbacks(
260 + GTensor&& tC_gRow,
261 + RTensor&& tC_rRow,
262 + CTensor&& tC_cRow,
263 + ProblemShape problem_shape,
264 + Params const* params_ptr
265 + ):
266 + tC_gRow(cute::forward<GTensor>(tC_gRow)),
267 + tC_rRow(cute::forward<RTensor>(tC_rRow)),
268 + tC_cRow(cute::forward<CTensor>(tC_cRow)),
269 + n(get<1>(problem_shape)),
270 + params_ptr(params_ptr) { }
271 +
272 + GTensor tC_gRow;
273 + RTensor tC_rRow;
274 + CTensor tC_cRow;
275 + Params const* params_ptr;
276 + int n;
277 +
278 + // This function is modified from VisitorRowBroadcast
279 + CUTLASS_DEVICE void
280 + begin_epilogue() {
281 + clear(tC_rRow);
282 + auto src_v = filter(tC_gRow);
283 + auto coord_v = filter(tC_cRow);
284 + auto dst_v = filter(tC_rRow);
285 +
286 + if (params_ptr->ptr_row != nullptr) {
287 + // In this case we are loading from a row vector and broadcasting
288 + CUTLASS_PRAGMA_UNROLL
289 + for (int i = 0; i < size(src_v); ++i) {
290 + bool guard = get<1>(coord_v(i)) < n;
291 + cutlass::arch::global_load<VecType, sizeof(VecType)>(
292 + dst_v(i), (void const*)&src_v(i), guard);
293 + }
294 + } else {
295 + // In this case we are broadcasting 0
296 + VecType filled_vec;
297 + CUTLASS_PRAGMA_UNROLL
298 + for (int i = 0; i < VecLength; i++) {
299 + reinterpret_cast<Element*>(&filled_vec)[i] = Element{0};
300 + }
301 +
302 + CUTLASS_PRAGMA_UNROLL
303 + for (int i = 0; i < size(src_v); ++i) {
304 + if (get<1>(coord_v(i)) < n) {
305 + dst_v(i) = filled_vec;
306 + }
307 + }
308 + }
309 + }
310 +
311 + template <class ElementAccumulator, int FragmentSize>
312 + CUTLASS_DEVICE auto // returns an Array
313 + visit(int iter_idx, int row_idx, int column_idx, int frg_idx,
314 + Array<ElementAccumulator, FragmentSize> const& frg_acc) {
315 + Tensor rRow_frg = recast<Array<Element, FragmentSize>>(coalesce(tC_rRow));
316 + return rRow_frg(column_idx);
317 + }
318 + };
319 +
320 + template <class ProblemShape>
321 + CUTLASS_DEVICE auto
322 + get_callbacks(
323 + gemm::GemmCoord threadblock_tile_offset,
324 + int thread_idx,
325 + ProblemShape problem_shape
326 + ) {
327 + Tensor mRow = make_tensor(
328 + make_gmem_ptr(params_ptr->ptr_row),
329 + problem_shape,
330 + params_ptr->dRow);
331 +
332 + // VECTOR, FRAGMENT_COLUMN
333 + Tensor tC_gRow = recast<VecType>(
334 + ThreadMap::partition(mRow, thread_idx, threadblock_tile_offset)
335 + )(_,_,_0{},_0{},_0{},_0{});
336 + Tensor tC_rRow = make_tensor_like(tC_gRow);
337 +
338 + // Generate the pred tensor
339 + Tensor cRow = make_identity_tensor(mRow.shape());
340 + Tensor tC_cRow = outer_partition(
341 + ThreadMap::partition(cRow, thread_idx, threadblock_tile_offset)(_,_,_0{},_0{},_0{},_0{}),
342 + Shape<Int<VecLength>>{},
343 + (_0{})
344 + );
345 +
346 + return Callbacks<
347 + decltype(tC_gRow), decltype(tC_rRow),
348 + decltype(tC_cRow), ProblemShape>(
349 + cute::move(tC_gRow),
350 + cute::move(tC_rRow),
351 + cute::move(tC_cRow),
352 + problem_shape,
353 + params_ptr
354 + );
355 + }
356 +
357 +};
358 +
359 +
360 /////////////////////////////////////////////////////////////////////////////////////////////////
361
362 // Column vector broadcast
@@ -217,7 +367,7 @@ template<
367 >
368 struct VisitorColOrScalarBroadcast {
369
220 - // This struct has been modified to have a bool indicating that ptr_col is a
370 + // This struct has been modified to have a bool indicating that ptr_col is a
371 // scalar that must be broadcast.
372 struct Arguments {
373 Element const* ptr_col = nullptr;
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+57
@@ -50,6 +50,25 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
50 }
51 }
52
53 +void cutlass_scaled_mm_azp_sm75(torch::Tensor& out, torch::Tensor const& a,
54 + torch::Tensor const& b,
55 + torch::Tensor const& a_scales,
56 + torch::Tensor const& b_scales,
57 + torch::Tensor const& azp_adj,
58 + c10::optional<torch::Tensor> const& azp,
59 + c10::optional<torch::Tensor> const& bias) {
60 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
61 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
62 +
63 + if (azp) {
64 + return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
65 + out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
66 + } else {
67 + return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzp>(
68 + out, a, b, a_scales, b_scales, azp_adj, bias);
69 + }
70 +}
71 +
72 template <template <typename, typename> typename Epilogue,
73 typename... EpilogueArgs>
74 void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
@@ -87,6 +106,25 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
106 }
107 }
108
109 +void cutlass_scaled_mm_azp_sm80(torch::Tensor& out, torch::Tensor const& a,
110 + torch::Tensor const& b,
111 + torch::Tensor const& a_scales,
112 + torch::Tensor const& b_scales,
113 + torch::Tensor const& azp_adj,
114 + c10::optional<torch::Tensor> const& azp,
115 + c10::optional<torch::Tensor> const& bias) {
116 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
117 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
118 +
119 + if (azp) {
120 + return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
121 + out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
122 + } else {
123 + return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzp>(
124 + out, a, b, a_scales, b_scales, azp_adj, bias);
125 + }
126 +}
127 +
128 template <template <typename, typename> typename Epilogue,
129 typename... EpilogueArgs>
130 void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
@@ -139,3 +177,22 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
177 out, a, b, a_scales, b_scales);
178 }
179 }
180 +
181 +void cutlass_scaled_mm_azp_sm89(torch::Tensor& out, torch::Tensor const& a,
182 + torch::Tensor const& b,
183 + torch::Tensor const& a_scales,
184 + torch::Tensor const& b_scales,
185 + torch::Tensor const& azp_adj,
186 + c10::optional<torch::Tensor> const& azp,
187 + c10::optional<torch::Tensor> const& bias) {
188 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
189 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
190 +
191 + if (azp) {
192 + return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
193 + out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
194 + } else {
195 + return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzp>(
196 + out, a, b, a_scales, b_scales, azp_adj, bias);
197 + }
198 +}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+217 -36
@@ -73,19 +73,63 @@ struct enable_sm89_to_sm90 : Kernel {
73 };
74
75 /*
76 - * This class provides the common ScaleA and ScaleB descriptors for the
77 - * ScaledEpilogue and ScaledEpilogueBias classes.
76 + * This class provides the common load descriptors for the
77 + * ScaledEpilogue[...] classes
78 */
79 template <typename ElementD, typename OutputTileThreadMap>
80 struct ScaledEpilogueBase {
81 protected:
82 using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
83
84 - using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
85 - OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
84 + template <typename T>
85 + using ColOrScalarLoad =
86 + cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
87 + OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
88 +
89 + template <typename T>
90 + using RowOrScalarLoad =
91 + cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92 + OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
93 +
94 + template <typename T>
95 + using ColLoad = cutlass::epilogue::threadblock::VisitorColBroadcast<
96 + OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
97 +
98 + template <typename T>
99 + using RowLoad = cutlass::epilogue::threadblock::VisitorRowBroadcast<
100 + OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
101 +
102 + template <typename T>
103 + using RowOrZeroLoad =
104 + cutlass::epilogue::threadblock::VisitorRowOrZeroBroadcast<
105 + OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
106 +
107 + // This utility function constructs the arguments for the load descriptors
108 + // from a tensor. It can handle both row and column, as well as row/column or
109 + // scalar cases.
110 + template <typename Descriptor, typename T>
111 + static auto args_from_tensor(torch::Tensor const& tensor) {
112 + using Arguments = typename Descriptor::Arguments;
113 + auto* data_ptr = static_cast<T*>(tensor.data_ptr());
114 + if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
115 + std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
116 + return Arguments{data_ptr, tensor.numel() != 1};
117 + } else {
118 + // it would technically work but no use case as data_ptr is never nullptr
119 + static_assert(!std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
120 + return Arguments{data_ptr};
121 + }
122 + }
123
87 - using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
88 - OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
124 + // This overload handles the case where there might not be a tensor, in which
125 + // case a nullptr is passed and a constant (0) is used.
126 + template <typename Descriptor, typename T>
127 + static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
128 + static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
129 + using Arguments = typename Descriptor::Arguments;
130 + auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
131 + return Arguments{data_ptr};
132 + }
133 };
134
135 /*
@@ -110,8 +154,8 @@ struct ScaledEpilogue
154 private:
155 using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156 using Accum = typename SUPER::Accum;
113 - using ScaleA = typename SUPER::ScaleA;
114 - using ScaleB = typename SUPER::ScaleB;
157 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
158 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
159
160 using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161 cutlass::multiplies, float, float,
@@ -131,28 +175,32 @@ struct ScaledEpilogue
175
176 static ArgumentType prepare_args(torch::Tensor const& a_scales,
177 torch::Tensor const& b_scales) {
134 - using ScaleAArgs = typename ScaleA::Arguments;
135 - using ScaleBArgs = typename ScaleB::Arguments;
136 -
137 - ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
138 - ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
178 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
179 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
180
140 - typename EVTCompute0::Arguments evt0_compute_args{b_args};
141 -
142 - typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
143 - return evt_compute_args;
181 + typename EVTCompute0::Arguments evt0_args{b_args};
182 + return ArgumentType{a_args, evt0_args};
183 }
184 };
185
186 +/*
187 + * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
188 + * This bias can also be used in the per-tensor azp case, where the activation
189 + * zero point (azp) is used to compute an azp correction term,
190 + * which is folded into the bias.
191 + *
192 + * The bias tensor must be per-output channel.
193 + * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
194 + */
195 template <typename ElementD, typename OutputTileThreadMap>
196 struct ScaledEpilogueBias
149 - : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
150 - private:
197 + : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
198 + protected:
199 using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
200 using Accum = typename SUPER::Accum;
153 - using ScaleA = typename SUPER::ScaleA;
154 - using ScaleB = typename SUPER::ScaleB;
155 -
201 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
202 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
203 + using Bias = typename SUPER::template RowLoad<ElementD>;
204 using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
205 cutlass::multiplies, float, float,
206 cutlass::FloatRoundStyle::round_to_nearest>;
@@ -164,30 +212,163 @@ struct ScaledEpilogueBias
212 cutlass::multiply_add, ElementD, float,
213 cutlass::FloatRoundStyle::round_to_nearest>;
214
167 - using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
168 - OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
169 -
215 public:
216 using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
217 EVTCompute0, Bias>;
218 using ArgumentType = typename EVTCompute::Arguments;
174 -
219 static ArgumentType prepare_args(torch::Tensor const& a_scales,
220 torch::Tensor const& b_scales,
221 torch::Tensor const& bias) {
178 - using ScaleAArgs = typename ScaleA::Arguments;
179 - using ScaleBArgs = typename ScaleB::Arguments;
180 - using BiasArgs = typename Bias::Arguments;
222 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
223 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
224 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
225 +
226 + typename EVTCompute0::Arguments evt0_args{b_args};
227 + return ArgumentType{a_args, evt0_args, bias_args};
228 + }
229 +};
230 +
231 +/*
232 + * This epilogue directly supports per-tensor azp in int32 form.
233 + * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
234 + * term, which should already be multiplied with the scalar azp.
235 + * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
236 + *
237 + * This epilogue also supports bias, which remains per-channel.
238 + */
239 +template <typename ElementD, typename OutputTileThreadMap>
240 +struct ScaledEpilogueBiasAzp
241 + : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
242 + private:
243 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
244 + using Accum = typename SUPER::Accum;
245 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
246 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
247 + using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
248 +
249 + // This is the full AZP term, azp * J @ B, shape (1,n)
250 + using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
251 +
252 + // Compute float(accum - azp_adj), both operands are int32_t
253 + using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
254 + cutlass::minus, float, int32_t,
255 + cutlass::FloatRoundStyle::round_to_nearest>;
256 +
257 + using EVTComputeAzp =
258 + cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Accum, AzpWithAdj>;
259 +
260 + using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
261 + cutlass::multiplies, float, float,
262 + cutlass::FloatRoundStyle::round_to_nearest>;
263 +
264 + using EVTComputeScaleB =
265 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
266 + EVTComputeAzp>;
267 +
268 + using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
269 + cutlass::multiply_add, ElementD, float,
270 + cutlass::FloatRoundStyle::round_to_nearest>;
271 +
272 + public:
273 + using EVTCompute =
274 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
275 + EVTComputeScaleB, Bias>;
276 +
277 + using ArgumentType = typename EVTCompute::Arguments;
278 +
279 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
280 + torch::Tensor const& b_scales,
281 + torch::Tensor const& azp_adj,
282 + c10::optional<torch::Tensor> const& bias) {
283 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
284 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
285 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
286 + auto azp_adj_args =
287 + SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
288 +
289 + typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
290 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
291 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
292 + }
293 +};
294 +
295 +/*
296 + * This epilogue supports per-token azp by computing and applying
297 + * the correction term using a rank-1 update. If the term were materialized,
298 + * it would require O(m*n) space, and this way it only requires O(m+n) space.
299 + * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
300 + * point for each row of A.
301 + * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
302 + *
303 + * This epilogue also supports bias, which remains per-channel.
304 + */
305 +template <typename ElementD, typename OutputTileThreadMap>
306 +struct ScaledEpilogueBiasAzpToken
307 + : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
308 + private:
309 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
310 + using Accum = typename SUPER::Accum;
311 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
312 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
313 + using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
314 +
315 + // Per-token azp term, shape (m,1)
316 + using Azp = typename SUPER::template ColLoad<int32_t>;
317
182 - ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
183 - ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
184 - BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
318 + // This is the AZP adjustment term, J @ B, shape (1,n)
319 + using AzpAdj = typename SUPER::template RowLoad<int32_t>;
320 +
321 + // Compute azp * azp_adj
322 + using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
323 + cutlass::multiplies, int32_t, int32_t,
324 + cutlass::FloatRoundStyle::round_to_nearest>;
325
186 - typename EVTCompute0::Arguments evt0_compute_args{b_args};
326 + using EVTComputeAzp =
327 + cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Azp, AzpAdj>;
328
188 - typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
189 - bias_args};
190 - return evt_compute_args;
329 + // Compute float(accum - azp*azp_adj), all operands are int32_t
330 + using ComputeAcc = cutlass::epilogue::threadblock::VisitorCompute<
331 + cutlass::minus, float, int32_t,
332 + cutlass::FloatRoundStyle::round_to_nearest>;
333 +
334 + using EVTComputeAcc =
335 + cutlass::epilogue::threadblock::Sm80EVT<ComputeAcc, Accum, EVTComputeAzp>;
336 +
337 + using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
338 + cutlass::multiplies, float, float,
339 + cutlass::FloatRoundStyle::round_to_nearest>;
340 +
341 + using EVTComputeScaleB =
342 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
343 + EVTComputeAcc>;
344 +
345 + using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
346 + cutlass::multiply_add, ElementD, float,
347 + cutlass::FloatRoundStyle::round_to_nearest>;
348 +
349 + public:
350 + using EVTCompute =
351 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
352 + EVTComputeScaleB, Bias>;
353 +
354 + using ArgumentType = typename EVTCompute::Arguments;
355 +
356 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
357 + torch::Tensor const& b_scales,
358 + torch::Tensor const& azp_adj,
359 + torch::Tensor const& azp,
360 + c10::optional<torch::Tensor> const& bias) {
361 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
362 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
363 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
364 + auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
365 + auto azp_adj_args =
366 + SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
367 +
368 + typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
369 + typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
370 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
371 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
372 }
373 };
374
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+230 -28
@@ -58,21 +58,63 @@ struct enable_sm90_or_later : Kernel {
58 };
59
60 /*
61 - * This class provides the common ScaleA and ScaleB descriptors for the
62 - * ScaledEpilogue and ScaledEpilogueBias classes.
61 + * This class provides the common load descriptors for the
62 + * ScaledEpilogue[...] classes
63 */
64 template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
65 struct ScaledEpilogueBase {
66 protected:
67 using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
68
69 - using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
70 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
69 + template <typename T>
70 + using ColOrScalarLoad = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
71 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
72 Stride<Int<1>, Int<0>, Int<0>>>;
73
73 - using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
74 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
74 + template <typename T>
75 + using RowOrScalarLoad = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
76 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
77 Stride<Int<0>, Int<1>, Int<0>>>;
78 +
79 + // Don't want to support nullptr by default
80 + template <typename T, bool EnableNullPtr = false>
81 + using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
82 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
83 + Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
84 +
85 + // Don't want to support nullptr by default
86 + template <typename T, bool EnableNullPtr = false>
87 + using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
88 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
89 + Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
90 +
91 + // This utility function constructs the arguments for the load descriptors
92 + // from a tensor. It can handle both row and column, as well as row/column or
93 + // scalar cases.
94 + template <typename Descriptor, typename T>
95 + static auto args_from_tensor(torch::Tensor const& tensor) {
96 + using Arguments = typename Descriptor::Arguments;
97 + auto* data_ptr = static_cast<T*>(tensor.data_ptr());
98 + if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
99 + std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
100 + return Arguments{data_ptr, tensor.numel() != 1};
101 + } else {
102 + static_assert(!std::is_same_v<Descriptor, ColLoad<T, true>> &&
103 + !std::is_same_v<Descriptor, RowLoad<T, true>>);
104 + return Arguments{data_ptr};
105 + }
106 + }
107 +
108 + // This overload handles the case where there might not be a tensor, in which
109 + // case a nullptr is passed and a constant (0) is used.
110 + template <typename Descriptor, typename T>
111 + static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
112 + using Arguments = typename Descriptor::Arguments;
113 + auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
114 + static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
115 + std::is_same_v<Descriptor, RowLoad<T, true>>);
116 + return Arguments{data_ptr};
117 + }
118 };
119
120 /*
@@ -97,8 +139,8 @@ struct ScaledEpilogue
139 private:
140 using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
141 using Accum = typename SUPER::Accum;
100 - using ScaleA = typename SUPER::ScaleA;
101 - using ScaleB = typename SUPER::ScaleB;
142 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
143 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
144
145 using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
146 cutlass::multiplies, float, float,
@@ -118,24 +160,32 @@ struct ScaledEpilogue
160
161 static ArgumentType prepare_args(torch::Tensor const& a_scales,
162 torch::Tensor const& b_scales) {
121 - using ScaleA_Args = typename ScaleA::Arguments;
122 - using ScaleB_Args = typename ScaleB::Arguments;
123 -
124 - ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
125 - ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
163 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
164 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
165
127 - return ArgumentType{a_args, {b_args}};
166 + typename EVTCompute0::Arguments evt0_args{b_args};
167 + return ArgumentType{a_args, evt0_args};
168 }
169 };
170
171 +/*
172 + * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
173 + * This bias can also be used in the per-tensor azp case, where the activation
174 + * zero point (azp) is used to compute an azp correction term,
175 + * which is folded into the bias.
176 + *
177 + * The bias tensor must be per-output channel.
178 + * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
179 + */
180 template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
181 struct ScaledEpilogueBias
182 : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
183 private:
184 using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
185 using Accum = typename SUPER::Accum;
137 - using ScaleA = typename SUPER::ScaleA;
138 - using ScaleB = typename SUPER::ScaleB;
186 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
187 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
188 + using Bias = typename SUPER::template RowLoad<ElementD>;
189
190 using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
191 cutlass::multiplies, float, float,
@@ -148,27 +198,160 @@ struct ScaledEpilogueBias
198 cutlass::multiply_add, ElementD, float,
199 cutlass::FloatRoundStyle::round_to_nearest>;
200
151 - using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast<
152 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, ElementD,
153 - Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<ElementD>, false>;
154 -
201 public:
202 using EVTCompute =
203 cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
158 - using ArgumentType = typename EVTCompute::Arguments;
204
205 + using ArgumentType = typename EVTCompute::Arguments;
206 static ArgumentType prepare_args(torch::Tensor const& a_scales,
207 torch::Tensor const& b_scales,
208 torch::Tensor const& bias) {
163 - using ScaleA_Args = typename ScaleA::Arguments;
164 - using ScaleB_Args = typename ScaleB::Arguments;
165 - using Bias_Args = typename Bias::Arguments;
209 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
210 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
211 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
212 +
213 + typename EVTCompute0::Arguments evt0_args{b_args};
214 + return ArgumentType{a_args, evt0_args, bias_args};
215 + }
216 +};
217 +
218 +/*
219 + * This epilogue directly supports per-tensor azp in int32 form.
220 + * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
221 + * term, which should already be multiplied with the scalar azp.
222 + * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
223 + *
224 + * This epilogue also supports bias, which remains per-channel.
225 + */
226 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
227 +struct ScaledEpilogueBiasAzp
228 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
229 + private:
230 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
231 + using Accum = typename SUPER::Accum;
232 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
233 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
234 + using Bias = typename SUPER::template RowLoad<ElementD, true>;
235 +
236 + // This is the full AZP term, azp * J @ B, shape (1,n)
237 + using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
238 +
239 + // Compute float(accum - azp_adj), both operands are int32_t
240 + using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
241 + cutlass::minus, float, int32_t,
242 + cutlass::FloatRoundStyle::round_to_nearest>;
243 +
244 + using EVTComputeAzp =
245 + cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Accum, AzpWithAdj>;
246 +
247 + using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
248 + cutlass::multiplies, float, float,
249 + cutlass::FloatRoundStyle::round_to_nearest>;
250 +
251 + using EVTComputeScaleB =
252 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAzp>;
253 +
254 + using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
255 + cutlass::multiply_add, ElementD, float,
256 + cutlass::FloatRoundStyle::round_to_nearest>;
257 +
258 + public:
259 + using EVTCompute =
260 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
261 + EVTComputeScaleB, Bias>;
262 + using ArgumentType = typename EVTCompute::Arguments;
263 +
264 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
265 + torch::Tensor const& b_scales,
266 + torch::Tensor const& azp_adj,
267 + c10::optional<torch::Tensor> const& bias) {
268 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
269 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
270 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
271 + auto azp_adj_args =
272 + SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
273 +
274 + typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
275 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
276 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
277 + }
278 +};
279 +
280 +/*
281 + * This epilogue supports per-token azp by computing and applying
282 + * the correction term using a rank-1 update. If the term were materialized,
283 + * it would require O(m*n) space, and this way it only requires O(m+n) space.
284 + * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
285 + * point for each row of A.
286 + * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
287 + *
288 + * This epilogue also supports bias, which remains per-channel.
289 + */
290 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
291 +struct ScaledEpilogueBiasAzpToken
292 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
293 + private:
294 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
295 + using Accum = typename SUPER::Accum;
296 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
297 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
298 + using Bias = typename SUPER::template RowLoad<ElementD, true>;
299 +
300 + // Per-token azp term, shape (m,1)
301 + using Azp = typename SUPER::template ColLoad<int32_t>;
302 +
303 + // This is the AZP adjustment term, J @ B, shape (1,n)
304 + using AzpAdj = typename SUPER::template RowLoad<int32_t>;
305 +
306 + // Compute azp * azp_adj
307 + using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
308 + cutlass::multiplies, int32_t, int32_t,
309 + cutlass::FloatRoundStyle::round_to_nearest>;
310 +
311 + using EVTComputeAzp =
312 + cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Azp, AzpAdj>;
313
167 - ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
168 - ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
169 - Bias_Args bias_args{static_cast<ElementD*>(bias.data_ptr())};
314 + // Compute float(accum - azp*azp_adj), all operands are int32_t
315 + using ComputeAcc = cutlass::epilogue::fusion::Sm90Compute<
316 + cutlass::minus, float, int32_t,
317 + cutlass::FloatRoundStyle::round_to_nearest>;
318
171 - return ArgumentType{a_args, {b_args}, bias_args};
319 + using EVTComputeAcc =
320 + cutlass::epilogue::fusion::Sm90EVT<ComputeAcc, Accum, EVTComputeAzp>;
321 +
322 + using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
323 + cutlass::multiplies, float, float,
324 + cutlass::FloatRoundStyle::round_to_nearest>;
325 +
326 + using EVTComputeScaleB =
327 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAcc>;
328 +
329 + using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
330 + cutlass::multiply_add, ElementD, float,
331 + cutlass::FloatRoundStyle::round_to_nearest>;
332 +
333 + public:
334 + using EVTCompute =
335 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
336 + EVTComputeScaleB, Bias>;
337 + using ArgumentType = typename EVTCompute::Arguments;
338 +
339 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
340 + torch::Tensor const& b_scales,
341 + torch::Tensor const& azp_adj,
342 + torch::Tensor const& azp,
343 + c10::optional<torch::Tensor> const& bias) {
344 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
345 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
346 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
347 + auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
348 + auto azp_adj_args =
349 + SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
350 +
351 + typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
352 + typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
353 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
354 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
355 }
356 };
357
@@ -546,4 +729,23 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
729 }
730 }
731
732 +void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
733 + torch::Tensor const& b,
734 + torch::Tensor const& a_scales,
735 + torch::Tensor const& b_scales,
736 + torch::Tensor const& azp_adj,
737 + c10::optional<torch::Tensor> const& azp,
738 + c10::optional<torch::Tensor> const& bias) {
739 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
740 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
741 +
742 + if (azp) {
743 + return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzpToken>(
744 + out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
745 + } else {
746 + return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzp>(
747 + out, a, b, a_scales, b_scales, azp_adj, bias);
748 + }
749 +}
750 +
751 #endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+104 -7
@@ -29,6 +29,40 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
29 c10::optional<torch::Tensor> const& bias);
30 #endif
31
32 +void cutlass_scaled_mm_azp_sm75(torch::Tensor& c, torch::Tensor const& a,
33 + torch::Tensor const& b,
34 + torch::Tensor const& a_scales,
35 + torch::Tensor const& b_scales,
36 + torch::Tensor const& azp_adj,
37 + c10::optional<torch::Tensor> const& azp,
38 + c10::optional<torch::Tensor> const& bias);
39 +
40 +void cutlass_scaled_mm_azp_sm80(torch::Tensor& c, torch::Tensor const& a,
41 + torch::Tensor const& b,
42 + torch::Tensor const& a_scales,
43 + torch::Tensor const& b_scales,
44 + torch::Tensor const& azp_adj,
45 + c10::optional<torch::Tensor> const& azp,
46 + c10::optional<torch::Tensor> const& bias);
47 +
48 +void cutlass_scaled_mm_azp_sm89(torch::Tensor& c, torch::Tensor const& a,
49 + torch::Tensor const& b,
50 + torch::Tensor const& a_scales,
51 + torch::Tensor const& b_scales,
52 + torch::Tensor const& azp_adj,
53 + c10::optional<torch::Tensor> const& azp,
54 + c10::optional<torch::Tensor> const& bias);
55 +
56 +#if defined CUDA_VERSION && CUDA_VERSION >= 12000
57 +void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
58 + torch::Tensor const& b,
59 + torch::Tensor const& a_scales,
60 + torch::Tensor const& b_scales,
61 + torch::Tensor const& azp_adj,
62 + c10::optional<torch::Tensor> const& azp,
63 + c10::optional<torch::Tensor> const& bias);
64 +#endif
65 +
66 bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
67 // CUTLASS FP8 kernels need at least
68 // CUDA 12.0 on SM90 systems (Hopper)
@@ -45,18 +79,20 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
79 return false;
80 }
81
48 -void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
49 - torch::Tensor const& b, torch::Tensor const& a_scales,
50 - torch::Tensor const& b_scales,
51 - c10::optional<torch::Tensor> const& bias) {
52 - int32_t major_capability;
53 - int32_t minor_capability;
82 +int32_t get_sm_version_num() {
83 + int32_t major_capability, minor_capability;
84 cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
85 0);
86 cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
87 0);
88 int32_t version_num = major_capability * 10 + minor_capability;
89 + return version_num;
90 +}
91
92 +void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
93 + torch::Tensor const& b, torch::Tensor const& a_scales,
94 + torch::Tensor const& b_scales,
95 + c10::optional<torch::Tensor> const& bias) {
96 // Checks for conformality
97 TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
98 TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
@@ -77,7 +113,7 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
113 }
114
115 at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
80 -
116 + int32_t version_num = get_sm_version_num();
117 if (version_num >= 90) {
118 // Hopper
119
@@ -99,3 +135,64 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
135 cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
136 }
137 }
138 +
139 +void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
140 + torch::Tensor const& b,
141 + torch::Tensor const& a_scales,
142 + torch::Tensor const& b_scales,
143 + torch::Tensor const& azp_adj,
144 + c10::optional<torch::Tensor> const& azp,
145 + c10::optional<torch::Tensor> const& bias) {
146 + // Checks for conformality
147 + TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
148 + TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
149 + b.size(1) == c.size(1));
150 + TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
151 + TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
152 +
153 + // Check for strides and alignment
154 + TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
155 + TORCH_CHECK(b.stride(0) == 1); // Column-major
156 + TORCH_CHECK(c.stride(0) % 16 == 0 &&
157 + b.stride(1) % 16 == 0); // 16 Byte Alignment
158 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
159 +
160 + // bias, azp, azp_adj are all 1d
161 + // bias and azp_adj have n elements, azp has m elements
162 + if (bias) {
163 + TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous());
164 + }
165 + if (azp) {
166 + TORCH_CHECK(azp->numel() == a.size(0) && azp->is_contiguous());
167 + }
168 + TORCH_CHECK(azp_adj.numel() == b.size(1) && azp_adj.is_contiguous());
169 +
170 + // azp & bias types
171 + TORCH_CHECK(azp_adj.dtype() == torch::kInt32);
172 + TORCH_CHECK(!azp || azp->dtype() == torch::kInt32);
173 + TORCH_CHECK(!bias || bias->dtype() == c.dtype(),
174 + "currently bias dtype must match output dtype ", c.dtype());
175 +
176 + at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
177 + int32_t version_num = get_sm_version_num();
178 + if (version_num >= 90) {
179 + // Hopper
180 +
181 + // Guard against compilation issues for sm90 kernels
182 +#if defined CUDA_VERSION && CUDA_VERSION >= 12000
183 + cutlass_scaled_mm_azp_sm90(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
184 +#else
185 + cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
186 +#endif
187 + } else if (version_num == 89) {
188 + // Ada Lovelace
189 + cutlass_scaled_mm_azp_sm89(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
190 + } else if (version_num >= 80) {
191 + // Ampere
192 + cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
193 + } else {
194 + // Turing
195 + TORCH_CHECK(version_num >= 75);
196 + cutlass_scaled_mm_azp_sm75(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
197 + }
198 +}
\ No newline at end of file
csrc/torch_bindings.cpp
+10 -1
@@ -166,13 +166,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
166 ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
167
168 // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
169 - // quantization.
169 + // quantization, as well as bias
170 ops.def(
171 "cutlass_scaled_mm(Tensor! out, Tensor a,"
172 " Tensor b, Tensor a_scales,"
173 " Tensor b_scales, Tensor? bias) -> ()");
174 ops.impl("cutlass_scaled_mm", torch::kCUDA, &cutlass_scaled_mm);
175
176 + // CUTLASS w8a8 GEMM, supporting asymmetric per-tensor or per-row/column
177 + // quantization.
178 + ops.def(
179 + "cutlass_scaled_mm_azp(Tensor! out, Tensor a,"
180 + " Tensor b, Tensor a_scales,"
181 + " Tensor b_scales, Tensor azp_adj,"
182 + " Tensor? azp, Tensor? bias) -> ()");
183 + ops.impl("cutlass_scaled_mm_azp", torch::kCUDA, &cutlass_scaled_mm_azp);
184 +
185 // Check if cutlass scaled_mm is supported for CUDA devices of the given
186 // capability
187 ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
tests/kernels/test_cutlass.py
+119 -1
@@ -28,13 +28,16 @@ def to_int8(tensor: torch.Tensor):
28 return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
29
30
31 +def rand_int8(shape: tuple, device: str = "cuda"):
32 + return to_int8(torch.rand(shape, device=device) * 255 - 128)
33 +
34 +
35 def baseline_scaled_mm(a: torch.Tensor,
36 b: torch.Tensor,
37 scale_a: torch.Tensor,
38 scale_b: torch.Tensor,
39 out_dtype: Type[torch.dtype],
40 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
37 -
41 output = (scale_a * (scale_b * (torch.mm(
42 a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
43 if bias is not None:
@@ -221,6 +224,121 @@ def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
224 use_bias)
225
226
227 +@pytest.mark.parametrize("m", [32, 64, 128])
228 +@pytest.mark.parametrize("n", [16, 32, 64])
229 +@pytest.mark.parametrize("k", [64, 128, 256])
230 +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
231 +@pytest.mark.skip
232 +def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int,
233 + out_dtype: torch.dtype):
234 + # Currently, the test is failing because folding azp into
235 + # 16-bit bias loses too much precision
236 + scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
237 + scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
238 +
239 + aq_i8 = rand_int8((m, k))
240 + bq_i8 = rand_int8((n, k)).t()
241 +
242 + aq_i32 = aq_i8.to(dtype=torch.int32)
243 + bq_i32 = bq_i8.to(dtype=torch.int32)
244 +
245 + aq_f32 = aq_i8.to(dtype=torch.float32)
246 + bq_f32 = bq_i8.to(dtype=torch.float32)
247 +
248 + b_dq = scale_b * bq_f32
249 +
250 + azp_a = torch.rand((1, ), device="cuda", dtype=torch.float32) * 10 + 1.5
251 + azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
252 + azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
253 +
254 + a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32)
255 + assert torch.allclose(a_dq, scale_a * aq_f32 + azp_a)
256 +
257 + baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype)
258 +
259 + J = torch.ones((1, k), device="cuda", dtype=torch.float32)
260 + azp_bias = (azp_a * scale_b * (J @ bq_f32)).to(out_dtype)
261 + assert azp_bias.shape == (1, n)
262 + assert azp_bias[0, :].shape == (n, )
263 +
264 + baseline_q = (scale_a.to(device='cpu') * scale_b.to(device='cpu') * (
265 + (aq_i32 + azp_aq_i8).to(device='cpu') @ bq_i32.to(device='cpu'))).to(
266 + dtype=out_dtype, device='cuda')
267 +
268 + out = ops.cutlass_scaled_mm(aq_i8,
269 + bq_i8,
270 + scale_a,
271 + scale_b,
272 + out_dtype=out_dtype,
273 + bias=azp_bias[0, :])
274 + assert torch.allclose(out, baseline_dq, rtol=1e-2, atol=1e0)
275 + assert torch.allclose(out, baseline_q, rtol=1e-2, atol=1e0)
276 +
277 +
278 +@pytest.mark.parametrize("m", [32, 64, 128])
279 +@pytest.mark.parametrize("n", [16, 32, 64])
280 +@pytest.mark.parametrize("k", [64, 128, 256])
281 +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
282 +@pytest.mark.parametrize("use_bias", [True, False])
283 +@pytest.mark.parametrize("azp_per_token", [True, False])
284 +def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
285 + use_bias: bool, azp_per_token: bool):
286 + m_azp = m if azp_per_token else 1
287 + scale_a = torch.randn((m_azp, 1), device="cuda", dtype=torch.float32) / 10
288 + scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
289 +
290 + aq_i8 = rand_int8((m, k))
291 + aq_i32 = aq_i8.to(dtype=torch.int32)
292 + aq_f32 = aq_i8.to(dtype=torch.float32)
293 +
294 + bq_i8 = rand_int8((n, k)).t()
295 + bq_i32 = bq_i8.to(dtype=torch.int32)
296 + bq_f32 = bq_i8.to(dtype=torch.float32)
297 + b_dq = scale_b * bq_f32
298 +
299 + azp_a = torch.rand(
300 + (m_azp, 1), device="cuda", dtype=torch.float32) * 10 + 1.5
301 + azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
302 + azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
303 +
304 + a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32)
305 + assert torch.allclose(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3)
306 +
307 + if use_bias:
308 + bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5
309 + else:
310 + bias = torch.zeros((1, n), device="cuda", dtype=out_dtype)
311 +
312 + baseline_dq = (torch.mm(a_dq, b_dq) + bias).to(out_dtype)
313 +
314 + # int32 mm not supported on CUDA
315 + a_noazp_i32_cpu = (aq_i32 - azp_aq_i8).to(device='cpu')
316 + cq = (a_noazp_i32_cpu @ bq_i32.to(device='cpu')).to(device='cuda')
317 + baseline_q = (scale_a * scale_b * cq + bias).to(dtype=out_dtype)
318 +
319 + # Hadamard is just the sum of the cols
320 + azp_adj_i32 = bq_i32.sum(dim=0, keepdim=True, dtype=torch.int32)
321 + azp_i32 = azp_aq_i8.to(dtype=torch.int32)
322 + func_bias = bias if use_bias else None
323 +
324 + if azp_per_token:
325 + out = ops.cutlass_scaled_mm_azp(aq_i8, bq_i8, scale_a, scale_b,
326 + out_dtype, azp_adj_i32, azp_i32,
327 + func_bias)
328 + else:
329 + azp_with_adj_i32 = azp_i32 * azp_adj_i32
330 + out = ops.cutlass_scaled_mm_azp(aq_i8, bq_i8, scale_a, scale_b,
331 + out_dtype, azp_with_adj_i32, None,
332 + func_bias)
333 +
334 + # bfloat16 precision is 7-bit mantissa -> 2^-8 ~ 0.4%
335 + # float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05%
336 + rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3
337 + atol = 1e-3
338 + assert torch.allclose(out, baseline_dq, rtol=rtol, atol=atol)
339 + assert torch.allclose(out, baseline_q, rtol=rtol, atol=atol)
340 +
341 +
342 # Test working with a subset of A and B
343 def test_cutlass_subset():
344 big_m, big_n, big_k = 1024, 1024, 1024
vllm/_custom_ops.py
+25 -1
@@ -241,6 +241,8 @@ def cutlass_scaled_mm(a: torch.Tensor,
241 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
242 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
243 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
244 + assert bias is None or bias.shape[0] == b.shape[
245 + 1] and bias.dtype == out_dtype
246
247 m = a.shape[0]
248 n = b.shape[1]
@@ -251,6 +253,28 @@ def cutlass_scaled_mm(a: torch.Tensor,
253 return out
254
255
256 +def cutlass_scaled_mm_azp(a: torch.Tensor,
257 + b: torch.Tensor,
258 + scale_a: torch.Tensor,
259 + scale_b: torch.Tensor,
260 + out_dtype: torch.dtype,
261 + azp_adj: torch.Tensor,
262 + azp: Optional[torch.Tensor] = None,
263 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
264 + assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
265 + assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
266 + assert bias is None or bias.numel(
267 + ) == b.shape[1] and bias.dtype == out_dtype
268 +
269 + m = a.shape[0]
270 + n = b.shape[1]
271 + out = torch.empty((m, n), dtype=out_dtype, device=a.device)
272 +
273 + torch.ops._C.cutlass_scaled_mm_azp(out, a, b, scale_a, scale_b, azp_adj,
274 + azp, bias)
275 + return out
276 +
277 +
278 # aqlm
279 def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
280 codebooks: torch.Tensor, scales: torch.Tensor,
@@ -572,7 +596,7 @@ for k, v in names_and_values.items():
596 if isinstance(v, fn_type) \
597 and v.__code__.co_filename == __file__ \
598 and any(arg is torch.Tensor or arg == "torch.Tensor"
575 - for arg in v.__annotations__.values()):
599 + for arg in v.__annotations__.values()):
600 names_and_values_to_update[k] = hint_on_error(v)
601
602 names_and_values.update(names_and_values_to_update)