A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel] (1/N) Machete - Hopper Optimized Mixed Precision Linear Kernel (#7174)
Lucas Wilkinson committed
Aug 20, 2024 at 09:09 UTC
5288c06aa03b100eab4f873452b65da941a1a232
28 files changed
+4828
-2
.gitignore
+3
@@ -87,6 +87,9 @@ target/
87
profile_default/
88
ipython_config.py
89
90
+# generated files
91
+**/generated/**
92
+
93
# pyenv
94
# For a library or package, you might want to ignore these files since the code is
95
# intended to run in multiple environments; otherwise, check them in:
CMakeLists.txt
+40
@@ -227,6 +227,46 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
227
"-gencode arch=compute_90a,code=sm_90a")
228
endif()
229
230
+ #
231
+ # For the Machete kernels we automatically generate sources for various
232
+ # preselected input type pairs and schedules.
233
+ # Generate sources:
234
+ execute_process(
235
+ COMMAND ${CMAKE_COMMAND} -E env
236
+ PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:$PYTHONPATH
237
+ ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py
238
+ RESULT_VARIABLE machete_generation_result
239
+ OUTPUT_VARIABLE machete_generation_output
240
+ OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
241
+ ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
242
+ )
243
+
244
+ if (NOT machete_generation_result EQUAL 0)
245
+ message(FATAL_ERROR "Machete generation failed."
246
+ " Result: \"${machete_generation_result}\""
247
+ "\nCheck the log for details: "
248
+ "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
249
+ else()
250
+ message(STATUS "Machete generation completed successfully.")
251
+ endif()
252
+
253
+ # Add machete generated sources
254
+ file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
255
+ list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
256
+ message(STATUS "Machete generated sources: ${MACHETE_GEN_SOURCES}")
257
+
258
+ # See comment above for scaled_mm_c3x (same if condition)
259
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
260
+ set_source_files_properties(
261
+ ${MACHETE_GEN_SOURCES}
262
+ PROPERTIES
263
+ COMPILE_FLAGS
264
+ "-gencode arch=compute_90a,code=sm_90a")
265
+ endif()
266
+
267
+ # Add pytorch binding
268
+ list(APPEND VLLM_EXT_SRC
269
+ csrc/quantization/machete/machete_pytorch.cu)
270
endif()
271
272
define_gpu_extension_target(
benchmarks/kernels/benchmark_machete.py
new
+372
@@ -0,0 +1,372 @@
1
+import argparse
2
+import copy
3
+import itertools
4
+import math
5
+import pickle as pkl
6
+import time
7
+from typing import Callable, Iterable, List, Tuple
8
+
9
+import torch
10
+import torch.utils.benchmark as TBenchmark
11
+from torch.utils.benchmark import Measurement as TMeasurement
12
+from weight_shapes import WEIGHT_SHAPES
13
+
14
+from vllm import _custom_ops as ops
15
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
16
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, marlin_permute_scales)
17
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
18
+ MarlinWorkspace)
19
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
20
+ gptq_pack, pack_rows, quantize_weights)
21
+from vllm.scalar_type import ScalarType, scalar_types
22
+from vllm.utils import FlexibleArgumentParser
23
+
24
+DEFAULT_MODELS = ["meta-llama/Llama-3-8b", "meta-llama/Llama-2-70b-hf"]
25
+DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512, 1024]
26
+DEFAULT_TP_SIZES = [1]
27
+
28
+
29
+def machete_pack_weights(w_q: torch.tensor, wtype: ScalarType) -> torch.tensor:
30
+ w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
31
+ w_q = w_q.t().contiguous().t() # make col major
32
+ return ops.machete_prepack_B(w_q, wtype)
33
+
34
+
35
+def make_bench_tensors(
36
+ atype: torch.dtype, wtype: ScalarType, group_size: int, m: int, n: int,
37
+ k: int
38
+) -> Tuple[torch.tensor, List[Tuple[torch.tensor, torch.tensor, torch.tensor,
39
+ torch.tensor]]]:
40
+ assert wtype.is_integer(), "TODO: support floating point weights"
41
+
42
+ # we want to make sure that weights don't fit into L2 cache between runs so
43
+ # we construct enough weights to exceed L2 cache, which is 50mb on a H100
44
+ # so we target total weight size > 2*50mb
45
+ num_weights = math.ceil(2 * 50 * 1024**2 * 8 / (k * n * wtype.size_bits))
46
+
47
+ a = torch.randn((m, k), device="cuda", dtype=atype) * 5
48
+ weights = [
49
+ torch.randn((k, n), device="cuda", dtype=atype)
50
+ for _ in range(num_weights)
51
+ ]
52
+ quanitized_weights = [
53
+ quantize_weights(w, wtype, group_size) for w in weights
54
+ ]
55
+
56
+ return a, quanitized_weights
57
+
58
+
59
+# impl
60
+
61
+
62
+# bench
63
+def bench_fn(label: str, sub_label: str, description: str,
64
+ fn: Callable) -> TMeasurement:
65
+
66
+ min_run_time = 1
67
+ return TBenchmark.Timer(
68
+ stmt="fn()",
69
+ globals={
70
+ "fn": fn
71
+ },
72
+ label=label,
73
+ sub_label=sub_label,
74
+ description=description,
75
+ ).blocked_autorange(min_run_time=min_run_time)
76
+
77
+
78
+def loop_over_weights(
79
+ a: torch.tensor, weights: List[Tuple[torch.tensor, torch.tensor,
80
+ torch.tensor, torch.tensor]],
81
+ fn: Callable[[torch.tensor, torch.tensor, torch.tensor, torch.tensor],
82
+ None]):
83
+ for w_ref, w_q, w_s, _ in weights:
84
+ fn(a, w_ref, w_q, w_s)
85
+
86
+
87
+def bench(atype: torch.dtype,
88
+ wtype: ScalarType,
89
+ group_size: int,
90
+ m: int,
91
+ k: int,
92
+ n: int,
93
+ label: str,
94
+ sub_label: str,
95
+ benchmark_marlinv1: bool = True,
96
+ sweep_schedules: bool = True) -> Iterable[TMeasurement]:
97
+ a, weights = make_bench_tensors(atype, wtype, group_size, m, n, k)
98
+ sub_label += f", L={len(weights)}"
99
+
100
+ weights_machete = [(w_ref, machete_pack_weights(w_q, wtype), w_s, w_zp)
101
+ for w_ref, w_q, w_s, w_zp in weights]
102
+
103
+ timers = []
104
+ # pytorch impl
105
+ timers.append(
106
+ bench_fn(
107
+ label, sub_label, "torch.matmul", lambda: loop_over_weights(
108
+ a,
109
+ weights,
110
+ lambda a, w_ref, w_q, w_s: torch.matmul(a, w_ref),
111
+ )))
112
+
113
+ if benchmark_marlinv1:
114
+ w_ref = weights[0][0]
115
+
116
+ w_zp_empty = torch.empty(0, dtype=torch.int, device=w_ref.device)
117
+ sort_indices = torch.empty(0, dtype=torch.int, device=w_ref.device)
118
+ g_idx = torch.empty(0, dtype=torch.int, device=w_ref.device)
119
+
120
+ def marlinv1_pack_weights(w_q: torch.tensor) -> torch.tensor:
121
+ w_q_gptq = gptq_pack(w_q, wtype.size_bits, *w_ref.shape)
122
+ return ops.gptq_marlin_repack(w_q_gptq, sort_indices, *w_ref.shape,
123
+ wtype.size_bits)
124
+
125
+ def marlinv1_permute_scales(w_s: torch.tensor) -> torch.tensor:
126
+ return marlin_permute_scales(w_s, *w_ref.shape, group_size)
127
+
128
+ weights_marlinv1 = [(w_ref, marlinv1_pack_weights(w_q),
129
+ marlinv1_permute_scales(w_s), w_zp)
130
+ for w_ref, w_q, w_s, w_zp in weights]
131
+
132
+ workspace = MarlinWorkspace(w_ref.shape[1], GPTQ_MARLIN_MIN_THREAD_N,
133
+ GPTQ_MARLIN_MAX_PARALLEL)
134
+
135
+ # marlinv1
136
+ timers.append(
137
+ bench_fn(
138
+ label, sub_label, "marlin_orig", lambda: loop_over_weights(
139
+ a, weights_marlinv1, lambda a, w_ref, w_q, w_s: ops.
140
+ gptq_marlin_gemm(a,
141
+ w_q,
142
+ w_s,
143
+ w_zp_empty,
144
+ g_idx,
145
+ sort_indices,
146
+ workspace.scratch,
147
+ wtype,
148
+ size_m=a.shape[0],
149
+ size_n=w_ref.shape[1],
150
+ size_k=w_ref.shape[0],
151
+ is_k_full=True))))
152
+
153
+ # machete
154
+ timers.append(
155
+ bench_fn(
156
+ label, sub_label, "machete_heuristic", lambda: loop_over_weights(
157
+ a, weights_machete, lambda a, _, w_q, w_s: ops.machete_gemm(
158
+ a, w_q, wtype, b_scales=w_s, b_group_size=group_size))))
159
+
160
+ if sweep_schedules:
161
+ print("Finding best schedule for machete")
162
+ best = None
163
+ best_schedule = None
164
+ schedules = ops.machete_supported_schedules(wtype)
165
+ for schedule in reversed(schedules):
166
+
167
+ def run(a, _, w_q, w_s, schedule=schedule):
168
+ ops.machete_gemm(a,
169
+ w_q,
170
+ wtype,
171
+ w_s,
172
+ b_group_size=group_size,
173
+ schedule=schedule)
174
+
175
+ res = bench_fn(label, sub_label, "machete_best",
176
+ lambda: loop_over_weights(a, weights_machete, run))
177
+
178
+ print(f" {res.median:5.5} ", schedule)
179
+ if not best or res.median < best.median:
180
+ best = res
181
+ best_schedule = schedule
182
+ print("Best schedule:", best_schedule)
183
+ timers.append(best)
184
+
185
+ return timers
186
+
187
+
188
+# runner
189
+def print_timers(timers: Iterable[TMeasurement]):
190
+ compare = TBenchmark.Compare(timers)
191
+ compare.print()
192
+
193
+
194
+def run(dtype: torch.dtype, sweep_schedules: bool,
195
+ MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
196
+
197
+ results = []
198
+ for m, k, n in MKNs:
199
+ timers = bench(dtype,
200
+ scalar_types.uint4b8,
201
+ 128,
202
+ m,
203
+ k,
204
+ n,
205
+ f"{dtype}-gemm",
206
+ f"MKN=({m}x{k}x{n})",
207
+ sweep_schedules=sweep_schedules)
208
+ print_timers(timers)
209
+ results.extend(timers)
210
+
211
+ return results
212
+
213
+
214
+# output makers
215
+def make_output(
216
+ data: Iterable[TMeasurement],
217
+ MKNs: Iterable[Tuple[int, int, int]],
218
+ base_description: str,
219
+ timestamp=None,
220
+):
221
+
222
+ print(f"== All Results {base_description} ====")
223
+ print_timers(data)
224
+
225
+ # pickle all the results
226
+ timestamp = int(time.time()) if timestamp is None else timestamp
227
+ with open(f"{base_description}-{timestamp}.pkl", "wb") as f:
228
+ pkl.dump(data, f)
229
+
230
+
231
+# argparse runners
232
+
233
+
234
+def run_square_bench(args):
235
+ dim_sizes = list(
236
+ range(args.dim_start, args.dim_end + 1, args.dim_increment))
237
+ MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
238
+ data = run(args.dtype, args.sweep_schedules, MKNs)
239
+
240
+ make_output(data, MKNs, f"square_bench-{args.dtype}")
241
+
242
+
243
+def run_range_bench(args):
244
+ dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
245
+ n = len(dim_sizes)
246
+ Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
247
+ Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
248
+ Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
249
+ MKNs = list(zip(Ms, Ks, Ns))
250
+ data = run(args.dtype, args.sweep_schedules, MKNs)
251
+
252
+ make_output(data, MKNs, f"range_bench-{args.dtype}")
253
+
254
+
255
+def run_model_bench(args):
256
+
257
+ print("Benchmarking models:")
258
+ for i, model in enumerate(args.models):
259
+ print(f"[{i}] {model}")
260
+
261
+ def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:
262
+ KNs = []
263
+ for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):
264
+ KN[tp_split_dim] = KN[tp_split_dim] // tp_size
265
+ KNs.append(KN)
266
+ return KNs
267
+
268
+ model_bench_data = []
269
+ models_tps = list(itertools.product(args.models, args.tp_sizes))
270
+ for model, tp_size in models_tps:
271
+ Ms = args.batch_sizes
272
+ KNs = model_shapes(model, tp_size)
273
+ MKNs = []
274
+ for m in Ms:
275
+ for k, n in KNs:
276
+ MKNs.append((m, k, n))
277
+
278
+ data = run(args.dtype, args.sweep_schedules, MKNs)
279
+ model_bench_data.append(data)
280
+
281
+ # Print all results
282
+ for data, model_tp in zip(model_bench_data, models_tps):
283
+ model, tp_size = model_tp
284
+ print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
285
+ print_timers(data)
286
+
287
+ timestamp = int(time.time())
288
+
289
+ all_data = []
290
+ for d in model_bench_data:
291
+ all_data.extend(d)
292
+ # pickle all data
293
+ with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
294
+ pkl.dump(all_data, f)
295
+
296
+
297
+if __name__ == "__main__":
298
+
299
+ def to_torch_dtype(dt):
300
+ if dt == "bfloat16":
301
+ return torch.bfloat16
302
+ if dt == "float16":
303
+ return torch.float16
304
+ raise ValueError("unsupported dtype")
305
+
306
+ parser = FlexibleArgumentParser(
307
+ description="""
308
+Benchmark Machete GEMM.
309
+
310
+ To run square GEMMs:
311
+ python3 ./benchmarks/kernels/benchmark_machete.py --dtype float16 square_bench --dim-start 128 --dim-end 512 --dim-increment 64
312
+
313
+ To run constant N and K and sweep M:
314
+ python3 ./benchmarks/kernels/benchmark_machete.py --dtype float16 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384
315
+
316
+ To run dimensions from a model:
317
+ python3 ./benchmarks/kernels/benchmark_machete.py --dtype float16 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1
318
+
319
+ Output:
320
+ - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs.
321
+ """, # noqa: E501
322
+ formatter_class=argparse.RawTextHelpFormatter,
323
+ )
324
+
325
+ parser.add_argument(
326
+ "--dtype",
327
+ type=to_torch_dtype,
328
+ required=True,
329
+ help="Available options are ['bfloat16', 'float16']",
330
+ )
331
+ parser.add_argument(
332
+ "--sweep-schedules",
333
+ action="store_true",
334
+ help="Run a sweep over all supported schedules",
335
+ )
336
+ subparsers = parser.add_subparsers(dest="cmd", required=True)
337
+
338
+ square_parser = subparsers.add_parser("square_bench")
339
+ square_parser.add_argument("--dim-start", type=int, required=True)
340
+ square_parser.add_argument("--dim-end", type=int, required=True)
341
+ square_parser.add_argument("--dim-increment", type=int, required=True)
342
+ square_parser.set_defaults(func=run_square_bench)
343
+
344
+ range_parser = subparsers.add_parser("range_bench")
345
+ range_parser.add_argument("--dim-start", type=int, required=True)
346
+ range_parser.add_argument("--dim-end", type=int, required=True)
347
+ range_parser.add_argument("--dim-increment", type=int, required=True)
348
+ range_parser.add_argument("--m-constant", type=int, default=None)
349
+ range_parser.add_argument("--n-constant", type=int, default=None)
350
+ range_parser.add_argument("--k-constant", type=int, default=None)
351
+ range_parser.set_defaults(func=run_range_bench)
352
+
353
+ model_parser = subparsers.add_parser("model_bench")
354
+ model_parser.add_argument(
355
+ "--models",
356
+ nargs="+",
357
+ type=str,
358
+ default=DEFAULT_MODELS,
359
+ choices=WEIGHT_SHAPES.keys(),
360
+ )
361
+ model_parser.add_argument("--tp-sizes",
362
+ nargs="+",
363
+ type=int,
364
+ default=DEFAULT_TP_SIZES)
365
+ model_parser.add_argument("--batch-sizes",
366
+ nargs="+",
367
+ type=int,
368
+ default=DEFAULT_BATCH_SIZES)
369
+ model_parser.set_defaults(func=run_model_bench)
370
+
371
+ args = parser.parse_args()
372
+ args.func(args)
benchmarks/kernels/graph_machete_bench.py
new
+64
@@ -0,0 +1,64 @@
1
+import math
2
+import pickle
3
+import re
4
+from collections import defaultdict
5
+from typing import List
6
+
7
+import matplotlib.pyplot as plt
8
+import pandas as pd
9
+import seaborn as sns
10
+from torch.utils.benchmark import Measurement as TMeasurement
11
+
12
+from vllm.utils import FlexibleArgumentParser
13
+
14
+if __name__ == "__main__":
15
+ parser = FlexibleArgumentParser(
16
+ description='Benchmark the latency of processing a single batch of '
17
+ 'requests till completion.')
18
+ parser.add_argument('filename', type=str)
19
+
20
+ args = parser.parse_args()
21
+
22
+ with open(args.filename, 'rb') as f:
23
+ data: List[TMeasurement] = pickle.load(f)
24
+
25
+ results = defaultdict(lambda: list())
26
+ for v in data:
27
+ result = re.search(r"MKN=\(\d+x(\d+x\d+)\)", v.task_spec.sub_label)
28
+ if result is not None:
29
+ KN = result.group(1)
30
+ else:
31
+ raise Exception("MKN not found")
32
+ result = re.search(r"MKN=\((\d+)x\d+x\d+\)", v.task_spec.sub_label)
33
+ if result is not None:
34
+ M = result.group(1)
35
+ else:
36
+ raise Exception("MKN not found")
37
+
38
+ kernel = v.task_spec.description
39
+ results[KN].append({
40
+ "kernel": kernel,
41
+ "batch_size": M,
42
+ "median": v.median
43
+ })
44
+
45
+ rows = int(math.ceil(len(results) / 2))
46
+ fig, axs = plt.subplots(rows, 2, figsize=(12, 5 * rows))
47
+ axs = axs.flatten()
48
+ axs_idx = 0
49
+ for shape, data in results.items():
50
+ plt.sca(axs[axs_idx])
51
+ df = pd.DataFrame(data)
52
+ sns.lineplot(data=df,
53
+ x="batch_size",
54
+ y="median",
55
+ hue="kernel",
56
+ style="kernel",
57
+ markers=True,
58
+ dashes=False,
59
+ palette="Dark2")
60
+ plt.title(f"Shape: {shape}")
61
+ plt.ylabel("time (median, s)")
62
+ axs_idx += 1
63
+ plt.tight_layout()
64
+ plt.savefig("graph_machete_bench.pdf")
benchmarks/kernels/weight_shapes.py
new
+43
@@ -0,0 +1,43 @@
1
+# Weight Shapes are in the format
2
+# ([K, N], TP_SPLIT_DIM)
3
+# Example:
4
+# A shape of ([14336, 4096], 0) indicates the following GEMM shape,
5
+# - TP1 : K = 14336, N = 4096
6
+# - TP2 : K = 7168, N = 4096
7
+# A shape of ([4096, 6144], 1) indicates the following GEMM shape,
8
+# - TP1 : K = 4096, N = 6144
9
+# - TP4 : K = 4096, N = 1536
10
+
11
+# TP1 shapes
12
+WEIGHT_SHAPES = {
13
+ "mistralai/Mistral-7B-v0.1": [
14
+ ([4096, 6144], 1),
15
+ ([4096, 4096], 0),
16
+ ([4096, 28672], 1),
17
+ ([14336, 4096], 0),
18
+ ],
19
+ "meta-llama/Llama-2-7b-hf": [
20
+ ([4096, 12288], 1),
21
+ ([4096, 4096], 0),
22
+ ([4096, 22016], 1),
23
+ ([11008, 4096], 0),
24
+ ],
25
+ "meta-llama/Llama-3-8b": [
26
+ ([4096, 6144], 1),
27
+ ([4096, 4096], 0),
28
+ ([4096, 28672], 1),
29
+ ([14336, 4096], 0),
30
+ ],
31
+ "meta-llama/Llama-2-13b-hf": [
32
+ ([5120, 15360], 1),
33
+ ([5120, 5120], 0),
34
+ ([5120, 27648], 1),
35
+ ([13824, 5120], 0),
36
+ ],
37
+ "meta-llama/Llama-2-70b-hf": [
38
+ ([8192, 10240], 1),
39
+ ([8192, 8192], 0),
40
+ ([8192, 57344], 1),
41
+ ([28672, 8192], 0),
42
+ ],
43
+}
csrc/cuda_utils.h
+10
@@ -1,5 +1,15 @@
1
#pragma once
2
3
+#if defined(__CUDACC__) || defined(_NVHPC_CUDA)
4
+ #define HOST_DEVICE_INLINE __forceinline__ __host__ __device__
5
+ #define DEVICE_INLINE __forceinline__ __device__
6
+ #define HOST_INLINE __forceinline__ __host__
7
+#else
8
+ #define HOST_DEVICE_INLINE inline
9
+ #define DEVICE_INLINE inline
10
+ #define HOST_INLINE inline
11
+#endif
12
+
13
int64_t get_device_attribute(int64_t attribute, int64_t device_id);
14
15
int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id);
csrc/cutlass_extensions/cute_utils.cuh
new
+68
@@ -0,0 +1,68 @@
1
+#pragma once
2
+
3
+#include <cute/tensor.hpp>
4
+#include <torch/all.h>
5
+namespace cute {
6
+
7
+////////////////////////////////////////////////////////////////////
8
+// layout utils
9
+////////////////////////////////////////////////////////////////////
10
+
11
+// Permute layout based on indices, example:
12
+// permute_layout<1, 0>(layout) will swap the two dimensions
13
+// permute_layout<0, 2, 1>(layout) will swap the last two dimensions
14
+template <size_t... I, typename Layout>
15
+CUTE_HOST_DEVICE static constexpr auto permute_layout(Layout l) {
16
+ static_assert(rank(l) == sizeof...(I), "Invalid permutation, rank mismatch");
17
+ return cute::make_layout(cute::get<I>(l)...);
18
+}
19
+
20
+// is the layout f(x) = x
21
+template <typename Layout>
22
+CUTE_HOST_DEVICE static constexpr bool is_identity_layout() {
23
+ if constexpr (std::is_same_v<Layout, void>)
24
+ return true;
25
+ else {
26
+ constexpr auto coalesced_layout = coalesce(Layout{});
27
+ if constexpr (rank(coalesced_layout) == 1 &&
28
+ stride<0>(coalesced_layout) == 1) {
29
+ return true;
30
+ }
31
+ return false;
32
+ }
33
+}
34
+
35
+////////////////////////////////////////////////////////////////////
36
+// Pointer utils
37
+////////////////////////////////////////////////////////////////////
38
+
39
+template <class PointerType>
40
+static constexpr auto get_logical_ptr(PointerType* ptr) {
41
+ if constexpr (cute::sizeof_bits_v<PointerType> < 8) {
42
+ return cute::subbyte_iterator<PointerType>(ptr);
43
+ } else {
44
+ return ptr;
45
+ }
46
+}
47
+
48
+////////////////////////////////////////////////////////////////////
49
+// Misc utils
50
+////////////////////////////////////////////////////////////////////
51
+
52
+template <typename T, typename Elements>
53
+CUTE_HOST_DEVICE static constexpr auto create_auto_vectorizing_copy() {
54
+ constexpr auto bits = sizeof_bits_v<T> * Elements{};
55
+ if constexpr (bits % 128 == 0) {
56
+ return AutoVectorizingCopyWithAssumedAlignment<128>{};
57
+ } else if constexpr (bits % 64 == 0) {
58
+ return AutoVectorizingCopyWithAssumedAlignment<64>{};
59
+ } else if constexpr (bits % 32 == 0) {
60
+ return AutoVectorizingCopyWithAssumedAlignment<32>{};
61
+ } else if constexpr (bits % 16 == 0) {
62
+ return AutoVectorizingCopyWithAssumedAlignment<16>{};
63
+ } else {
64
+ return AutoVectorizingCopyWithAssumedAlignment<8>{};
65
+ }
66
+}
67
+
68
+}; // namespace cute
csrc/cutlass_extensions/torch_utils.hpp
new
+154
@@ -0,0 +1,154 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+
5
+#include "cute/layout.hpp"
6
+#include "cutlass/layout/matrix.h"
7
+#include "cutlass/bfloat16.h"
8
+#include "cutlass/half.h"
9
+
10
+using ColumnMajor = typename cutlass::layout::ColumnMajor;
11
+using RowMajor = typename cutlass::layout::RowMajor;
12
+
13
+namespace cute {
14
+
15
+namespace detail {
16
+
17
+template <class T, class F, class G, int... I>
18
+CUTE_HOST_DEVICE constexpr auto tapply_with_idx(T&& t, F&& f, G&& g,
19
+ seq<I...>) {
20
+ return g(f(cute::get<I>(static_cast<T&&>(t)), I)...);
21
+}
22
+
23
+template <class F, int... I>
24
+CUTE_HOST_DEVICE constexpr auto make_shape_from_idx(F&& f, seq<I...>) {
25
+ return make_shape(f(I)...);
26
+}
27
+
28
+}; // namespace detail
29
+
30
+template <class T, class F>
31
+CUTE_HOST_DEVICE constexpr auto transform_with_idx(T const& t, F&& f) {
32
+ if constexpr (cute::is_tuple<T>::value) {
33
+ return detail::tapply_with_idx(
34
+ t, f, [](auto const&... a) { return cute::make_tuple(a...); },
35
+ tuple_seq<T>{});
36
+ } else {
37
+ return f(t);
38
+ }
39
+
40
+ CUTE_GCC_UNREACHABLE;
41
+}
42
+
43
+// calls: make_shape(f(0), f(1), ..., f(N-1))
44
+template <int N, class F>
45
+CUTE_HOST_DEVICE constexpr auto make_shape_from_idx(F&& f) {
46
+ return detail::make_shape_from_idx(f, make_seq<N>{});
47
+}
48
+
49
+}; // namespace cute
50
+
51
+// Make a layout from a tensor with `rank(Stride{})`, where the shape is the
52
+// shape of the passed in tensor and the strides are of type `Stride` and
53
+// contain the strides of the passed in tensor, checking that any static strides
54
+// in `Stride{}` match the strides of the passed in tensor.
55
+// If `tensor.dim() < rank(Stride{})`, the shape is padded with 1s and the extra
56
+// strides are set to be 0 or 1.
57
+template <typename Stride>
58
+static inline auto make_cute_layout(torch::Tensor const& tensor,
59
+ std::string_view name = "tensor") {
60
+ TORCH_CHECK(tensor.dim() <= rank(Stride{}));
61
+ auto stride = cute::transform_with_idx(
62
+ Stride{}, [&](auto const& stride_ele, auto const& idx) {
63
+ using StrideEle = std::decay_t<decltype(stride_ele)>;
64
+
65
+ if (idx < tensor.dim()) {
66
+ if constexpr (cute::is_static_v<StrideEle>) {
67
+ TORCH_CHECK(StrideEle::value == tensor.stride(idx), "Expected ",
68
+ name, ".stride(", idx, ") to be ", StrideEle::value);
69
+ return StrideEle{};
70
+ } else {
71
+ return tensor.stride(idx);
72
+ }
73
+ } else {
74
+ // Extra strides are assumed to be 0 or 1
75
+ if constexpr (cute::is_static_v<StrideEle>) {
76
+ static_assert(StrideEle::value == 0 || StrideEle::value == 1);
77
+ }
78
+ return StrideEle{};
79
+ }
80
+ });
81
+
82
+ auto shape = cute::make_shape_from_idx<rank(Stride{})>([&](auto const& idx) {
83
+ if (idx < tensor.dim())
84
+ return tensor.size(idx);
85
+ else
86
+ return int64_t(1);
87
+ });
88
+
89
+ return make_layout(shape, stride);
90
+}
91
+
92
+template <typename Stride>
93
+static inline auto maybe_make_cute_layout(
94
+ c10::optional<torch::Tensor> const& tensor,
95
+ std::string_view name = "tensor") {
96
+ using Layout = decltype(make_cute_layout<Stride>(*tensor));
97
+
98
+ if (tensor) {
99
+ return std::optional<Layout>{make_cute_layout<Stride>(*tensor, name)};
100
+ } else {
101
+ return std::optional<Layout>{};
102
+ }
103
+}
104
+
105
+//
106
+// Torch Type to Cutlass Type (equivalent_cutlass_type)
107
+//
108
+
109
+template <typename T>
110
+struct equivalent_cutlass_type {
111
+ using type = T;
112
+};
113
+
114
+template <typename T>
115
+using equivalent_cutlass_type_t = typename equivalent_cutlass_type<T>::type;
116
+
117
+template <>
118
+struct equivalent_cutlass_type<c10::Half> {
119
+ using type = cutlass::half_t;
120
+};
121
+
122
+template <>
123
+struct equivalent_cutlass_type<c10::BFloat16> {
124
+ using type = cutlass::bfloat16_t;
125
+};
126
+
127
+//
128
+// equivalent_scalar_t (basically inverse of equivalent_cutlass_type)
129
+//
130
+
131
+// Return a `c10::CppTypeToScalarType<T>` compatible type, i.e. get the C++ from
132
+// c10 that is equivalent to T, e.g.: `cutlass::half_t -> c10::Half`
133
+template <typename T>
134
+struct equivalent_scalar_type {
135
+ using type = T;
136
+};
137
+
138
+template <typename T>
139
+using equivalent_scalar_type_t = typename equivalent_scalar_type<T>::type;
140
+
141
+template <>
142
+struct equivalent_scalar_type<cutlass::half_t> {
143
+ using type = c10::Half;
144
+};
145
+
146
+template <>
147
+struct equivalent_scalar_type<cutlass::bfloat16_t> {
148
+ using type = c10::BFloat16;
149
+};
150
+
151
+// get equivalent c10::ScalarType tag from compile time type
152
+template <typename T>
153
+static inline constexpr c10::ScalarType equivalent_scalar_type_v =
154
+ c10::CppTypeToScalarType<equivalent_scalar_type_t<T>>::value;
\ No newline at end of file
csrc/cutlass_extensions/vllm_collective_builder.cuh
new
+43
@@ -0,0 +1,43 @@
1
+#pragma once
2
+
3
+#include "cutlass/gemm/collective/collective_builder.hpp"
4
+
5
+namespace cutlass::gemm::collective {
6
+using namespace cute;
7
+
8
+//
9
+// VLLMCollectiveBuilder is a wrapper around CollectiveBuilder that allows for
10
+// for custom kernel tags, allowing you to build custom collectives. Without
11
+// touching the cutlass library headers, using `CutlassKernelTag` will mean it
12
+// will resort to using the standard cutlass collective builder.
13
+//
14
+
15
+// Use the default Cutlass collective builder, i.e. use an unmodified cutless
16
+// collective
17
+struct CutlassKernelTag {};
18
+
19
+template <class KernelTag, class ArchTag, class OpClass, class ElementA,
20
+ class GmemLayoutA, int AlignmentA, class ElementB, class GmemLayoutB,
21
+ int AlignmentB, class ElementAccumulator, class TileShape_MNK,
22
+ class ClusterShape_MNK, class StageCountType,
23
+ class KernelScheduleType, class Enable = void>
24
+struct VLLMCollectiveBuilder {
25
+ static_assert(sizeof(ElementA) == 0,
26
+ "Could not build a collective for given parameters.");
27
+};
28
+
29
+template <class ArchTag, class OpClass, class ElementA, class GmemLayoutA,
30
+ int AlignmentA, class ElementB, class GmemLayoutB, int AlignmentB,
31
+ class ElementAccumulator, class TileShape_MNK, class ClusterShape_MNK,
32
+ class StageCountType, class KernelScheduleType>
33
+struct VLLMCollectiveBuilder<
34
+ CutlassKernelTag, ArchTag, OpClass, ElementA, GmemLayoutA, AlignmentA,
35
+ ElementB, GmemLayoutB, AlignmentB, ElementAccumulator, TileShape_MNK,
36
+ ClusterShape_MNK, StageCountType, KernelScheduleType> {
37
+ using CollectiveOp = typename CollectiveBuilder<
38
+ ArchTag, OpClass, ElementA, GmemLayoutA, AlignmentA, ElementB,
39
+ GmemLayoutB, AlignmentB, ElementAccumulator, TileShape_MNK,
40
+ ClusterShape_MNK, StageCountType, KernelScheduleType>::CollectiveOp;
41
+};
42
+
43
+}; // namespace cutlass::gemm::collective
\ No newline at end of file
csrc/cutlass_extensions/vllm_custom_types.cuh
new
+50
@@ -0,0 +1,50 @@
1
+#pragma once
2
+
3
+#include "cutlass/integer_subbyte.h"
4
+
5
+namespace cutlass {
6
+
7
+///////////////////////////////////////////////////////////////////////////////////////////////////
8
+
9
+template <int Bits, int Bias, bool Signed = false>
10
+struct vllm_biased_integer_subbyte : public integer_subbyte<Bits, Signed> {
11
+ using Base = integer_subbyte<Bits, Signed>;
12
+
13
+ using Storage = typename Base::Storage;
14
+ using xint_t = typename Base::xint_t;
15
+
16
+ using Base::bits_mask_;
17
+ using Base::sign_mask_;
18
+ using Base::storage;
19
+
20
+ //
21
+ // Methods
22
+ //
23
+
24
+ /// No operation
25
+ vllm_biased_integer_subbyte() = default;
26
+
27
+ /// Conversion from integer type
28
+ CUTLASS_HOST_DEVICE explicit vllm_biased_integer_subbyte(int value)
29
+ : Base(value) {}
30
+ CUTLASS_HOST_DEVICE explicit vllm_biased_integer_subbyte(unsigned value)
31
+ : Base(value) {}
32
+ CUTLASS_HOST_DEVICE explicit vllm_biased_integer_subbyte(double value)
33
+ : Base(value) {}
34
+};
35
+///////////////////////////////////////////////////////////////////////////////////////////////////
36
+
37
+// "GPTQ" types, i.e. symmetric quantization
38
+using vllm_uint4b8_t = vllm_biased_integer_subbyte<4, 8>; // u4b8
39
+using vllm_uint8b128_t = vllm_biased_integer_subbyte<8, 128>; // u8b128
40
+
41
+///////////////////////////////////////////////////////////////////////////////////////////////////
42
+
43
+template <int Bits, int Bias, bool Signed>
44
+struct sizeof_bits<vllm_biased_integer_subbyte<Bits, Bias, Signed>> {
45
+ static constexpr int value = Bits;
46
+};
47
+
48
+///////////////////////////////////////////////////////////////////////////////////////////////////
49
+
50
+} // namespace cutlass
csrc/cutlass_extensions/vllm_cutlass_library_extension.py
new
+49
@@ -0,0 +1,49 @@
1
+import enum
2
+from typing import Dict, Union
3
+
4
+from cutlass_library import *
5
+
6
+#
7
+# Extend cutlass library with custom types, and missing values
8
+#
9
+
10
+
11
+class VLLMDataType(enum.Enum):
12
+ u4b8 = enum_auto()
13
+ u8b128 = enum_auto()
14
+
15
+
16
+class MixedInputKernelScheduleType(enum.Enum):
17
+ TmaWarpSpecializedMixedInput = enum_auto()
18
+ TmaWarpSpecializedPingpongMixedInput = enum_auto()
19
+ TmaWarpSpecializedCooperativeMixedInput = enum_auto()
20
+
21
+
22
+VLLMDataTypeNames: Dict[Union[VLLMDataType, DataType], str] = {
23
+ **DataTypeNames, # type: ignore
24
+ **{
25
+ VLLMDataType.u4b8: "u4b8",
26
+ VLLMDataType.u8b128: "u8b128",
27
+ }
28
+}
29
+
30
+VLLMDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
31
+ **DataTypeTag, # type: ignore
32
+ **{
33
+ VLLMDataType.u4b8: "cutlass::vllm_uint4b8_t",
34
+ VLLMDataType.u8b128: "cutlass::vllm_uint8b128_t",
35
+ }
36
+}
37
+
38
+VLLMKernelScheduleTag: Dict[Union[
39
+ MixedInputKernelScheduleType, KernelScheduleType], str] = {
40
+ **KernelScheduleTag, # type: ignore
41
+ **{
42
+ MixedInputKernelScheduleType.TmaWarpSpecializedMixedInput:
43
+ "cutlass::gemm::KernelTmaWarpSpecializedMixedInput",
44
+ MixedInputKernelScheduleType.TmaWarpSpecializedPingpongMixedInput:
45
+ "cutlass::gemm::KernelTmaWarpSpecializedPingpongMixedInput",
46
+ MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput:
47
+ "cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput",
48
+ }
49
+ }
csrc/cutlass_extensions/vllm_numeric_conversion.cuh
new
+795
@@ -0,0 +1,795 @@
1
+#pragma once
2
+
3
+#include "cutlass/numeric_conversion.h"
4
+#include "cutlass_extensions/vllm_custom_types.cuh"
5
+#include "cutlass_extensions/cute_utils.cuh"
6
+
7
+// this file extends:
8
+// https://github.com/NVIDIA/cutlass/blob/cutlass-3.5.0/include/cutlass/numeric_conversion.h
9
+// with vllm specific type conversions, namely: vllm_uint4b8_t, vllm_uint8b128_t
10
+// as well as adds interleaved numeric array converters for specific types.
11
+// (interleaved numeric array converters can be more efficient for subbyte
12
+// types)
13
+
14
+namespace cutlass {
15
+
16
+// InterleavedNumericArrayConverter is like NumericArrayConverter but also
17
+// deinterleaves converted elements based on IlvBlkLayout, interleaving can
18
+// make subbyte converts more efficient by allowing for efficient extraction
19
+// of subbyte elements from a 32bit register.
20
+template <typename IlvBlkLayout, typename T, typename S, int N,
21
+ FloatRoundStyle Round = FloatRoundStyle::round_to_nearest,
22
+ class Enable = void>
23
+struct InterleavedNumericArrayConverter {
24
+ using Converter = NumericArrayConverter<T, S, N, Round>;
25
+
26
+ using result_type = typename Converter::result_type;
27
+ using source_type = typename Converter::source_type;
28
+
29
+ CUTLASS_DEVICE
30
+ static result_type convert(source_type const& source) {
31
+ CUTE_INVALID_CONTROL_PATH(
32
+ "InterleavedNumericArrayConverter not implemented\n");
33
+ return {};
34
+ }
35
+
36
+ CUTLASS_DEVICE
37
+ result_type operator()(source_type const& s) const { return convert(s); }
38
+};
39
+
40
+template <typename IlvBlkLayout, typename T, typename S, int N,
41
+ FloatRoundStyle Round>
42
+struct InterleavedNumericArrayConverter<
43
+ IlvBlkLayout, T, S, N, Round,
44
+ std::enable_if_t<is_identity_layout<IlvBlkLayout>()>> {
45
+ using Converter = NumericArrayConverter<T, S, N, Round>;
46
+
47
+ using result_type = typename Converter::result_type;
48
+ using source_type = typename Converter::source_type;
49
+
50
+ CUTLASS_DEVICE
51
+ static result_type convert(source_type const& source) {
52
+ return Converter::convert(source);
53
+ }
54
+
55
+ CUTLASS_DEVICE
56
+ result_type operator()(source_type const& s) const { return convert(s); }
57
+};
58
+
59
+// TODO (LucasWilkinson): Implement
60
+// for Array<cutlass::float8_e4m3fn, N> <= Array<vllm_uint4b8_t, N>
61
+
62
+// ....
63
+
64
+template <typename RegConvert32bit, typename T, typename S, int N>
65
+struct ArrayConverterPacked32Bit {
66
+ using result_type = Array<T, N>;
67
+ using source_type = Array<S, N>;
68
+
69
+ using result_packed_8_t = Array<T, 8>;
70
+ using result_packed_4_t = Array<T, 4>;
71
+ using result_packed_2_t = Array<T, 2>;
72
+ using src_packed_8_t = Array<S, 8>;
73
+ using src_packed_4_t = Array<S, 4>;
74
+ using src_packed_2_t = Array<S, 2>;
75
+
76
+ static_assert(N % 2 == 0, "N must be a multiple of 2");
77
+ static_assert(cutlass::sizeof_bits_v<S> >= 4); // TODO: add 16 packed sources
78
+ static_assert(32 % cutlass::sizeof_bits_v<S> == 0);
79
+ static constexpr auto src_elems_per_32bit_reg =
80
+ 32 / cutlass::sizeof_bits_v<S>;
81
+
82
+ // Maybe not Valid. ScalarConverter will not actually work unless
83
+ // NumericConverter<T, S, Round> is implemented. However it won't be used
84
+ // anyways since we assert N % 2 == 0, just here for compliance with
85
+ // VectorizedConverter.
86
+ using ScalarConverter = NumericConverter<T, S>;
87
+
88
+ template <typename PackedSrc>
89
+ CUTLASS_DEVICE static uint32_t to_reg(PackedSrc const& source) {
90
+ if constexpr (sizeof(PackedSrc) == 1) {
91
+ return static_cast<uint32_t>(reinterpret_cast<const uint8_t&>(source));
92
+ } else if constexpr (sizeof(PackedSrc) == 2) {
93
+ return static_cast<uint32_t>(reinterpret_cast<const uint16_t&>(source));
94
+ } else {
95
+ static_assert(sizeof(PackedSrc) == 4);
96
+ return reinterpret_cast<const uint32_t&>(source);
97
+ }
98
+ }
99
+
100
+ // The core converter uses bit tricks to construct a known FP16 number, then
101
+ // does a subtraction in FP16 for the final result.
102
+ template <typename PackedResultType, typename PackedSrcType>
103
+ CUTLASS_DEVICE static PackedResultType packed_convert(
104
+ PackedSrcType const& source) {
105
+ static_assert(PackedSrcType::kElements == PackedResultType::kElements);
106
+ static_assert(PackedResultType::kElements == 2 ||
107
+ PackedResultType::kElements == 4 ||
108
+ PackedResultType::kElements == 8,
109
+ "Invalid PackedResultType must be 2, 4 or 8.");
110
+ static_assert(std::is_same_v<typename PackedSrcType::Element, S>);
111
+ static_assert(std::is_same_v<typename PackedResultType::Element, T>);
112
+
113
+ return RegConvert32bit::template convert<PackedResultType>(to_reg(source));
114
+ }
115
+
116
+ friend class detail::VectorizedConverter;
117
+
118
+ public:
119
+ CUTLASS_DEVICE static result_type convert(source_type const& source) {
120
+ result_type result;
121
+ using ConverterType =
122
+ ArrayConverterPacked32Bit<RegConvert32bit,
123
+ typename result_type::Element,
124
+ typename source_type::Element, N>;
125
+
126
+ if constexpr (src_elems_per_32bit_reg >= 8) {
127
+ detail::VectorizedConverter::convert<
128
+ ConverterType, result_packed_8_t, src_packed_8_t, result_packed_4_t,
129
+ src_packed_4_t, result_packed_2_t, src_packed_2_t>(result, source);
130
+ } else if constexpr (src_elems_per_32bit_reg >= 4) {
131
+ detail::VectorizedConverter::convert<ConverterType, result_packed_4_t,
132
+ src_packed_4_t, result_packed_2_t,
133
+ src_packed_2_t>(result, source);
134
+ } else {
135
+ detail::VectorizedConverter::convert<ConverterType, result_packed_2_t,
136
+ src_packed_2_t>(result, source);
137
+ }
138
+
139
+ return result;
140
+ }
141
+};
142
+
143
+// for Array<cutlass::half_t, N> <= Array<vllm_uint4b8_t, N>
144
+template <FloatRoundStyle Round, int N>
145
+struct NumericArrayConverter<cutlass::half_t, vllm_uint4b8_t, N, Round> {
146
+ using result_type = Array<cutlass::half_t, N>;
147
+ using source_type = Array<vllm_uint4b8_t, N>;
148
+
149
+ struct RegConvert {
150
+ template <typename PackedResultType>
151
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
152
+ using RegArray =
153
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
154
+ sizeof(PackedResultType)>;
155
+ RegArray r;
156
+
157
+ // Below constructs the following temporary:
158
+ // fp16s_01 = {0x00, i4_01, 0x00, i4_01}
159
+ // fp16s_23 = {0x00, i4_23, 0x00, i4_23}
160
+ // fp16s_45 = {0x00, i4_45, 0x00, i4_45}
161
+ // fp16s_67 = {0x00, i4_67, 0x00, i4_67}
162
+ // We use inline asm instead of __byte_perm intrinsic since we don't want
163
+ // the documented (& 0x7) on the index. NVCC might be able to optimize it
164
+ // out since the index is a constexpr, but we choose to be safe about it
165
+ // here.
166
+ uint32_t prmt_indices[4] = {0x4040, 0x4141, 0x4242, 0x4343};
167
+ static_assert(RegArray::kElements <= 4,
168
+ "Too many inputs for F16 -> I4 vector converter");
169
+ CUTLASS_PRAGMA_UNROLL
170
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
171
+ asm volatile(
172
+ "{\n"
173
+ " prmt.b32 %0, %1, %2, %3;\n"
174
+ "}\n"
175
+ : "=r"(r[ii])
176
+ : "r"(src), "n"(0), "r"(prmt_indices[ii]));
177
+ }
178
+
179
+ // Since the stored 4bit values are biased by 8 we get stored_val = (x+8)
180
+ // we are trying to construct x and a fp16 value
181
+ // The below XOR does the following:
182
+ // 1) Sets the exponent bits of the FP16 to the correct value for the
183
+ // FP16 magic_num. We will be constructing {1024+16*(x1+8), 1024+(x0+8)},
184
+ // where x1 in the high nibble and x0 is the low nibble then using hfma
185
+ // to subtract 1032 from that
186
+ // The AND does the following:
187
+ // 1) Clear the set bits for the int4 we will ignore.
188
+ // We use lop3 so that we can use 1 instruction for AND and XOR.
189
+ static constexpr uint32_t xor_mask = 0x64006400;
190
+ static constexpr uint32_t and_mask = 0xFFF0FF0F;
191
+ static constexpr uint32_t immLut = (0xf0 & 0xcc) ^ 0xaa;
192
+
193
+ // For each operand, computes:
194
+ // r[i] = (r[i] & and_mask) ^ xor_mask
195
+ CUTLASS_PRAGMA_UNROLL
196
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
197
+ asm volatile(
198
+ "{\n"
199
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
200
+ "}\n"
201
+ : "+r"(r[ii])
202
+ : "n"(and_mask), "n"(xor_mask), "n"(immLut));
203
+ }
204
+
205
+ // We will issue 2 hfmas that do the following:
206
+ // {x1, x0} = {1024+16*(x1+8), 1024+(x0+8)} * {1/16, 1} - {72, 1032}
207
+ // = {x1 + 1152, x0 + 1032} * {1/16, 1} - {72, 1032}
208
+ static constexpr uint32_t hfma_bias_rep = 0xD480E408; // {72, 1032}
209
+ static constexpr uint32_t hfma_scale_rep = 0x2C003C00; // {1 / 16, 1}
210
+
211
+ const half2& hfma_bias = reinterpret_cast<const half2&>(hfma_bias_rep);
212
+ const half2& hfma_scale = reinterpret_cast<const half2&>(hfma_scale_rep);
213
+ CUTLASS_PRAGMA_UNROLL
214
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
215
+ half2& fp16x2_val = reinterpret_cast<__half2&>(r[ii]);
216
+ fp16x2_val = __hfma2(hfma_scale, fp16x2_val, hfma_bias);
217
+ }
218
+
219
+ return reinterpret_cast<PackedResultType&>(r);
220
+ };
221
+ };
222
+
223
+ public:
224
+ CUTLASS_DEVICE
225
+ static result_type convert(source_type const& source) {
226
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
227
+ typename source_type::Element,
228
+ N>::convert(source);
229
+ }
230
+
231
+ CUTLASS_DEVICE
232
+ result_type operator()(source_type const& s) const { return convert(s); }
233
+};
234
+
235
+// for Array<cutlass::half_t, N> <= Array<vllm_uint4b8_t, N>
236
+// for IlvdLayout: (2, 4):(4, 1)
237
+template <FloatRoundStyle Round, int N>
238
+struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
239
+ cutlass::half_t, vllm_uint4b8_t, N,
240
+ Round, void> {
241
+ using IlvdLayout = Layout<Shape<_2, _4>, Stride<_4, _1>>;
242
+ static_assert(N % size(IlvdLayout{}) == 0);
243
+
244
+ using result_type = Array<cutlass::half_t, N>;
245
+ using source_type = Array<vllm_uint4b8_t, N>;
246
+
247
+ static FloatRoundStyle const round_style = Round;
248
+
249
+ private:
250
+ struct RegConvert {
251
+ template <typename PackedResultType>
252
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
253
+ using RegArray =
254
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
255
+ sizeof(PackedResultType)>;
256
+ RegArray r;
257
+
258
+ static_assert(PackedResultType::kElements <= size(IlvdLayout{}));
259
+ static constexpr uint32_t xor_mask = 0x64006400;
260
+
261
+ for (int ii = 0; ii < RegArray::kElements; ii += 2) {
262
+ auto src_ = src >> (4 * (ii));
263
+ r[ii + 0] = src_;
264
+ r[ii + 1] = src_;
265
+
266
+ static constexpr uint32_t and_xor_imm_lut = (0xf0 & 0xcc) ^ 0xaa;
267
+
268
+ static constexpr uint32_t low_nib_mask = 0x000F000F;
269
+ static constexpr uint32_t high_nib_mask = 0x00F000F0;
270
+
271
+ asm volatile(
272
+ "{\n"
273
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
274
+ "}\n"
275
+ : "+r"(r[ii + 0])
276
+ : "n"(low_nib_mask), "n"(xor_mask), "n"(and_xor_imm_lut));
277
+
278
+ asm volatile(
279
+ "{\n"
280
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
281
+ "}\n"
282
+ : "+r"(r[ii + 1])
283
+ : "n"(high_nib_mask), "n"(xor_mask), "n"(and_xor_imm_lut));
284
+
285
+ // For low nibble:
286
+ // {x1, x0} = {1024+(x1+8), 1024+(x0+8)} * {1, 1} - {1032, 1032}
287
+ // For high nibble:
288
+ // {x1, x0} = {1024+16*(x1+8), 1024+16*(x0+8)} * {1/16, 1/16}
289
+ // - {72, 72}
290
+ static constexpr uint32_t low_nib_bias = 0x64086408; // {1032, 1032}
291
+ static constexpr uint32_t high_nib_scale = 0x2C002C00; // {1/16, 1/16}
292
+ static constexpr uint32_t high_nib_bias = 0xD480D480; // {-72, -72}
293
+
294
+ {
295
+ half2& fp16x2_val = reinterpret_cast<__half2&>(r[ii + 0]);
296
+ fp16x2_val =
297
+ __hsub2(fp16x2_val, reinterpret_cast<const half2&>(low_nib_bias));
298
+ }
299
+
300
+ {
301
+ half2& fp16x2_val = reinterpret_cast<__half2&>(r[ii + 1]);
302
+ fp16x2_val = __hfma2(fp16x2_val,
303
+ reinterpret_cast<const half2&>(high_nib_scale),
304
+ reinterpret_cast<const half2&>(high_nib_bias));
305
+ }
306
+ }
307
+
308
+ return reinterpret_cast<PackedResultType&>(r);
309
+ };
310
+ };
311
+
312
+ public:
313
+ CUTLASS_DEVICE
314
+ static result_type convert(source_type const& source) {
315
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
316
+ typename source_type::Element,
317
+ N>::convert(source);
318
+ }
319
+
320
+ CUTLASS_DEVICE
321
+ result_type operator()(source_type const& s) const { return convert(s); }
322
+};
323
+
324
+// for Array<cutlass::half_t, N> <= Array<uint4_t, N>
325
+// for IlvdLayout: (2, 4):(4, 1)
326
+template <FloatRoundStyle Round, int N>
327
+struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
328
+ cutlass::half_t, uint4_t, N, Round,
329
+ void> {
330
+ using IlvdLayout = Layout<Shape<_2, _4>, Stride<_4, _1>>;
331
+ static_assert(N % size(IlvdLayout{}) == 0);
332
+
333
+ using result_type = Array<cutlass::half_t, N>;
334
+ using source_type = Array<uint4_t, N>;
335
+
336
+ static FloatRoundStyle const round_style = Round;
337
+
338
+ private:
339
+ struct RegConvert {
340
+ template <typename PackedResultType>
341
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
342
+ using RegArray =
343
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
344
+ sizeof(PackedResultType)>;
345
+ RegArray r;
346
+
347
+ static_assert(PackedResultType::kElements <= size(IlvdLayout{}));
348
+ static constexpr uint32_t xor_mask = 0x64006400;
349
+
350
+ for (int ii = 0; ii < RegArray::kElements; ii += 2) {
351
+ auto src_ = src >> (4 * (ii));
352
+ r[ii + 0] = src_;
353
+ r[ii + 1] = src_;
354
+
355
+ static constexpr uint32_t and_xor_imm_lut = (0xf0 & 0xcc) ^ 0xaa;
356
+
357
+ static constexpr uint32_t low_nib_mask = 0x000F000F;
358
+ static constexpr uint32_t high_nib_mask = 0x00F000F0;
359
+
360
+ asm volatile(
361
+ "{\n"
362
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
363
+ "}\n"
364
+ : "+r"(r[ii + 0])
365
+ : "n"(low_nib_mask), "n"(xor_mask), "n"(and_xor_imm_lut));
366
+
367
+ asm volatile(
368
+ "{\n"
369
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
370
+ "}\n"
371
+ : "+r"(r[ii + 1])
372
+ : "n"(high_nib_mask), "n"(xor_mask), "n"(and_xor_imm_lut));
373
+
374
+ // For low nibble:
375
+ // {x1, x0} = {1024+x1, 1024+x0} - {1024, 1024}
376
+ // For high nibble:
377
+ // {x1, x0} = {1024+16*x1, 1024+16*x0} * {1/16, 1/16} - {64, 64}
378
+ static constexpr uint32_t low_nib_bias = 0x64006400; // {1024, 1024}
379
+ static constexpr uint32_t high_nib_scale = 0x2C002C00; // {1/16, 1/16}
380
+ static constexpr uint32_t high_nib_bias = 0xD400D400; // {-64, -64}
381
+
382
+ {
383
+ half2& fp16x2_val = reinterpret_cast<__half2&>(r[ii + 0]);
384
+ fp16x2_val =
385
+ __hsub2(fp16x2_val, reinterpret_cast<const half2&>(low_nib_bias));
386
+ }
387
+
388
+ {
389
+ half2& fp16x2_val = reinterpret_cast<__half2&>(r[ii + 1]);
390
+ fp16x2_val = __hfma2(fp16x2_val,
391
+ reinterpret_cast<const half2&>(high_nib_scale),
392
+ reinterpret_cast<const half2&>(high_nib_bias));
393
+ }
394
+ }
395
+
396
+ return reinterpret_cast<PackedResultType&>(r);
397
+ };
398
+ };
399
+
400
+ public:
401
+ CUTLASS_DEVICE
402
+ static result_type convert(source_type const& source) {
403
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
404
+ typename source_type::Element,
405
+ N>::convert(source);
406
+ }
407
+
408
+ CUTLASS_DEVICE
409
+ result_type operator()(source_type const& s) const { return convert(s); }
410
+};
411
+
412
+// for Array<cutlass::half_t, N> <= Array<vllm_uint8b128_t, N>
413
+template <FloatRoundStyle Round, int N>
414
+struct NumericArrayConverter<cutlass::half_t, vllm_uint8b128_t, N, Round> {
415
+ using result_type = Array<cutlass::half_t, N>;
416
+ using source_type = Array<vllm_uint8b128_t, N>;
417
+
418
+ struct RegConvert {
419
+ template <typename PackedResultType>
420
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
421
+ // Hold output FP16s in reg. We need 1 reg for every 2 elements
422
+ using RegArray =
423
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
424
+ sizeof(PackedResultType)>;
425
+ RegArray r;
426
+
427
+ uint32_t const prmt_indices[2] = {0x5150, 0x5352};
428
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
429
+
430
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
431
+ asm volatile("prmt.b32 %0,%1,%2,%3;\n"
432
+ : "=r"(r[ii])
433
+ : "r"(src), "n"(start_byte_for_fp16),
434
+ "r"(prmt_indices[ii]));
435
+ }
436
+
437
+ // -128 is folded into bias subtraction, i.e. the 0x80 in the low bytes
438
+ static constexpr uint32_t bias_rep = 0x64806480;
439
+ const half2& bias = reinterpret_cast<const half2&>(bias_rep);
440
+ CUTLASS_PRAGMA_UNROLL
441
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
442
+ half2& fp16x2_val = reinterpret_cast<__half2&>(r[ii]);
443
+ fp16x2_val = __hsub2(fp16x2_val, bias);
444
+ }
445
+
446
+ return reinterpret_cast<PackedResultType&>(r);
447
+ };
448
+ };
449
+
450
+ public:
451
+ CUTLASS_DEVICE
452
+ static result_type convert(source_type const& source) {
453
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
454
+ typename source_type::Element,
455
+ N>::convert(source);
456
+ }
457
+
458
+ CUTLASS_DEVICE
459
+ result_type operator()(source_type const& s) const { return convert(s); }
460
+};
461
+
462
+// for Array<cutlass::float, N> <= Array<vllm_uint8b128_t, N>
463
+template <FloatRoundStyle Round, int N>
464
+struct NumericArrayConverter<float, vllm_uint8b128_t, N, Round> {
465
+ using result_type = Array<float, N>;
466
+ using source_type = Array<vllm_uint8b128_t, N>;
467
+ static FloatRoundStyle const round_style = Round;
468
+
469
+ private:
470
+ struct RegConvert {
471
+ template <typename PackedResultType>
472
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
473
+ PackedResultType r;
474
+
475
+ // __byte_perm simulates the add.u32 0x4B000000 to every u8 element of
476
+ // u8x4 source and stores the result in r (without introducing extra
477
+ // cvt.u32.u8 instruction)
478
+ uint32_t const prmt_indices[4] = {0x7650, 0x7651, 0x7652, 0x7653};
479
+ uint32_t* result_as_int = reinterpret_cast<uint32_t*>(&r);
480
+ for (int ii = 0; ii < PackedResultType::kElements; ++ii) {
481
+ result_as_int[ii] = __byte_perm(src, 0x4B000000, prmt_indices[ii]);
482
+ // Subtract the magic number 0x4B000000 from tmp in floating-point
483
+ // arithmetic to obtain final result
484
+ r[ii] -= (8388608.f + 128.f); // fold in -128 bias
485
+ }
486
+
487
+ return r;
488
+ };
489
+ };
490
+
491
+ public:
492
+ CUTLASS_DEVICE
493
+ static result_type convert(source_type const& source) {
494
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
495
+ typename source_type::Element,
496
+ N>::convert(source);
497
+ }
498
+
499
+ CUTLASS_DEVICE
500
+ result_type operator()(source_type const& s) const { return convert(s); }
501
+};
502
+
503
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800)
504
+
505
+// for Array<cutlass::bfloat16_t, N> <= Array<vllm_uint4b8_t, N>
506
+template <FloatRoundStyle Round, int N>
507
+struct NumericArrayConverter<cutlass::bfloat16_t, vllm_uint4b8_t, N, Round> {
508
+ using result_type = Array<cutlass::bfloat16_t, N>;
509
+ using source_type = Array<vllm_uint4b8_t, N>;
510
+
511
+ static FloatRoundStyle const round_style = Round;
512
+
513
+ private:
514
+ struct RegConvert {
515
+ template <typename PackedResultType>
516
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src_reg) {
517
+ // Hold output BF16s in reg. We need 1 reg for every 2 elements
518
+ using RegArray =
519
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
520
+ sizeof(PackedResultType)>;
521
+ RegArray r;
522
+ uint32_t src_reg_shifted = src_reg >> 4;
523
+
524
+ // Below constructs the following temporary:
525
+ uint32_t const prmt_indices[4] = {0xF4F0, 0xF5F1, 0xF6F2, 0xF7F3};
526
+ static_assert(RegArray::kElements <= 4,
527
+ "Too many inputs for uint4b8_t -> BF16 vector converter");
528
+ CUTLASS_PRAGMA_UNROLL
529
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
530
+ asm volatile(
531
+ "{\n"
532
+ " prmt.b32 %0, %1, %2, %3;\n"
533
+ "}\n"
534
+ : "=r"(r[ii])
535
+ : "r"(src_reg), "r"(src_reg_shifted), "r"(prmt_indices[ii]));
536
+ }
537
+
538
+ // Since the stored 4bit values are biased by 8 we get stored_val = (x+8)
539
+ // we are trying to construct x and a BF16 value
540
+ // The below XOR does the following:
541
+ // 1) Sets the exponent bits of the BF16 to the correct value for the
542
+ // BF16 magic_num. We will be constructing {128 + (x1+8), 128 + (x0+8)}
543
+ // and subtracting 136 to get {x1, x0}
544
+ static constexpr uint32_t xor_mask = 0x43004300;
545
+ static constexpr uint32_t and_mask = 0x000F000F;
546
+ static constexpr uint32_t immLut = (0xf0 & 0xcc) ^ 0xaa;
547
+
548
+ // For each operand, computes:
549
+ // r[i] = (r[i] & and_mask) ^ xor_mask
550
+ CUTLASS_PRAGMA_UNROLL
551
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
552
+ asm volatile(
553
+ "{\n"
554
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
555
+ "}\n"
556
+ : "+r"(r[ii])
557
+ : "n"(and_mask), "n"(xor_mask), "n"(immLut));
558
+ }
559
+
560
+ // We will issue 2 bfmas that do the following:
561
+ // high BF16:
562
+ // hi_bf16 - 136, lo_bf16 - 136
563
+
564
+ // This is the BF16 {136, 136} represented as an integer.
565
+ static constexpr uint32_t bias_rep = 0x43084308;
566
+ const __nv_bfloat162& bias =
567
+ reinterpret_cast<const __nv_bfloat162&>(bias_rep);
568
+
569
+ CUTLASS_PRAGMA_UNROLL
570
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
571
+ __nv_bfloat162& bf16x2_val = reinterpret_cast<__nv_bfloat162&>(r[ii]);
572
+ bf16x2_val = __hsub2(bf16x2_val, bias);
573
+ }
574
+
575
+ return reinterpret_cast<PackedResultType&>(r);
576
+ }
577
+ };
578
+
579
+ public:
580
+ CUTLASS_DEVICE
581
+ static result_type convert(source_type const& source) {
582
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
583
+ typename source_type::Element,
584
+ N>::convert(source);
585
+ }
586
+
587
+ CUTLASS_DEVICE
588
+ result_type operator()(source_type const& s) const { return convert(s); }
589
+};
590
+
591
+// for Array<cutlass::bfloat16_t, N> <= Array<vllm_uint4b8_t, N>
592
+// for IlvdLayout: (2, 4):(4, 1)
593
+template <FloatRoundStyle Round, int N>
594
+struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
595
+ cutlass::bfloat16_t, vllm_uint4b8_t, N,
596
+ Round, void> {
597
+ using IlvdLayout = Layout<Shape<_2, _4>, Stride<_4, _1>>;
598
+ static_assert(N % size(IlvdLayout{}) == 0);
599
+
600
+ using result_type = Array<cutlass::bfloat16_t, N>;
601
+ using source_type = Array<vllm_uint4b8_t, N>;
602
+
603
+ private:
604
+ struct RegConvert {
605
+ template <typename PackedResultType>
606
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
607
+ using RegArray =
608
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
609
+ sizeof(PackedResultType)>;
610
+ RegArray r;
611
+
612
+ static_assert(PackedResultType::kElements <= size(IlvdLayout{}));
613
+ static constexpr uint32_t or_mask = 0x43004300;
614
+
615
+ // Unlike float16 where the mantissa is large enough to contain 2
616
+ // nibbles, bfloat16 can only fit one, so we can only convert one
617
+ // nibble at a time
618
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
619
+ r[ii] = src >> (4 * ii);
620
+
621
+ static constexpr uint32_t and_or_imm_lut = (0xf0 & 0xcc) | 0xaa;
622
+ static constexpr uint32_t low_nib_mask = 0x000F000F;
623
+
624
+ asm volatile(
625
+ "{\n"
626
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
627
+ "}\n"
628
+ : "+r"(r[ii + 0])
629
+ : "n"(low_nib_mask), "n"(or_mask), "n"(and_or_imm_lut));
630
+
631
+ // For low nibble:
632
+ // {x1, x0} = {128+(x1+8), 128+(x0+8)} * {1, 1} - {136, 136}
633
+ static constexpr uint32_t low_nib_bias = 0x43084308; // {136, 136}
634
+
635
+ {
636
+ __nv_bfloat162& fp16x2_val = reinterpret_cast<__nv_bfloat162&>(r[ii]);
637
+ fp16x2_val =
638
+ __hsub2(fp16x2_val,
639
+ reinterpret_cast<const __nv_bfloat162&>(low_nib_bias));
640
+ }
641
+ }
642
+
643
+ return reinterpret_cast<PackedResultType&>(r);
644
+ };
645
+ };
646
+
647
+ public:
648
+ CUTLASS_DEVICE
649
+ static result_type convert(source_type const& source) {
650
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
651
+ typename source_type::Element,
652
+ N>::convert(source);
653
+ }
654
+
655
+ CUTLASS_DEVICE
656
+ result_type operator()(source_type const& s) const { return convert(s); }
657
+};
658
+
659
+// for Array<cutlass::bfloat16_t, N> <= Array<uint4_t, N>
660
+// for IlvdLayout: (2, 4):(4, 1)
661
+template <FloatRoundStyle Round, int N>
662
+struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
663
+ cutlass::bfloat16_t, uint4_t, N, Round,
664
+ void> {
665
+ using IlvdLayout = Layout<Shape<_2, _4>, Stride<_4, _1>>;
666
+ static_assert(N % size(IlvdLayout{}) == 0);
667
+
668
+ using result_type = Array<cutlass::bfloat16_t, N>;
669
+ using source_type = Array<uint4_t, N>;
670
+
671
+ private:
672
+ struct RegConvert {
673
+ template <typename PackedResultType>
674
+ CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
675
+ using RegArray =
676
+ cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
677
+ sizeof(PackedResultType)>;
678
+ RegArray r;
679
+
680
+ static_assert(PackedResultType::kElements <= size(IlvdLayout{}));
681
+ static constexpr uint32_t or_mask = 0x43004300;
682
+
683
+ // Unlike float16 where the mantissa is large enough to contain 2
684
+ // nibbles, bfloat16 can only fit one, so we can only convert one
685
+ // nibble at a time
686
+ for (int ii = 0; ii < RegArray::kElements; ++ii) {
687
+ r[ii] = src >> (4 * ii);
688
+
689
+ static constexpr uint32_t and_or_imm_lut = (0xf0 & 0xcc) | 0xaa;
690
+ static constexpr uint32_t low_nib_mask = 0x000F000F;
691
+
692
+ asm volatile(
693
+ "{\n"
694
+ " lop3.b32 %0, %0, %1, %2, %3;\n"
695
+ "}\n"
696
+ : "+r"(r[ii])
697
+ : "n"(low_nib_mask), "n"(or_mask), "n"(and_or_imm_lut));
698
+
699
+ // For low nibble:
700
+ // {x1, x0} = {128 + x1, 128 + x0} * {1, 1} - {128, 128}
701
+ static constexpr uint32_t low_nib_bias = 0x43004300; // {128, 128}
702
+
703
+ {
704
+ __nv_bfloat162& fp16x2_val = reinterpret_cast<__nv_bfloat162&>(r[ii]);
705
+ fp16x2_val =
706
+ __hsub2(fp16x2_val,
707
+ reinterpret_cast<const __nv_bfloat162&>(low_nib_bias));
708
+ }
709
+ }
710
+
711
+ return reinterpret_cast<PackedResultType&>(r);
712
+ };
713
+ };
714
+
715
+ public:
716
+ CUTLASS_DEVICE
717
+ static result_type convert(source_type const& source) {
718
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
719
+ typename source_type::Element,
720
+ N>::convert(source);
721
+ }
722
+
723
+ CUTLASS_DEVICE
724
+ result_type operator()(source_type const& s) const { return convert(s); }
725
+};
726
+
727
+// for Array<cutlass::bfloat16_t, N> <= Array<vllm_uint8b128_t, N>
728
+template <FloatRoundStyle Round, int N>
729
+struct NumericArrayConverter<cutlass::bfloat16_t, vllm_uint8b128_t, N, Round> {
730
+ using result_type = Array<cutlass::bfloat16_t, N>;
731
+ using source_type = Array<vllm_uint8b128_t, N>;
732
+ static FloatRoundStyle const round_style = Round;
733
+
734
+ private:
735
+ using result_packed_4_t = Array<cutlass::bfloat16_t, 4>;
736
+ using result_packed_2_t = Array<cutlass::bfloat16_t, 2>;
737
+ using src_packed_4_t = Array<vllm_uint8b128_t, 4>;
738
+ using src_packed_2_t = Array<vllm_uint8b128_t, 2>;
739
+
740
+ // Not Valid, not supported, only here to satisfy the interface and to avoid
741
+ // a compile error. ScalarConverter will not actually work until
742
+ // NumericConverter<cutlass::bfloat16_t, vllm_uint8b128_t, Round> is
743
+ // implemented
744
+ using ScalarConverter =
745
+ NumericConverter<cutlass::bfloat16_t, vllm_uint8b128_t, Round>;
746
+
747
+ template <typename PackedResultType, typename PackedSrcType>
748
+ CUTLASS_DEVICE static PackedResultType packed_convert(
749
+ PackedSrcType const& source) {
750
+ static_assert(
751
+ (platform::is_same<PackedSrcType, src_packed_2_t>::value &&
752
+ platform::is_same<PackedResultType, result_packed_2_t>::value) ||
753
+ (platform::is_same<PackedSrcType, src_packed_4_t>::value &&
754
+ platform::is_same<PackedResultType, result_packed_4_t>::value),
755
+ "Invalid PackedSrcType/PackedResultType must be 2 or 4 to use private "
756
+ "convert dispatch.");
757
+
758
+ NumericArrayConverter<float, vllm_uint8b128_t, PackedResultType::kElements,
759
+ Round>
760
+ convert_uint8_to_f32;
761
+ Array<float, PackedResultType::kElements> tmp =
762
+ convert_uint8_to_f32(source);
763
+ NumericArrayConverter<cutlass::bfloat16_t, float,
764
+ PackedResultType::kElements, Round>
765
+ convert_f32_to_bf16_;
766
+ return convert_f32_to_bf16_(tmp);
767
+ }
768
+
769
+ friend class detail::VectorizedConverter;
770
+
771
+ public:
772
+ CUTLASS_DEVICE
773
+ static result_type convert(source_type const& source) {
774
+ result_type result;
775
+ using ConverterType =
776
+ NumericArrayConverter<typename result_type::Element,
777
+ typename source_type::Element, N, Round>;
778
+ detail::VectorizedConverter::convert<ConverterType, result_packed_4_t,
779
+ src_packed_4_t, result_packed_2_t,
780
+ src_packed_2_t>(result, source);
781
+
782
+ return result;
783
+ }
784
+
785
+ CUTLASS_DEVICE
786
+ result_type operator()(source_type const& s) const { return convert(s); }
787
+};
788
+
789
+#endif
790
+
791
+/////////////////////////////////////////////////////////////////////////////////////////////////
792
+
793
+} // namespace cutlass
794
+
795
+/////////////////////////////////////////////////////////////////////////////////////////////////
csrc/ops.h
+19
@@ -83,6 +83,25 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
83
torch::Tensor& b_scales, torch::Tensor& workspace,
84
int64_t size_m, int64_t size_n, int64_t size_k);
85
86
+namespace machete {
87
+
88
+std::vector<std::string> supported_schedules(
89
+ vllm::ScalarTypeTorchPtr const& btype);
90
+
91
+torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
92
+ vllm::ScalarTypeTorchPtr const& btype,
93
+ c10::optional<torch::Tensor> const& scales,
94
+ c10::optional<torch::Tensor> const& zeros,
95
+ c10::optional<int64_t> group_size,
96
+ c10::optional<torch::Tensor> const& C,
97
+ c10::optional<double> alpha, c10::optional<double> beta,
98
+ c10::optional<std::string> schedule);
99
+
100
+torch::Tensor prepack_B(torch::Tensor const& B,
101
+ vllm::ScalarTypeTorchPtr const& btype);
102
+
103
+}; // namespace machete
104
+
105
torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
106
torch::Tensor& b_meta,
107
torch::Tensor& b_scales,
csrc/quantization/machete/Readme.md
new
+45
@@ -0,0 +1,45 @@
1
+# Machete (Mixed Precision Cutlass-Based GEMM)
2
+
3
+Machete is a spiritual successor to the Marlin kernel but optimized for Hopper architectures and based on Cutlass. Being based on Cutlass, new type pairs and epilogues are easier to add compared to Marlin.
4
+
5
+## Overview
6
+
7
+Machete effectively performs
8
+
9
+```
10
+scale_type = w_s.dtype
11
+compute_type = a.dtype
12
+out = (w_q.to(scale_type) * w_s - w_z.to(scale_type)) @ a
13
+```
14
+
15
+Where `w_q` is a quantized weight matrix, `w_s` is the quantization scales, and
16
+`w_z` is the quantization zeropoints.
17
+
18
+> **_NOTE:_** `w_z` is added after the scales so we can
19
+use FMA operations, but this means they must have the scales pre-applied if the
20
+supplied zeropoints assume that they will be subtracted before the scales are
21
+applied.
22
+
23
+## API
24
+
25
+The main optimization within Machete is prepacking the weight matrix to more closely match the tensor core layouts, allowing for wider shared memory loads when loading the weight matrix. This means that the weight matrix must be prepacked before calling `machete_gemm`. The flow looks something like:
26
+
27
+```
28
+from vllm import _custom_ops as ops
29
+
30
+...
31
+W_q_packed = ops.machete_prepack_B(w_q, wtype)
32
+output = ops.machete_gemm(
33
+ a,
34
+ b_q=W_q_packed,
35
+ b_type=wtype,
36
+ b_scales=w_s,
37
+ b_group_size=group_size
38
+)
39
+```
40
+
41
+## Code Generation
42
+
43
+Since Machete is based on Cutlass, we can generate multiple type pairs and different tile shapes using the same kernel template. We generate multiple instantiations of this template using `generate.py`.
44
+
45
+New type pairs (`TypeConfig`s) can be appended to `impl_configs` (in `generate()`), and these will get automatically generated (assuming they can be supported without issues). For each `TypeConfig`, you must also provide an `ImplConfig`, which bundles a `TypeConfig` with a list of `ScheduleConfig`s, `Specialization`s, and a default heuristic. The `ScheduleConfig`s (which contain info on tile shapes, tile scheduler, etc.) can perform differently for different problem shapes, and there is almost never one `ScheduleConfig` that works well for all problem shapes, so it is generally beneficial to generate different `ScheduleConfig`s for different potential problem shapes. This is where the heuristic comes in. For each `TypeConfig`, a default heuristic should be provided. This maps different problem shapes to different `ScheduleConfig`s and is used when the user does not provide the `schedule` parameter to `machete_gemm`. The `Specialization`s define what feature combinations to generate, i.e., `with_zeropoints`, `with_scales`, etc. We can reduce compile times and the final binary size by limiting the set of feature combinations we generate.
\ No newline at end of file
csrc/quantization/machete/generate.py
new
+446
@@ -0,0 +1,446 @@
1
+import itertools
2
+import math
3
+import os
4
+import shutil
5
+from collections.abc import Iterable
6
+from dataclasses import dataclass
7
+from typing import List, Optional, Tuple, Union
8
+
9
+import jinja2
10
+# yapf conflicts with isort for this block
11
+# yapf: disable
12
+from vllm_cutlass_library_extension import (DataType, EpilogueScheduleTag,
13
+ EpilogueScheduleType,
14
+ MixedInputKernelScheduleType,
15
+ TileSchedulerTag,
16
+ TileSchedulerType, VLLMDataType,
17
+ VLLMDataTypeNames, VLLMDataTypeTag,
18
+ VLLMKernelScheduleTag)
19
+
20
+# yapf: enable
21
+
22
+#
23
+# Generator templating
24
+#
25
+
26
+DISPATCH_TEMPLATE = """
27
+#include "../machete_mm_launcher.cuh"
28
+
29
+namespace machete {
30
+using GemmDispatcher_ = GemmDispatcher<
31
+ {{DataTypeTag[type_config.element_a]}}, // ElementA
32
+ {{DataTypeTag[type_config.element_b]}}, // ElementB
33
+ {{DataTypeTag[type_config.element_d]}}, // ElementD
34
+ {{DataTypeTag[type_config.accumulator]}}, // Accumulator
35
+ {{DataTypeTag[type_config.element_b_scale]}}, // Scales
36
+ {{DataTypeTag[type_config.element_b_zeropoint]}}>; // Zeropoints
37
+
38
+{% for s in schedules %}extern torch::Tensor
39
+impl_{{type_name}}_sch_{{ gen_sch_name(s) }}(PyTorchArguments args);
40
+{% endfor %}
41
+template <>
42
+torch::Tensor GemmDispatcher_::dispatch(PyTorchArguments args) {
43
+ [[maybe_unused]] auto M = args.A.size(0);
44
+ [[maybe_unused]] auto N = args.B.size(1);
45
+ [[maybe_unused]] auto K = args.A.size(1);
46
+
47
+ if (!args.schedule) {
48
+ {%- for cond, s in heuristic %}
49
+ {%if cond is not none%}if ({{cond}})
50
+ {%- else %}else
51
+ {%- endif %}
52
+ return impl_{{ type_name }}_sch_{{ gen_sch_name(s) }}(args);{% endfor %}
53
+ }
54
+
55
+ {% for s in schedules %}
56
+ if (*args.schedule == "{{ gen_sch_name(s) }}") {
57
+ return impl_{{ type_name }}_sch_{{ gen_sch_name(s) }}(args);
58
+ }
59
+ {% endfor %}
60
+ TORCH_CHECK_NOT_IMPLEMENTED(false, "machete_gemm(..) is not implemented for "
61
+ "schedule = ", *args.schedule);
62
+}
63
+
64
+template <>
65
+std::vector<std::string> GemmDispatcher_::supported_schedules() {
66
+ return {
67
+ {% for s in schedules -%}
68
+ "{{ gen_sch_name(s) }}"{{ ",
69
+ " if not loop.last }}{%- endfor %}
70
+ };
71
+}
72
+
73
+}; // namespace machete
74
+"""
75
+
76
+IMPL_TEMPLATE = """
77
+#include "../machete_mm_launcher.cuh"
78
+
79
+namespace machete {
80
+template <typename Config, bool with_C, bool with_scales, bool with_zeropoints>
81
+using Kernel = MacheteKernelTemplate<
82
+ {{DataTypeTag[type_config.element_a]}}, // ElementA
83
+ {{DataTypeTag[type_config.element_b]}}, // ElementB
84
+ {{DataTypeTag[type_config.element_d]}}, // ElementD
85
+ {{DataTypeTag[type_config.accumulator]}}, // Accumulator
86
+ {{DataTypeTag[type_config.element_b_scale]}}, // Scales
87
+ {{DataTypeTag[type_config.element_b_zeropoint]}}, // Zeropoints
88
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
89
+ Config, with_C, with_scales, with_zeropoints>;
90
+
91
+{% for sch in schedules %}
92
+{% set schedule_name = gen_sch_name(sch) -%}
93
+struct sch_{{schedule_name}} {
94
+ using TileShapeNM = Shape<{{
95
+ to_cute_constant(sch.tile_shape_mn)|join(', ')}}>;
96
+ using ClusterShape = Shape<{{
97
+ to_cute_constant(sch.cluster_shape_mnk)|join(', ')}}>;
98
+ // TODO: Reimplement
99
+ // using KernelSchedule = {{KernelScheduleTag[sch.kernel_schedule]}};
100
+ using EpilogueSchedule = {{EpilogueScheduleTag[sch.epilogue_schedule]}};
101
+ using TileScheduler = {{TileSchedulerTag[sch.tile_scheduler]}};
102
+ using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
103
+};
104
+
105
+torch::Tensor
106
+impl_{{type_name}}_sch_{{schedule_name}}(PyTorchArguments args) {
107
+ bool with_C = args.C.has_value(), with_scales = args.scales.has_value(),
108
+ with_zeropoints = args.zeros.has_value();
109
+
110
+ {% for s in specializations %}
111
+ if (with_C == {{s.with_C|lower}}
112
+ && with_zeropoints == {{s.with_zeropoints|lower}}
113
+ && with_scales == {{s.with_scales|lower}}) {
114
+ return run_impl<Kernel<sch_{{schedule_name}}, {{s.with_C|lower}},
115
+ {{s.with_scales|lower}}, {{s.with_zeropoints|lower}}>>(args);
116
+ }{% endfor %}
117
+
118
+ TORCH_CHECK_NOT_IMPLEMENTED(
119
+ false, "for the sake of compile times and binary size machete_mm(..) is "
120
+ " not implemented for with_C=", with_C, ", with_scales=", with_scales,
121
+ ", with_zeropoints=", with_zeropoints,
122
+ " (for {{type_name}}_sch_{{schedule_name}})");
123
+}
124
+{% endfor %}
125
+
126
+}; // namespace machete
127
+"""
128
+
129
+PREPACK_TEMPLATE = """
130
+#include "../machete_prepack_launcher.cuh"
131
+
132
+namespace machete {
133
+using PrepackBDispatcher_ = PrepackBDispatcher<
134
+ {{DataTypeTag[type_config.element_a]}}, // ElementA
135
+ {{DataTypeTag[type_config.element_b]}}, // ElementB
136
+ {{DataTypeTag[type_config.element_d]}}, // ElementD
137
+ {{DataTypeTag[type_config.accumulator]}}, // Accumulator
138
+ {{DataTypeTag[type_config.element_b_scale]}}, // Scales
139
+ {{DataTypeTag[type_config.element_b_zeropoint]}}>; // Zeropoints
140
+
141
+using PrepackedLayoutB = PrepackedLayoutBTemplate<
142
+ {{DataTypeTag[type_config.element_a]}}, // ElementA
143
+ {{DataTypeTag[type_config.element_b]}}, // ElementB
144
+ {{DataTypeTag[type_config.element_d]}}, // ElementD
145
+ {{DataTypeTag[type_config.accumulator]}}, // Accumulator
146
+ cutlass::layout::ColumnMajor,
147
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>;
148
+
149
+template <>
150
+torch::Tensor PrepackBDispatcher_::dispatch(torch::Tensor B) {
151
+ return prepack_impl<PrepackedLayoutB>(B);
152
+}
153
+}; // namespace machete
154
+"""
155
+
156
+TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput
157
+TmaCoop = EpilogueScheduleType.TmaWarpSpecializedCooperative
158
+
159
+
160
+@dataclass
161
+class ScheduleConfig:
162
+ tile_shape_mn: Tuple[int, int]
163
+ cluster_shape_mnk: Tuple[int, int, int]
164
+ kernel_schedule: MixedInputKernelScheduleType
165
+ epilogue_schedule: EpilogueScheduleType
166
+ tile_scheduler: TileSchedulerType
167
+
168
+
169
+@dataclass
170
+class TypeConfig:
171
+ element_a: DataType
172
+ element_b: Union[DataType, VLLMDataType]
173
+ element_b_scale: DataType
174
+ element_b_zeropoint: DataType
175
+ element_d: DataType
176
+ accumulator: DataType
177
+
178
+
179
+@dataclass
180
+class Specialization:
181
+ with_C: bool
182
+ with_zeropoints: bool
183
+ with_scales: bool
184
+
185
+
186
+@dataclass
187
+class ImplConfig:
188
+ type_config: TypeConfig
189
+ schedule_configs: List[ScheduleConfig]
190
+ specializations: List[Specialization]
191
+ heuristic: List[Tuple[Optional[str], ScheduleConfig]]
192
+
193
+
194
+def generate_schedule_name(schedule_config: ScheduleConfig) -> str:
195
+ tile_shape = (
196
+ f"{schedule_config.tile_shape_mn[0]}x{schedule_config.tile_shape_mn[1]}"
197
+ )
198
+ cluster_shape = (f"{schedule_config.cluster_shape_mnk[0]}" +
199
+ f"x{schedule_config.cluster_shape_mnk[1]}" +
200
+ f"x{schedule_config.cluster_shape_mnk[2]}")
201
+ kernel_schedule = VLLMKernelScheduleTag[schedule_config.kernel_schedule]\
202
+ .split("::")[-1]
203
+ epilogue_schedule = EpilogueScheduleTag[
204
+ schedule_config.epilogue_schedule].split("::")[-1]
205
+ tile_scheduler = TileSchedulerTag[schedule_config.tile_scheduler]\
206
+ .split("::")[-1]
207
+
208
+ return (f"{tile_shape}_{cluster_shape}_{kernel_schedule}" +
209
+ f"_{epilogue_schedule}_{tile_scheduler}")
210
+
211
+
212
+# mostly unique shorter schedule_name
213
+def generate_terse_schedule_name(schedule_config: ScheduleConfig) -> str:
214
+ kernel_terse_names_replace = {
215
+ "KernelTmaWarpSpecializedCooperativeMixedInput_": "TmaMI_",
216
+ "TmaWarpSpecializedCooperative_": "TmaCoop_",
217
+ "StreamKScheduler": "streamK",
218
+ }
219
+
220
+ schedule_name = generate_schedule_name(schedule_config)
221
+ for orig, terse in kernel_terse_names_replace.items():
222
+ schedule_name = schedule_name.replace(orig, terse)
223
+ return schedule_name
224
+
225
+
226
+# unique type_name
227
+def generate_type_signature(kernel_type_config: TypeConfig):
228
+ element_a = VLLMDataTypeNames[kernel_type_config.element_a]
229
+ element_b = VLLMDataTypeNames[kernel_type_config.element_b]
230
+ element_d = VLLMDataTypeNames[kernel_type_config.element_d]
231
+ accumulator = VLLMDataTypeNames[kernel_type_config.accumulator]
232
+ element_scale = VLLMDataTypeNames[kernel_type_config.element_b_scale]
233
+ element_zeropoint = VLLMDataTypeNames[
234
+ kernel_type_config.element_b_zeropoint]
235
+
236
+ return (f"{element_a}{element_b}{element_d}"
237
+ f"{accumulator}{element_scale}{element_zeropoint}")
238
+
239
+
240
+# non-unique shorter type_name
241
+def generate_terse_type_signature(kernel_type_config: TypeConfig):
242
+ element_a = VLLMDataTypeNames[kernel_type_config.element_a]
243
+ element_b = VLLMDataTypeNames[kernel_type_config.element_b]
244
+
245
+ return f"{element_a}{element_b}"
246
+
247
+
248
+def is_power_of_two(n):
249
+ return (n != 0) and (n & (n - 1) == 0)
250
+
251
+
252
+def to_cute_constant(value: List[int]):
253
+
254
+ def _to_cute_constant(value: int):
255
+ if is_power_of_two(value):
256
+ return f"_{value}"
257
+ else:
258
+ return f"Int<{value}>"
259
+
260
+ if isinstance(value, Iterable):
261
+ return [_to_cute_constant(value) for value in value]
262
+ else:
263
+ return _to_cute_constant(value)
264
+
265
+
266
+template_globals = {
267
+ "DataTypeTag": VLLMDataTypeTag,
268
+ "KernelScheduleTag": VLLMKernelScheduleTag,
269
+ "EpilogueScheduleTag": EpilogueScheduleTag,
270
+ "TileSchedulerTag": TileSchedulerTag,
271
+ "to_cute_constant": to_cute_constant,
272
+ "gen_sch_name": generate_terse_schedule_name,
273
+}
274
+
275
+
276
+def create_template(template_str):
277
+ template = jinja2.Template(template_str)
278
+ template.globals.update(template_globals)
279
+ return template
280
+
281
+
282
+mm_dispatch_template = create_template(DISPATCH_TEMPLATE)
283
+mm_impl_template = create_template(IMPL_TEMPLATE)
284
+prepack_dispatch_template = create_template(PREPACK_TEMPLATE)
285
+
286
+
287
+def create_sources(impl_config: ImplConfig, num_impl_files=2):
288
+ sources = []
289
+
290
+ type_name = generate_type_signature(impl_config.type_config)
291
+ terse_type_name = generate_terse_type_signature(impl_config.type_config)
292
+
293
+ sources.append((
294
+ f"machete_mm_{terse_type_name}",
295
+ mm_dispatch_template.render(type_name=type_name,
296
+ type_config=impl_config.type_config,
297
+ schedules=impl_config.schedule_configs,
298
+ heuristic=impl_config.heuristic),
299
+ ))
300
+
301
+ sources.append((
302
+ f"machete_prepack_{terse_type_name}",
303
+ prepack_dispatch_template.render(
304
+ type_name=type_name,
305
+ type_config=impl_config.type_config,
306
+ ),
307
+ ))
308
+
309
+ num_schedules = len(impl_config.schedule_configs)
310
+ schedules_per_file = math.ceil(num_schedules / num_impl_files)
311
+ for part, i in enumerate(range(0, num_schedules, schedules_per_file)):
312
+ file_schedules = impl_config.schedule_configs[i:i + schedules_per_file]
313
+
314
+ sources.append((
315
+ f"machete_mm_{terse_type_name}_impl_part{part}",
316
+ mm_impl_template.render(
317
+ type_name=type_name,
318
+ type_config=impl_config.type_config,
319
+ schedules=file_schedules,
320
+ specializations=impl_config.specializations,
321
+ ),
322
+ ))
323
+ return sources
324
+
325
+
326
+def generate():
327
+ # See csrc/quantization/machete/Readme.md, the Codegeneration for more info
328
+ # about how this works
329
+ SCRIPT_DIR = os.path.dirname(__file__)
330
+
331
+ schedules = [
332
+ ScheduleConfig(
333
+ tile_shape_mn=tile_shape_mn,
334
+ cluster_shape_mnk=cluster_shape_mnk,
335
+ kernel_schedule=kernel_schedule,
336
+ epilogue_schedule=epilogue_schedule,
337
+ tile_scheduler=tile_scheduler,
338
+ ) for tile_shape_mn, cluster_shape_mnk in (
339
+ ((128, 16), (1, 1, 1)),
340
+ ((128, 32), (1, 1, 1)),
341
+ ((128, 64), (1, 1, 1)),
342
+ ((128, 128), (1, 1, 1)),
343
+ ) for kernel_schedule in (TmaMI, ) for epilogue_schedule in (TmaCoop, )
344
+ for tile_scheduler in (TileSchedulerType.StreamK, )
345
+ ]
346
+
347
+ # For now we use the same heuristic for all types
348
+ default_heuristic = [
349
+ ("M > 64",
350
+ ScheduleConfig(
351
+ tile_shape_mn=(128, 128),
352
+ cluster_shape_mnk=(1, 1, 1),
353
+ kernel_schedule=TmaMI,
354
+ epilogue_schedule=TmaCoop,
355
+ tile_scheduler=TileSchedulerType.StreamK,
356
+ )),
357
+ ("M > 32",
358
+ ScheduleConfig(
359
+ tile_shape_mn=(128, 64),
360
+ cluster_shape_mnk=(1, 1, 1),
361
+ kernel_schedule=TmaMI,
362
+ epilogue_schedule=TmaCoop,
363
+ tile_scheduler=TileSchedulerType.StreamK,
364
+ )),
365
+ ("M > 16",
366
+ ScheduleConfig(
367
+ tile_shape_mn=(128, 32),
368
+ cluster_shape_mnk=(1, 1, 1),
369
+ kernel_schedule=TmaMI,
370
+ epilogue_schedule=TmaCoop,
371
+ tile_scheduler=TileSchedulerType.StreamK,
372
+ )),
373
+ (None,
374
+ ScheduleConfig(tile_shape_mn=(128, 16),
375
+ cluster_shape_mnk=(1, 1, 1),
376
+ kernel_schedule=TmaMI,
377
+ epilogue_schedule=TmaCoop,
378
+ tile_scheduler=TileSchedulerType.StreamK))
379
+ ]
380
+
381
+ impl_configs = []
382
+
383
+ GPTQ_kernel_type_configs = list(
384
+ (TypeConfig(
385
+ element_a=element_a,
386
+ element_b=element_b,
387
+ element_b_scale=element_a,
388
+ element_b_zeropoint=element_a,
389
+ element_d=element_a,
390
+ accumulator=DataType.f32,
391
+ ) for element_b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
392
+ for element_a in (DataType.f16, DataType.bf16)))
393
+
394
+ GPTQ_kernel_specializations = [
395
+ Specialization(with_C=False, with_zeropoints=False, with_scales=True)
396
+ ]
397
+
398
+ impl_configs += [
399
+ ImplConfig(x[0], x[1], x[2], x[3])
400
+ for x in zip(GPTQ_kernel_type_configs, itertools.repeat(schedules),
401
+ itertools.repeat(GPTQ_kernel_specializations),
402
+ itertools.repeat(default_heuristic))
403
+ ]
404
+
405
+ AWQ_kernel_type_configs = list(
406
+ (TypeConfig(
407
+ element_a=element_a,
408
+ element_b=element_b,
409
+ element_b_scale=element_a,
410
+ element_b_zeropoint=element_a,
411
+ element_d=element_a,
412
+ accumulator=DataType.f32,
413
+ ) for element_b in (DataType.u4, DataType.u8)
414
+ for element_a in (DataType.f16, DataType.bf16)))
415
+
416
+ AWQ_kernel_specializations = [
417
+ Specialization(with_C=False, with_zeropoints=True, with_scales=True)
418
+ ]
419
+
420
+ impl_configs += [
421
+ ImplConfig(x[0], x[1], x[2], x[3])
422
+ for x in zip(AWQ_kernel_type_configs, itertools.repeat(schedules),
423
+ itertools.repeat(AWQ_kernel_specializations),
424
+ itertools.repeat(default_heuristic))
425
+ ]
426
+
427
+ output_dir = os.path.join(SCRIPT_DIR, "generated")
428
+
429
+ # Delete the "generated" directory if it exists
430
+ if os.path.exists(output_dir):
431
+ shutil.rmtree(output_dir)
432
+
433
+ # Create the "generated" directory
434
+ os.makedirs(output_dir)
435
+
436
+ # Render each group of configurations into separate files
437
+ for impl_config in impl_configs:
438
+ for filename, code in create_sources(impl_config):
439
+ filepath = os.path.join(output_dir, f"{filename}.cu")
440
+ with open(filepath, "w") as output_file:
441
+ output_file.write(code)
442
+ print(f"Rendered template to {filepath}")
443
+
444
+
445
+if __name__ == "__main__":
446
+ generate()
csrc/quantization/machete/machete_collective_builder.cuh
new
+33
@@ -0,0 +1,33 @@
1
+#pragma once
2
+
3
+#include "cutlass_extensions/vllm_collective_builder.cuh"
4
+#include "machete_mainloop.cuh"
5
+
6
+namespace cutlass::gemm::collective {
7
+using namespace cute;
8
+
9
+struct MacheteKernelTag {};
10
+
11
+template <class ElementPairA_, class GmemLayoutA_, int AlignmentA,
12
+ class ElementPairB_, class GmemLayoutB_, int AlignmentB,
13
+ class ElementAccumulator, class TileShape_MNK, class ClusterShape_MNK,
14
+ class StageCountType, class KernelScheduleType>
15
+struct VLLMCollectiveBuilder<
16
+ MacheteKernelTag, arch::Sm90, arch::OpClassTensorOp, ElementPairA_,
17
+ GmemLayoutA_, AlignmentA, ElementPairB_, GmemLayoutB_, AlignmentB,
18
+ ElementAccumulator, TileShape_MNK, ClusterShape_MNK, StageCountType,
19
+ KernelScheduleType,
20
+ cute::enable_if_t<(
21
+ cute::is_same_v<KernelScheduleType,
22
+ KernelTmaWarpSpecializedMixedInput> ||
23
+ cute::is_same_v<KernelScheduleType,
24
+ KernelTmaWarpSpecializedPingpongMixedInput> ||
25
+ cute::is_same_v<KernelScheduleType,
26
+ KernelTmaWarpSpecializedCooperativeMixedInput>)>> {
27
+ using CollectiveOp = machete::MacheteCollectiveMma<
28
+ ElementPairA_, GmemLayoutA_, AlignmentA, ElementPairB_, GmemLayoutB_,
29
+ AlignmentB, ElementAccumulator, TileShape_MNK, ClusterShape_MNK,
30
+ StageCountType, KernelScheduleType>;
31
+};
32
+
33
+}; // namespace cutlass::gemm::collective
\ No newline at end of file
csrc/quantization/machete/machete_interleaving_utils.cuh
new
+35
@@ -0,0 +1,35 @@
1
+#pragma once
2
+
3
+#include "cutlass/cutlass.h"
4
+#include "cute/layout.hpp"
5
+
6
+namespace machete {
7
+
8
+using namespace cute;
9
+
10
+// get an interleaved block layout where each element consecutive element has a
11
+// stride of bit_stride and the block width is blk_bit_width,
12
+// examples:
13
+// size_bits<T> = 8, bit_stride = 8, blk_bit_width = 32 -> 4:1
14
+// size_bits<T> = 8, bit_stride = 16, blk_bit_width = 32 -> (2, 2):(2, 1)
15
+// size_bits<T> = 4, bit_stride = 8, blk_bit_width = 32 -> (4, 2):(2, 1)
16
+// size_bits<T> = 4, bit_stride = 16, blk_bit_width = 32 -> (2, 4):(4, 1)
17
+template <typename T, int bit_stride, int blk_bit_width>
18
+CUTE_HOST_DEVICE static constexpr auto get_interleaved_blk_layout() {
19
+ static_assert(blk_bit_width % bit_stride == 0);
20
+ static_assert(bit_stride % cute::sizeof_bits_v<T> == 0);
21
+
22
+ constexpr auto elems_per_blk = blk_bit_width / cute::sizeof_bits_v<T>;
23
+
24
+ if constexpr (cute::sizeof_bits_v<T> == bit_stride) {
25
+ // identity layout
26
+ return Layout<Shape<Int<elems_per_blk>>>{};
27
+ } else {
28
+ constexpr auto elems_per_stride = bit_stride / cute::sizeof_bits_v<T>;
29
+ constexpr auto num_strides = elems_per_blk / elems_per_stride;
30
+ return Layout<Shape<Int<num_strides>, Int<elems_per_stride>>,
31
+ Stride<Int<elems_per_stride>, Int<1>>>{};
32
+ }
33
+}
34
+
35
+}; // namespace machete
csrc/quantization/machete/machete_mainloop.cuh
new
+1473
@@ -0,0 +1,1473 @@
1
+//
2
+// Based off of:
3
+// cutlass/gemm/collective/sm90_mma_tma_gmma_rs_warpspecialized_mixed_input.hpp
4
+// Specifically:
5
+// https://github.com/NVIDIA/cutlass/tree/06b21349bcf6ddf6a1686a47a137ad1446579db9/include/cutlass/gemm/collective/sm90_mma_tma_gmma_rs_warpspecialized_mixed_input.hpp
6
+// Referred to as upstream from in the comments
7
+//
8
+// The main optimization machete implements compared to upstream is to prepack
9
+// the weight matrix to more closely match the shape of the wgmma instructions
10
+// allowing for wider (ideally 128bit) shared memory loads. For subbyte types
11
+// this is done by packing values from multiple wgmma loads (for a single
12
+// thread) into a single 128bit load. This is very similar to layout used in
13
+// Marlin, although specific to the wgmma instructions.
14
+//
15
+// Since the wgmma instructions only support sourcing from registers for the A
16
+// operand, and we want to upconvert/decompress the weight values/elements
17
+// before feeding them into the tensor cores in registers, we need the weight
18
+// matrix to be A. To achieve this we compute the transpose of Y = XW^t as
19
+// Y^t = W^tX^t. This is mostly done outside of this file in
20
+// csrc/quantization/machete/machete_mm_kernel.cuh, but this why A is the
21
+// quantized/narrow type and has the prepacked layout despite the API being:
22
+// B_prepacked = machete_prepack_B(B)
23
+// Y = machete_mm(A, B_prepacked)
24
+//
25
+#pragma once
26
+
27
+// clang-format off
28
+#include "cutlass/cutlass.h"
29
+#include "cutlass/numeric_conversion.h"
30
+#include "cute/arch/cluster_sm90.hpp"
31
+#include "cute/arch/copy_sm90.hpp"
32
+#include "cutlass/gemm/gemm.h"
33
+#include "cutlass/detail/dependent_false.hpp"
34
+#include "cutlass/gemm/dispatch_policy.hpp"
35
+#include "cutlass/detail/layout.hpp"
36
+
37
+#include "cute/algorithm/functional.hpp"
38
+#include "cute/atom/mma_atom.hpp"
39
+#include "cute/atom/copy_traits_sm90_tma.hpp"
40
+#include "cute/algorithm/gemm.hpp"
41
+#include "cute/tensor_predicate.hpp"
42
+#include "cute/numeric/arithmetic_tuple.hpp"
43
+#include "cutlass/pipeline/pipeline.hpp"
44
+#include "cutlass/transform/collective/sm90_wgmma_transpose.hpp"
45
+#include "cutlass/trace.h"
46
+
47
+#include "cutlass/detail/collective.hpp"
48
+// clang-format on
49
+
50
+#include "cutlass_extensions/cute_utils.cuh"
51
+
52
+namespace machete {
53
+
54
+using namespace cute;
55
+using namespace cutlass;
56
+using namespace cutlass::gemm;
57
+using namespace cutlass::gemm::collective;
58
+using namespace cutlass::gemm::collective::detail;
59
+
60
+template <class ElementATuple_, class GmemLayoutA, int AlignmentA,
61
+ class ElementB_, class GmemLayoutB, int AlignmentB,
62
+ class ElementAccumulator_, class TileShape_MNK,
63
+ class ClusterShape_MNK, class StageCountType,
64
+ class KernelScheduleType>
65
+struct MacheteCollectiveMma {
66
+ using Schedule = KernelScheduleType;
67
+ static_assert(
68
+ cute::is_same_v<Schedule, KernelTmaWarpSpecialized> ||
69
+ cute::is_same_v<Schedule, KernelTmaWarpSpecializedMixedInput> ||
70
+ cute::is_same_v<Schedule, KernelTmaWarpSpecializedPingpong> ||
71
+ cute::is_same_v<Schedule,
72
+ KernelTmaWarpSpecializedPingpongMixedInput> ||
73
+ cute::is_same_v<Schedule, KernelTmaWarpSpecializedCooperative> ||
74
+ cute::is_same_v<Schedule,
75
+ KernelTmaWarpSpecializedCooperativeMixedInput>,
76
+ "KernelSchedule must be one of the warp specialized policies");
77
+
78
+ public:
79
+ static constexpr bool ALayoutIsPrepacked = true;
80
+
81
+ // Prepacked block shape (N is M in the transposed problem)
82
+ using PPBlockShape_MK = typename GmemLayoutA::PPBlockShape_NK;
83
+ // Prepacked blocks per dim for a single MMA tile
84
+ using PPBlocksPerTile_MK = decltype(make_shape(
85
+ size<0>(TileShape_MNK{}) / size<0>(PPBlockShape_MK{}),
86
+ size<2>(TileShape_MNK{}) / size<1>(PPBlockShape_MK{})));
87
+
88
+ using IlvdBlkLayout = typename GmemLayoutA::IlvdBlkLayout;
89
+
90
+ static_assert(size<0>(TileShape_MNK{}) % size<0>(PPBlockShape_MK{}) == 0,
91
+ "M in PPBlockShape_MK must evenly divide M TileShape_MNK");
92
+ static_assert(size<2>(TileShape_MNK{}) % size<1>(PPBlockShape_MK{}) == 0,
93
+ "K in PPBlockShape_MK must evenly divide K TileShape_MNK");
94
+
95
+ using ArchTag = arch::Sm90;
96
+ using TileShape = TileShape_MNK;
97
+ using ClusterShape = ClusterShape_MNK;
98
+ using ElementA = deduce_mixed_width_dtype_t<0, ElementATuple_>;
99
+ using StrideA = TagToStrideA_t<layout::RowMajor>;
100
+ using ElementB = ElementB_;
101
+ using StrideB = TagToStrideB_t<GmemLayoutB>;
102
+ using ElementAccumulator = ElementAccumulator_;
103
+ using ElementMma = ElementB;
104
+ using ElementATuple =
105
+ cute::conditional_t<!cute::is_tuple<ElementATuple_>::value,
106
+ cute::tuple<ElementA>, ElementATuple_>;
107
+
108
+ static constexpr cute::GMMA::Major GmmaMajorA =
109
+ gmma_rs_tag_to_major_A<layout::RowMajor>();
110
+ static constexpr cute::GMMA::Major GmmaMajorB =
111
+ gmma_rs_tag_to_major_B<GmemLayoutB>();
112
+
113
+ // For coop schedules we have two warp groups cooperatively issuing wgmma
114
+ // instructions so we use 2 atoms along the M dim (one for each warpgroup)
115
+ using AtomLayoutMNK = cute::conditional_t<
116
+ cute::is_same_v<KernelScheduleType,
117
+ KernelTmaWarpSpecializedCooperativeMixedInput>,
118
+ Layout<Shape<_2, _1, _1>>, Layout<Shape<_1, _1, _1>>>;
119
+
120
+ using TiledMma = decltype(cute::make_tiled_mma(
121
+ cute::GMMA::rs_op_selector<ElementMma, ElementMma, ElementAccumulator,
122
+ TileShape_MNK, GMMA::Major::K, GmmaMajorB>(),
123
+ AtomLayoutMNK{}));
124
+
125
+ private:
126
+ //
127
+ // the setup section (until "section setup end") contains a combination of
128
+ // modified code from (used as a starting point):
129
+ // `cutlass/gemm/collective/builders/sm90_gmma_builder.inl`
130
+ // `cutlass/gemm/collective/sm90_mma_tma_gmma_rs_warpspecialized_mixed_input.hpp`
131
+ // (upstream)
132
+ //
133
+ // however in-order to simplify the code we combine a lot of the logic from
134
+ // `CollectiveMma` and `CollectiveBuilder` into this class, this also makes
135
+ // sense given that we have flexibility on layouts here. We also simplify the
136
+ // code by only supporting scales and zeros for A (in the transposed problem,
137
+ // B from an API perspective), also since we force A to be the narrow type
138
+ // (i.e. the type to be upconverted) we can remove all the `SwapAB` logic in
139
+ // the upstream also simplifying the code. This section includes new logic
140
+ // (compared ustream) for handling the prepacked-A layouts (in the transposed
141
+ // problem, B from an API perspective)
142
+ //
143
+ using ElementScale = deduce_mixed_width_dtype_t<1, ElementATuple_>;
144
+ using ElementZero = deduce_mixed_width_dtype_t<2, ElementATuple_>;
145
+
146
+ static constexpr bool IsANarrow = cutlass::sizeof_bits<ElementA>::value <
147
+ cutlass::sizeof_bits<ElementB>::value;
148
+ static_assert(IsANarrow,
149
+ "A must be the narrow one since its the one that flows through "
150
+ "registers.");
151
+
152
+ public:
153
+ static constexpr int PipelineStages =
154
+ compute_stage_count_or_override_single_affine_transformed_input<
155
+ sm90_smem_capacity_bytes, ElementA, ElementB, ElementScale,
156
+ ElementZero, TileShape_MNK>(StageCountType{});
157
+
158
+ struct DispatchPolicy {
159
+ constexpr static int Stages = PipelineStages;
160
+ using ClusterShape = ClusterShape_MNK;
161
+ using Schedule = KernelScheduleType;
162
+ };
163
+
164
+ using GmemTiledCopyA =
165
+ decltype(sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
166
+ using GmemTiledCopyB =
167
+ decltype(sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
168
+
169
+ // ((T, V), (BlocksM, BlocksK), pipe) -> offset
170
+ using SmemLayoutA = decltype(GmemLayoutA::TVbNbKL_to_offset(
171
+ make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
172
+ Int<DispatchPolicy::Stages>{})));
173
+
174
+ using SmemLayoutAtomARowMajor =
175
+ decltype(rs_smem_selector<GmmaMajorA, ElementA,
176
+ decltype(cute::get<0>(TileShape_MNK{})),
177
+ decltype(cute::get<2>(TileShape_MNK{}))>());
178
+
179
+ using SmemLayoutAtomScale = Layout<
180
+ Shape<decltype(cute::shape<0>(SmemLayoutAtomARowMajor{})), cute::Int<1>>>;
181
+
182
+ using SmemLayoutAtomB =
183
+ decltype(rs_smem_selector<GmmaMajorB, ElementB,
184
+ decltype(cute::get<1>(TileShape_MNK{})),
185
+ decltype(cute::get<2>(TileShape_MNK{}))>());
186
+
187
+ using SmemCopyAtomA = Copy_Atom<cute::DefaultCopy, ElementA>;
188
+ using SmemCopyAtomB = void;
189
+
190
+ //
191
+ // Validity checks
192
+ //
193
+ static_assert(is_static<TileShape_MNK>::value);
194
+ static_assert(is_static<ClusterShape_MNK>::value);
195
+ static_assert(is_aligned<ElementA, AlignmentA, ElementB, AlignmentB,
196
+ tma_alignment_bytes>(),
197
+ "Should meet TMA alignment requirement\n");
198
+#ifndef CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED
199
+ static_assert(cutlass::detail::dependent_false<ElementA>,
200
+ "Unsupported Toolkit for SM90 Collective Builder\n");
201
+#endif
202
+
203
+ private:
204
+ enum class ConversionMode {
205
+ DirectConvert,
206
+ ConvertAndScale,
207
+ ConvertAndScaleWithZero
208
+ };
209
+
210
+ public:
211
+ //
212
+ // Type Aliases
213
+ //
214
+ using KernelSchedule = KernelScheduleType;
215
+
216
+ // For cases where we can't have a void type, we can use this to allow the
217
+ // code to compile when the scale / zero is void.
218
+ using NonVoidElementScale =
219
+ cute::conditional_t<cute::is_void_v<ElementScale>, float, ElementScale>;
220
+ using NonVoidElementZero =
221
+ cute::conditional_t<cute::is_void_v<ElementZero>, float, ElementZero>;
222
+
223
+ // These are always MN major
224
+ using StrideScale = cute::Stride<cute::Int<1>, int64_t, int64_t>;
225
+ // For cases where we can't have a void scale, we can use this to allow the
226
+ // code to compile when the scale is void.
227
+ using NonVoidStrideScale =
228
+ cute::conditional_t<cute::is_void_v<StrideScale>,
229
+ cute::Stride<_1, int64_t, int64_t>, StrideScale>;
230
+
231
+ static_assert((cutlass::gemm::detail::is_k_major<StrideA>()),
232
+ "The transformed matrix (A) must be K-major.");
233
+
234
+ static_assert((sizeof(ElementB) == 2) ||
235
+ (cutlass::gemm::detail::is_k_major<StrideA>() &&
236
+ cutlass::gemm::detail::is_k_major<StrideB>()),
237
+ "The unscaled element (matrix B) must be 2 bytes OR both "
238
+ "inputs must be K-major");
239
+
240
+ static_assert(cutlass::gemm::detail::is_mn_major<NonVoidStrideScale>(),
241
+ "Scale must be MN major [Col Major if A is scaled, Row Major "
242
+ "if B is scaled].");
243
+
244
+ static_assert(std::is_same_v<typename TiledMma::ValTypeC, ElementAccumulator>,
245
+ "TiledMma::ValTypeC must be the same as ElementAccumulator.");
246
+
247
+ using GmemTiledCopyScale = cute::SM90_TMA_LOAD;
248
+
249
+ using SmemCopyAtomScale = Copy_Atom<cute::DefaultCopy, NonVoidElementScale>;
250
+
251
+ // TMA converts f32 input to tf32 when copying from GMEM to SMEM
252
+ // For all other types, cast to size equivalent uint type to avoid any
253
+ // rounding by TMA.
254
+ static constexpr bool ConvertF32toTF32A = cute::is_same_v<float, ElementA>;
255
+ static constexpr bool ConvertF32toTF32B = cute::is_same_v<float, ElementB>;
256
+ using InternalElementA =
257
+ cute::conditional_t<ConvertF32toTF32A, tfloat32_t,
258
+ uint_bit_t<sizeof_bits_v<ElementA>>>;
259
+ using InternalElementB =
260
+ cute::conditional_t<ConvertF32toTF32B, tfloat32_t,
261
+ uint_bit_t<sizeof_bits_v<ElementB>>>;
262
+
263
+ using TransformA = cute::identity;
264
+ using TransformB = cute::identity;
265
+
266
+ static constexpr int IsSubbyteA = cute::sizeof_bits_v<InternalElementA> < 8;
267
+ using TmaElementA =
268
+ cute::conditional_t<IsSubbyteA, uint8_t, InternalElementA>;
269
+
270
+ using MainloopPipeline = cutlass::PipelineTmaAsync<DispatchPolicy::Stages>;
271
+ using PipelineState = cutlass::PipelineState<DispatchPolicy::Stages>;
272
+
273
+ using PipelineParams = typename MainloopPipeline::Params;
274
+ using ScaleTileShape = decltype(make_shape(shape<0>(TileShape{}),
275
+ shape<1>(SmemLayoutAtomScale{})));
276
+
277
+ static_assert(cute::rank(SmemLayoutAtomB{}) == 2,
278
+ "SmemLayoutAtom must be rank 2 (M/N, K)");
279
+ static_assert((size<1>(TileShape{}) % size<0>(SmemLayoutAtomB{})) == 0,
280
+ "SmemLayoutAtom must evenly divide tile shape.");
281
+ static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomB{})) == 0,
282
+ "SmemLayoutAtom must evenly divide tile shape.");
283
+
284
+ static_assert(rank(SmemLayoutAtomScale{}) == 2,
285
+ "SmemLayoutAtomScale must be rank 2");
286
+ static_assert((size<0>(TileShape{}) % size<0>(SmemLayoutAtomScale{})) == 0,
287
+ "SmemLayoutAtomScale must equal the tile shape.");
288
+ static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomScale{})) == 0,
289
+ "SmemLayoutAtomScale must evenly divide tile k shape.");
290
+
291
+ // Tile along modes in a way that maximizes the TMA box size.
292
+ using SmemLayoutACopy = decltype(tile_to_shape(
293
+ SmemLayoutAtomARowMajor{},
294
+ make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}),
295
+ Int<DispatchPolicy::Stages>{}),
296
+ conditional_t<::cutlass::gemm::detail::is_major<0, StrideA>(),
297
+ Step<_2, _1, _3>, Step<_1, _2, _3>>{}));
298
+
299
+ using SmemLayoutB = decltype(tile_to_shape(
300
+ SmemLayoutAtomB{},
301
+ make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}),
302
+ Int<DispatchPolicy::Stages>{}),
303
+ conditional_t<::cutlass::gemm::detail::is_major<0, StrideB>(),
304
+ Step<_2, _1, _3>, Step<_1, _2, _3>>{}));
305
+
306
+ // It is assumed that the scales and zero-points share the same smem layout
307
+ using SmemLayoutScale = decltype(tile_to_shape(
308
+ SmemLayoutAtomScale{},
309
+ make_shape(shape<0>(ScaleTileShape{}), shape<1>(ScaleTileShape{}),
310
+ Int<PipelineStages>{})));
311
+
312
+ // If A mn-layout and B mn-layout, transposing B matrix since WGMMA is k-major
313
+ // only (e.g. tf32, fp32, fp8, int8).
314
+ static constexpr bool IsLayoutAmnBmn =
315
+ cute::is_same_v<gemm::detail::StrideToLayoutTagA_t<StrideA>,
316
+ layout::ColumnMajor> &&
317
+ cute::is_same_v<gemm::detail::StrideToLayoutTagB_t<StrideB>,
318
+ layout::RowMajor>;
319
+
320
+ static_assert(DispatchPolicy::Stages >= 2,
321
+ "Specialization requires Stages set to value 2 or more.");
322
+ static_assert(not cute::is_base_of<cute::GMMA::DescriptorIterator,
323
+ typename TiledMma::FrgTypeA>::value &&
324
+ cute::is_base_of<cute::GMMA::DescriptorIterator,
325
+ typename TiledMma::FrgTypeB>::value,
326
+ "MMA atom must source A from rmem and B operand from smem_desc "
327
+ "for this mainloop.");
328
+ static_assert(cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD> ||
329
+ cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD_MULTICAST>,
330
+ "GmemTiledCopy - invalid SM90 TMA copy atom specified.");
331
+ static_assert(cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD> ||
332
+ cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD_MULTICAST>,
333
+ "GmemTiledCopy - invalid SM90 TMA copy atom specified.");
334
+
335
+ using GmmaSmemLayoutB = decltype(tile_to_shape(
336
+ SmemLayoutAtomB{},
337
+ make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}),
338
+ Int<DispatchPolicy::Stages>{}),
339
+ conditional_t<::cutlass::gemm::detail::is_major<0, StrideB>(),
340
+ Step<_2, _1, _3>, Step<_1, _2, _3>>{}));
341
+
342
+ // These two restrictions are related, so we place the assertions together.
343
+ // To relax them, we need to handle loading more than 1 row of scales for
344
+ // every main loop iteration. We must also handle updating the pipeline
345
+ // transaction bytes on the fly. NOTE: Deleting this assertion without
346
+ // required changes will cause the code to hang.
347
+ static_assert(size<1>(SmemLayoutAtomScale{}) == 1,
348
+ "size<1>(SmemLayoutAtomScale) must be 1.");
349
+
350
+ private:
351
+ static constexpr ConversionMode get_conversion_mode() {
352
+ if constexpr (cute::is_void_v<ElementScale>) {
353
+ return ConversionMode::DirectConvert;
354
+ } else if constexpr (cute::is_void_v<ElementZero>) {
355
+ return ConversionMode::ConvertAndScale;
356
+ } else {
357
+ return ConversionMode::ConvertAndScaleWithZero;
358
+ }
359
+ }
360
+
361
+ static constexpr ConversionMode KernelConversionMode = get_conversion_mode();
362
+ static constexpr bool ModeHasScales =
363
+ KernelConversionMode == ConversionMode::ConvertAndScale ||
364
+ KernelConversionMode == ConversionMode::ConvertAndScaleWithZero;
365
+
366
+ // Same as upstream, should be kept the same when possible
367
+ static constexpr auto elements_per_smem_scale() {
368
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
369
+ return 0;
370
+ } else if constexpr (ModeHasScales) {
371
+ return cute::cosize_v<SmemLayoutScale>;
372
+ } else {
373
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
374
+ "Type not handled in scale smem allocation.");
375
+ }
376
+ }
377
+
378
+ // Same as upstream, should be kept the same when possible
379
+ static constexpr auto elements_per_smem_zero() {
380
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert ||
381
+ KernelConversionMode == ConversionMode::ConvertAndScale) {
382
+ return 0;
383
+ } else if constexpr (KernelConversionMode ==
384
+ ConversionMode::ConvertAndScaleWithZero) {
385
+ return cute::cosize_v<SmemLayoutScale>;
386
+ } else {
387
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
388
+ "Type not handled in scale smem allocation.");
389
+ }
390
+ }
391
+
392
+ // Same as upstream, should be kept the same when possible, not formatte for
393
+ // easier comparison
394
+ // clang-format off
395
+ // These methods use some the public members of the class. For that reason, we define them after the public section.
396
+ static constexpr uint32_t
397
+ compute_tma_transaction_bytes_mk() {
398
+ constexpr uint32_t baseline_bytes = cutlass::bits_to_bytes(size<0>(SmemLayoutA{}) * size<1>(SmemLayoutA{}) * static_cast<uint32_t>(cute::sizeof_bits_v<InternalElementA>));
399
+
400
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
401
+ return baseline_bytes;
402
+ }
403
+ else if constexpr (ModeHasScales) {
404
+ constexpr uint32_t scale_tx_bytes = cutlass::bits_to_bytes(size<0>(SmemLayoutScale{}) * size<1>(SmemLayoutScale{}) * static_cast<uint32_t>(cute::sizeof_bits_v<ElementScale>));
405
+ static_assert(scale_tx_bytes % 128 == 0, "Each scale stage must be 128B aligned."); // required by TMA
406
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
407
+ return baseline_bytes + scale_tx_bytes;
408
+ }
409
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
410
+ // Scale and zero share smem layout
411
+ constexpr uint32_t zero_tx_bytes = cutlass::bits_to_bytes(size<0>(SmemLayoutScale{}) * size<1>(SmemLayoutScale{}) * static_cast<uint32_t>(cute::sizeof_bits_v<ElementZero>));
412
+ static_assert(zero_tx_bytes % 128 == 0, "Each zero stage must be 128B aligned."); // required by TMA
413
+ return baseline_bytes + scale_tx_bytes + zero_tx_bytes;
414
+ }
415
+ else {
416
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Type not handled in tma transaction bytes computation.");
417
+ }
418
+ }
419
+ else {
420
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Type not handled in tma transaction bytes computation.");
421
+ }
422
+ }
423
+
424
+ static constexpr uint32_t
425
+ compute_tma_transaction_bytes_nk() {
426
+ return cutlass::bits_to_bytes(size<0>(SmemLayoutB{}) * size<1>(SmemLayoutB{}) * static_cast<uint32_t>(cute::sizeof_bits_v<InternalElementB>));
427
+ }
428
+ // clang-format on
429
+
430
+ // ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
431
+ using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset(
432
+ make_shape(int32_t(0), int32_t(0), int32_t(0)))));
433
+
434
+ using ATensor = decltype(make_tensor(
435
+ get_logical_ptr(static_cast<InternalElementA const*>(nullptr)),
436
+ shape(GmemLayoutA::TVbNbKL_to_offset(
437
+ make_shape(int32_t(0), int32_t(0), int32_t(0)))),
438
+ PrepackedStrideA{}));
439
+
440
+ using BTensor = decltype(make_tensor(
441
+ get_logical_ptr(static_cast<InternalElementB const*>(nullptr)),
442
+ repeat_like(StrideB{}, int32_t(0)), StrideB{}));
443
+ using ScaleTensor = decltype(make_tensor(
444
+ get_logical_ptr(static_cast<NonVoidElementScale const*>(nullptr)),
445
+ repeat_like(NonVoidStrideScale{}, int32_t(0)), NonVoidStrideScale{}));
446
+
447
+ using ZeroTensor = decltype(make_tensor(
448
+ get_logical_ptr(static_cast<NonVoidElementZero const*>(nullptr)),
449
+ repeat_like(NonVoidStrideScale{}, int32_t(0)), NonVoidStrideScale{}));
450
+
451
+ static constexpr auto make_tma_copy_A(ATensor tensor_a = ATensor{}) {
452
+ return make_tma_copy<TmaElementA>(
453
+ GmemTiledCopyA{}, tensor_a, SmemLayoutA{}(_, _, cute::Int<0>{}),
454
+ shape(SmemLayoutA{}(_, _, cute::Int<0>{})),
455
+ size<1>(ClusterShape{})); // mcast along N mode for this M load, if any
456
+ }
457
+
458
+ static constexpr auto make_tma_copy_scale(
459
+ ScaleTensor tensor_scale = ScaleTensor{}) {
460
+ return make_tma_copy(GmemTiledCopyScale{}, tensor_scale,
461
+ SmemLayoutScale{}(_, _, cute::Int<0>{}),
462
+ ScaleTileShape{},
463
+ _1{}); // mcast along N mode for this M load, if any
464
+ }
465
+
466
+ static constexpr auto make_tma_copy_zero(
467
+ ZeroTensor tensor_zero = ZeroTensor{}) {
468
+ return make_tma_copy(GmemTiledCopyScale{}, tensor_zero,
469
+ SmemLayoutScale{}(_, _, cute::Int<0>{}),
470
+ ScaleTileShape{},
471
+ _1{}); // mcast along N mode for this M load, if any
472
+ }
473
+
474
+ static constexpr auto make_tma_copy_B(BTensor tensor_b = BTensor{}) {
475
+ return make_tma_copy(
476
+ GmemTiledCopyB{}, tensor_b, SmemLayoutB{}(_, _, cute::Int<0>{}),
477
+ make_shape(shape<1>(TileShape{}), shape<2>(TileShape{})),
478
+ size<0>(ClusterShape{})); // mcast along M mode for this N load, if any
479
+ }
480
+
481
+ public:
482
+ // Same as upstream, should be kept the same when possible, not formatted for
483
+ // easier comparison
484
+ // with `RealInternalElementA` -> `ElementA` since we support `SwapAB` logic
485
+ // clang-format off
486
+ static constexpr size_t SmemAlignmentA = cutlass::detail::alignment_for_swizzle(SmemLayoutA{});
487
+
488
+ static constexpr size_t SmemAlignmentB = cutlass::detail::alignment_for_swizzle(SmemLayoutB{});
489
+
490
+ // Just pick the max alignment of A and B since it is required to be at least 128B
491
+ static constexpr size_t SmemAlignmentScale = cute::max(SmemAlignmentA, SmemAlignmentB);
492
+
493
+ static_assert(SmemAlignmentA >= 128 and SmemAlignmentB >= 128, "Require at least 128B alignment");
494
+
495
+ struct SharedStorage
496
+ {
497
+ static constexpr int scale_elements = elements_per_smem_scale();
498
+ static constexpr int zero_elements = elements_per_smem_zero();
499
+ struct TensorStorage : cute::aligned_struct<cute::max(SmemAlignmentA, SmemAlignmentB)> {
500
+ cute::ArrayEngine<ElementA, cute::cosize_v<SmemLayoutA>> smem_A;
501
+ cute::ArrayEngine<typename TiledMma::ValTypeB, cute::cosize_v<SmemLayoutB>> smem_B;
502
+ cute::ArrayEngine<NonVoidElementScale, scale_elements> smem_scale;
503
+ cute::ArrayEngine<NonVoidElementZero, zero_elements> smem_zero;
504
+ } tensors;
505
+
506
+ using PipelineStorage = typename MainloopPipeline::SharedStorage;
507
+ PipelineStorage pipeline;
508
+ };
509
+ using TensorStorage = typename SharedStorage::TensorStorage;
510
+ using PipelineStorage = typename SharedStorage::PipelineStorage;
511
+
512
+ // Host side kernel arguments
513
+ struct Arguments {
514
+ ElementA const* ptr_A = nullptr;
515
+ StrideA dA{};
516
+ ElementB const* ptr_B = nullptr;
517
+ StrideB dB{};
518
+ ElementScale const* ptr_S = nullptr;
519
+ NonVoidStrideScale dS{};
520
+ int group_size = 0;
521
+ ElementZero const* ptr_Z = nullptr;
522
+ uint32_t mma_promotion_interval = 4;
523
+ };
524
+ // clang-format on
525
+
526
+ //
527
+ // section setup end
528
+ //
529
+
530
+ // Similar (but not idendtical) to upstream, should be kept the same when
531
+ // possible
532
+ // compared to upstream we use `make_tma_copy_A`, `make_tma_copy_B` etc. to
533
+ // define the TMA types
534
+ // Device side kernel params
535
+ struct Params {
536
+ public:
537
+ // Assumption: StrideA is congruent with Problem_MK
538
+ using TMA_A = decltype(make_tma_copy_A());
539
+ using TMA_Scale = decltype(make_tma_copy_scale());
540
+ using TMA_Zero = decltype(make_tma_copy_zero());
541
+ using TMA_B = decltype(make_tma_copy_B());
542
+
543
+ // required by outer loop: i.e.
544
+ // cutlass/gemm/kernel/sm90_gemm_tma_warpspecialized_cooperative.hpp
545
+ TMA_A tma_load_a;
546
+ TMA_B tma_load_b;
547
+ TMA_Scale tma_load_scale;
548
+ TMA_Zero tma_load_zero;
549
+ int64_t scale_k;
550
+ int group_size;
551
+ uint32_t tma_transaction_bytes = TmaTransactionBytes;
552
+ uint32_t tma_transaction_bytes_mk = TmaTransactionBytesMK;
553
+ uint32_t tma_transaction_bytes_nk = TmaTransactionBytesNK;
554
+ };
555
+
556
+ //
557
+ // Methods
558
+ //
559
+
560
+ // Similar (but not idendtical) to upstream, should be kept the same when
561
+ // possible
562
+ // compared to upstream we use `make_tma_copy_A` and `TVbNbKL_to_offset` here
563
+ // to handle the prepacked layout
564
+ template <class ProblemShape>
565
+ static constexpr Params to_underlying_arguments(
566
+ ProblemShape const& problem_shape, Arguments const& args,
567
+ void* workspace) {
568
+ (void)workspace;
569
+
570
+ // Optionally append 1s until problem shape is rank-4 (MNKL), in case it is
571
+ // only rank-3 (MNK)
572
+ auto problem_shape_MNKL = append<4>(problem_shape, 1);
573
+ auto [M, N, K, L] = problem_shape_MNKL;
574
+
575
+ auto ptr_A = reinterpret_cast<InternalElementA const*>(args.ptr_A);
576
+ auto ptr_B = reinterpret_cast<InternalElementB const*>(args.ptr_B);
577
+
578
+ auto make_logical_tensor = [&](auto ptr, auto shape, auto stride) {
579
+ return make_tensor(get_logical_ptr(ptr), make_layout(shape, stride));
580
+ };
581
+
582
+ typename Params::TMA_A tma_load_a;
583
+ typename Params::TMA_B tma_load_b;
584
+ typename Params::TMA_Scale tma_load_scale;
585
+ typename Params::TMA_Zero tma_load_zero;
586
+
587
+ auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
588
+ tma_load_a = make_tma_copy_A(
589
+ make_logical_tensor(ptr_A, shape(layout), stride(layout)));
590
+
591
+ tma_load_b = make_tma_copy_B(
592
+ make_logical_tensor(ptr_B, make_shape(N, K, L), args.dB));
593
+
594
+ if constexpr (ModeHasScales) {
595
+ tma_load_scale = make_tma_copy_scale(make_logical_tensor(
596
+ args.ptr_S, make_shape(M, args.group_size, L), args.dS));
597
+ }
598
+
599
+ if constexpr (KernelConversionMode ==
600
+ ConversionMode::ConvertAndScaleWithZero) {
601
+ tma_load_zero = make_tma_copy_zero(make_logical_tensor(
602
+ args.ptr_Z, make_shape(M, args.group_size, L), args.dS));
603
+ }
604
+
605
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
606
+ return {tma_load_a, tma_load_b, tma_load_scale, tma_load_zero, 0, 0};
607
+ } else if constexpr (ModeHasScales) {
608
+ auto scale_k = (K + args.group_size - 1) / args.group_size;
609
+
610
+ return {tma_load_a, tma_load_b, tma_load_scale,
611
+ tma_load_zero, scale_k, args.group_size};
612
+ } else {
613
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
614
+ "Conversion mode not handled in to_underlying_arguments.");
615
+ }
616
+ }
617
+
618
+ // Same as upstream, should be kept the same when possible, not formatted for
619
+ // easier comparison
620
+ // with `SwapAB ? N : M -> M` since we dont support SwapAB
621
+ // clang-format off
622
+ template<class ProblemShape>
623
+ static bool
624
+ can_implement(
625
+ ProblemShape const& problem_shape,
626
+ [[maybe_unused]] Arguments const& args) {
627
+ constexpr int tma_alignment_bits = 128;
628
+ auto problem_shape_MNKL = append<4>(problem_shape, 1);
629
+ auto [M,N,K,L] = problem_shape_MNKL;
630
+
631
+ bool implementable = true;
632
+ constexpr int min_tma_aligned_elements_A = tma_alignment_bits / cutlass::sizeof_bits<ElementA>::value;
633
+ implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_A>(cute::make_shape(M,K,L), StrideA{});
634
+ constexpr int min_tma_aligned_elements_B = tma_alignment_bits / cutlass::sizeof_bits<ElementB>::value;
635
+ implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_B>(cute::make_shape(N,K,L), StrideB{});
636
+
637
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
638
+ implementable = implementable && (args.ptr_S == nullptr);
639
+ implementable = implementable && (args.ptr_Z == nullptr);
640
+ }
641
+ else if constexpr (ModeHasScales) {
642
+ const int scale_mn = M;
643
+ const int scale_k = (K + args.group_size - 1) / args.group_size;
644
+ constexpr int min_tma_aligned_elements_scale = tma_alignment_bits / cutlass::sizeof_bits<ElementScale>::value;
645
+ implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_scale>(cute::make_shape(scale_mn,scale_k,L), StrideScale{});
646
+ implementable = implementable && (args.group_size == K || ((args.group_size % size<2>(TileShape{})) == 0));
647
+ implementable = implementable && args.group_size != 0;
648
+ implementable = implementable && (args.ptr_S != nullptr);
649
+
650
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
651
+ implementable = implementable && (args.ptr_Z == nullptr);
652
+ }
653
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
654
+ constexpr int min_tma_aligned_elements_zero = tma_alignment_bits / cutlass::sizeof_bits<ElementZero>::value;
655
+ implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_zero>(cute::make_shape(scale_mn,scale_k,L), StrideScale{});
656
+ implementable = implementable && (args.ptr_Z != nullptr);
657
+ }
658
+ else {
659
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in can_implement.");
660
+ }
661
+ }
662
+ else {
663
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in can_implement.");
664
+ }
665
+
666
+ if (!implementable) {
667
+ CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Problem Size doesn't meet the minimum alignment requirements for TMA.\n");
668
+ }
669
+ return implementable;
670
+ }
671
+
672
+ static constexpr int K_PIPE_MAX = DispatchPolicy::Stages;
673
+ static constexpr uint32_t TmaTransactionBytesMK = compute_tma_transaction_bytes_mk();
674
+ static constexpr uint32_t TmaTransactionBytesNK = compute_tma_transaction_bytes_nk();
675
+ static constexpr uint32_t TmaTransactionBytes = TmaTransactionBytesMK + TmaTransactionBytesNK;
676
+
677
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
678
+ CUTLASS_DEVICE
679
+ static void prefetch_tma_descriptors(Params const& mainloop_params) {
680
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_a.get_tma_descriptor());
681
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_b.get_tma_descriptor());
682
+
683
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
684
+ // Nothing extra to do
685
+ }
686
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
687
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_scale.get_tma_descriptor());
688
+ }
689
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
690
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_scale.get_tma_descriptor());
691
+ cute::prefetch_tma_descriptor(mainloop_params.tma_load_zero.get_tma_descriptor());
692
+ }
693
+ else {
694
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in TMA prefetch.");
695
+ }
696
+
697
+ }
698
+ // clang-format off
699
+
700
+ // Modified from upstream, should be kept close to that when possible
701
+ // the main difference is special handling for the prepacked A layout
702
+ //
703
+ // Set up the data needed by this collective for load and mma.
704
+ // Returns a tuple of tensors. The collective and the kernel layer have the
705
+ // contract Returned tuple must contain at least two elements, with the first
706
+ // two elements being: gA_mkl - The tma tensor, A after a local tile so it
707
+ // has shape (TILE_V,TILE_B,m,k,l) gB_nkl - The tma tensor, B after a local
708
+ // tile so it has shape (TILE_N,TILE_K,n,k,l) The rest of the tensors can be
709
+ // specified as needed by this collective.
710
+ // NOTE: TILE_B is the prepacked block index within a tile. TILE_V is the
711
+ // values within a prepacked block.
712
+ template <class ProblemShape_MNKL>
713
+ CUTLASS_DEVICE auto load_init(ProblemShape_MNKL const& problem_shape_MNKL,
714
+ Params const& mainloop_params) const {
715
+ using X = Underscore;
716
+ auto M = get<0>(problem_shape_MNKL), N = get<1>(problem_shape_MNKL),
717
+ K = get<2>(problem_shape_MNKL), L = get<3>(problem_shape_MNKL);
718
+
719
+ // (TILE_V,TILE_B,m,k,l)
720
+ auto make_gA_mkl = [&]() {
721
+ // ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
722
+ auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
723
+ Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(shape(layout));
724
+ return local_tile(mA_mkl,
725
+ make_shape(size<0>(layout), PPBlocksPerTile_MK{}),
726
+ make_coord(0, make_coord(_, _)));
727
+ };
728
+
729
+ // (TILE_N,TILE_K,n,k,l)
730
+ auto make_gB_nkl = [&]() {
731
+ Tensor mB_nkl =
732
+ mainloop_params.tma_load_b.get_tma_tensor(make_shape(N, K, L));
733
+ return local_tile(mB_nkl, TileShape{}, make_coord(_, _, _),
734
+ Step<X, _1, _1>{});
735
+ };
736
+
737
+ // (TILE_M,TILE_Scale_K,m,scale_k,l)
738
+ auto make_gS_mkl = [&]() {
739
+ auto scale_k = mainloop_params.scale_k;
740
+ Tensor mS_mkl = mainloop_params.tma_load_scale.get_tma_tensor(
741
+ make_shape(M, scale_k, L));
742
+ return local_tile(mS_mkl, ScaleTileShape{}, make_coord(_, _));
743
+ };
744
+
745
+ // (TILE_M,TILE_Scale_K,m,scale_k,l)
746
+ auto make_gZ_mkl = [&]() {
747
+ auto scale_k = mainloop_params.scale_k;
748
+ Tensor mZ_mkl = mainloop_params.tma_load_zero.get_tma_tensor(
749
+ make_shape(M, scale_k, L));
750
+ return local_tile(mZ_mkl, ScaleTileShape{}, make_coord(_, _));
751
+ };
752
+
753
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
754
+ return cute::make_tuple(make_gA_mkl(), make_gB_nkl());
755
+ } else if constexpr (KernelConversionMode ==
756
+ ConversionMode::ConvertAndScale) {
757
+ return cute::make_tuple(make_gA_mkl(), make_gB_nkl(), make_gS_mkl());
758
+ } else if constexpr (KernelConversionMode ==
759
+ ConversionMode::ConvertAndScaleWithZero) {
760
+ return cute::make_tuple(make_gA_mkl(), make_gB_nkl(), make_gS_mkl(),
761
+ make_gZ_mkl());
762
+ } else {
763
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
764
+ "Conversion mode not handled in load_init.");
765
+ }
766
+ }
767
+
768
+ // Similar to upstream, should be kept close to that when possible
769
+ // the main difference is in the layout comments
770
+ // clang-format off
771
+ /// Perform a collective-scoped matrix multiply-accumulate
772
+ /// Producer Perspective
773
+ /// This overload gets triggered when we have scales.
774
+ template <
775
+ class... Ts,
776
+ class KTileIterator, class BlockCoord
777
+ >
778
+ CUTLASS_DEVICE void
779
+ load(
780
+ Params const& mainloop_params,
781
+ MainloopPipeline pipeline,
782
+ PipelineState smem_pipe_write,
783
+ cute::tuple<Ts...> const& load_inputs,
784
+ BlockCoord const& blk_coord,
785
+ KTileIterator k_tile_iter, int k_tile_count,
786
+ int thread_idx,
787
+ uint32_t block_rank_in_cluster,
788
+ TensorStorage& shared_tensors) {
789
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
790
+ static_assert(sizeof... (Ts) == 2, "Direct convert needs two inputs");
791
+ }
792
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
793
+ static_assert(sizeof... (Ts) == 3, "Scaled convert needs three inputs");
794
+ }
795
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
796
+ static_assert(sizeof... (Ts) == 4, "Scaled and zero convert needs four inputs");
797
+ }
798
+ else {
799
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in TMA load.");
800
+ }
801
+
802
+ int lane_predicate = cute::elect_one_sync();
803
+
804
+ if (lane_predicate) {
805
+ Tensor sA_ = make_tensor(make_smem_ptr(shared_tensors.smem_A.begin()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE)
806
+ Tensor sB_ = make_tensor(make_smem_ptr(shared_tensors.smem_B.begin()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE)
807
+ Tensor sA = as_position_independent_swizzle_tensor(sA_); // (BLK_M,BLK_K,PIPE)
808
+ Tensor sB = as_position_independent_swizzle_tensor(sB_); // (BLK_N,BLK_K,PIPE)
809
+
810
+ //
811
+ // Prepare the TMA loads for A, B and Scales
812
+ //
813
+
814
+ constexpr uint32_t cluster_shape_x = get<0>(ClusterShape());
815
+ uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x};
816
+
817
+ Tensor gA_mkl = get<0>(load_inputs);
818
+ Tensor gB_nkl = get<1>(load_inputs);
819
+
820
+ auto block_tma_a = mainloop_params.tma_load_a.get_slice(cluster_local_block_id.y);
821
+ auto block_tma_b = mainloop_params.tma_load_b.get_slice(cluster_local_block_id.x);
822
+
823
+ // Partition the inputs based on the current block coordinates.
824
+ auto [m_coord, n_coord, k_coord, l_coord] = blk_coord;
825
+ Tensor gA = gA_mkl(_,_,m_coord,_,l_coord); // (TILE_V,TILE_B,k)
826
+ Tensor gB = gB_nkl(_,_,n_coord,_,l_coord); // (TILE_N,TILE_K,k)
827
+
828
+ // Applies the mapping from block_tma_a
829
+ Tensor tAgA = block_tma_a.partition_S(gA); // (TMA,TMA_M,TMA_K,k)
830
+ Tensor tAsA = block_tma_a.partition_D(sA); // (TMA,TMA_M,TMA_K,PIPE)
831
+
832
+ Tensor tBgB = block_tma_b.partition_S(gB); // (TMA,TMA_N,TMA_K,k)
833
+ Tensor tBsB = block_tma_b.partition_D(sB); // (TMA,TMA_N,TMA_K,PIPE)
834
+
835
+ uint16_t mcast_mask_a = 0;
836
+ uint16_t mcast_mask_b = 0;
837
+ uint16_t mcast_mask_s = 0;
838
+
839
+ // Issue TmaLoads
840
+ // Maps the tile -> block, value
841
+ if constexpr (cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD_MULTICAST>) {
842
+ auto block_layout = Layout<typename DispatchPolicy::ClusterShape>{}; // (m,n) -> block_id
843
+ for (int n = 0; n < size<1>(block_layout); ++n) {
844
+ mcast_mask_a |= (uint16_t(1) << block_layout(cluster_local_block_id.x,n,Int<0>{}));
845
+ }
846
+ }
847
+
848
+ if constexpr (cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD_MULTICAST>) {
849
+ auto block_layout = Layout<typename DispatchPolicy::ClusterShape>{}; // (m,n) -> block_id
850
+ for (int m = 0; m < size<0>(block_layout); ++m) {
851
+ mcast_mask_b |= (uint16_t(1) << block_layout(m,cluster_local_block_id.y,Int<0>{}));
852
+ }
853
+ }
854
+
855
+ auto extra_input_partitions = partition_extra_tma_inputs(mainloop_params, load_inputs, shared_tensors, cluster_local_block_id, m_coord, l_coord);
856
+
857
+ // Mainloop
858
+ CUTLASS_PRAGMA_NO_UNROLL
859
+ for ( ; k_tile_count > 0; --k_tile_count) {
860
+ // LOCK smem_pipe_write for _writing_
861
+ pipeline.producer_acquire(smem_pipe_write);
862
+
863
+ //
864
+ // Copy gmem to smem for *k_tile_iter
865
+ //
866
+
867
+ using BarrierType = typename MainloopPipeline::ProducerBarrierType;
868
+ BarrierType* tma_barrier = pipeline.producer_get_barrier(smem_pipe_write);
869
+
870
+ int write_stage = smem_pipe_write.index();
871
+ copy(mainloop_params.tma_load_a.with(*tma_barrier, mcast_mask_a), tAgA(_,_,_,*k_tile_iter), tAsA(_,_,_,write_stage));
872
+ copy(mainloop_params.tma_load_b.with(*tma_barrier, mcast_mask_b), tBgB(_,_,_,*k_tile_iter), tBsB(_,_,_,write_stage));
873
+
874
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
875
+ // Nothing extra to do.
876
+ }
877
+ else if constexpr (ModeHasScales) {
878
+ auto tSgS = get<0>(extra_input_partitions);
879
+ auto tSsS = get<1>(extra_input_partitions);
880
+
881
+ // Temporary factor which will determine which k tile to reload from gmem. Needed so we don't modify tma transaction bytes
882
+ // on the fly.
883
+ // We must do a ceiling divide here to correctly handle with group_size == K. In that case, we don't require that K
884
+ // is a multiple of the threadblock tile K
885
+ const int ReloadFactor = (mainloop_params.group_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{});
886
+ const int scale_load_k = *k_tile_iter / ReloadFactor; // This will always be 0 when group_size == K.
887
+ copy(mainloop_params.tma_load_scale.with(*tma_barrier, mcast_mask_s), tSgS(_,_,_,scale_load_k), tSsS(_,_,_,write_stage));
888
+
889
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
890
+ // Nothing extra to do
891
+ }
892
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
893
+ auto tZgZ = get<2>(extra_input_partitions);
894
+ auto tZsZ = get<3>(extra_input_partitions);
895
+ copy(mainloop_params.tma_load_zero.with(*tma_barrier, mcast_mask_s), tZgZ(_,_,_,scale_load_k), tZsZ(_,_,_,write_stage));
896
+ }
897
+ else {
898
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled for TMA copy op.");
899
+ }
900
+ }
901
+ else {
902
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled for TMA copy op.");
903
+ }
904
+
905
+ ++k_tile_iter;
906
+
907
+ // Advance smem_pipe_write
908
+ ++smem_pipe_write;
909
+ }
910
+ }
911
+ }
912
+ // clang-format off
913
+
914
+ // Same as upstream, should be kept the same when possible, not formatted for
915
+ // easier comparison
916
+ // clang-format off
917
+ // Perform a Producer Epilogue to prevent early exit of blocks in a Cluster
918
+ CUTLASS_DEVICE void
919
+ load_tail(MainloopPipeline pipeline, PipelineState smem_pipe_write) {
920
+ int lane_predicate = cute::elect_one_sync();
921
+
922
+ // Issue the epilogue waits
923
+ if (lane_predicate) {
924
+ /* This helps avoid early exit of blocks in Cluster
925
+ * Waits for all stages to either be released (all
926
+ * Consumer UNLOCKs), or if the stage was never used
927
+ * then would just be acquired since the phase was
928
+ * still inverted from make_producer_start_state
929
+ */
930
+ pipeline.producer_tail(smem_pipe_write);
931
+ }
932
+ }
933
+ // clang-format on
934
+
935
+ // Modified from upstream, should be kept close to that when possible
936
+ // the main differences are handling the prepacked A layout, and separating
937
+ // the loading of A from upcoverting A
938
+ //
939
+ // Perform a collective-scoped matrix multiply-accumulate
940
+ // Consumer Perspective
941
+ template <class FrgTensorC>
942
+ CUTLASS_DEVICE void mma(MainloopPipeline pipeline,
943
+ PipelineState smem_pipe_read, FrgTensorC& accum,
944
+ int k_tile_count, int thread_idx,
945
+ TensorStorage& shared_tensors,
946
+ Params const& mainloop_params) {
947
+ static_assert(is_rmem<FrgTensorC>::value,
948
+ "C tensor must be rmem resident.");
949
+ static_assert(cute::rank(SmemLayoutB{}) == 3,
950
+ "Smem layout must be rank 3.");
951
+ static_assert(cute::rank(SmemLayoutAtomB{}) == 2,
952
+ "SmemLayoutAtomB must be rank 2.");
953
+ static_assert(!cute::is_void_v<SmemCopyAtomA>,
954
+ "SM90 GMMA mainloops must specify a non-void copy atom for "
955
+ "RF sourced instructions.");
956
+ static_assert(cute::is_void_v<SmemCopyAtomB>,
957
+ "SM90 GMMA mainloops cannot have a non-void copy atom for "
958
+ "smem sourced instructions.");
959
+
960
+ // Obtain warp index
961
+ int warp_idx = canonical_warp_idx_sync();
962
+ [[maybe_unused]] int warp_group_thread_idx = thread_idx % 128;
963
+
964
+ // ((T, (FrgV,(RestM, RestK)), (BlocksM, BlocksK), pipe) -> offset
965
+ auto constexpr smem_A = SmemLayoutA{};
966
+
967
+ // convert:
968
+ // ((T, (MMA,(MMA_M, MMA_K)), (BlocksM, BlocksK), pipe) -> offset
969
+ // to:
970
+ // (T, MMA, ((MMA_M, BlocksM), (MMA_K, BlocksK)), pipe) -> offset
971
+ // which can be thought of as:
972
+ // (T, MMA, (MMA_M, MMA_K), pipe) -> offset
973
+ auto constexpr smem_A_mma_ =
974
+ make_layout(get<0, 0>(smem_A), get<0, 1, 0>(smem_A),
975
+ zip(get<0, 1, 1>(smem_A), get<1>(smem_A)), get<2>(smem_A));
976
+ // flatten to:
977
+ // (T, MMA, MMA_M, MMA_K, pipe) -> offset
978
+ auto constexpr smem_A_mma = smem_A_mma_(_, _, make_coord(_, _), _);
979
+
980
+ Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.begin()),
981
+ smem_A_mma); // (T, MMA, MMA_M, MMA_K, pipe)
982
+ Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.begin()),
983
+ SmemLayoutB{}); // (BLK_N,BLK_K,PIPE)
984
+
985
+ //
986
+ // Define C accumulators and A/B partitioning
987
+ //
988
+
989
+ TiledMma tiled_mma;
990
+ auto thread_mma = tiled_mma.get_thread_slice(thread_idx);
991
+
992
+ Tensor tCsA = sA(thread_idx, _, _, _, _); // (MMA,MMA_M,MMA_K,PIPE)
993
+ Tensor tCsB = thread_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE)
994
+
995
+ // Allocate fragments and descriptors
996
+ Tensor tCrA_load = make_tensor<ElementA>(
997
+ tCsA(_, _, _, Int<0>{}).shape()); // (MMA,MMA_N,MMA_K)
998
+ Tensor tCrA_mma = make_fragment_like<ElementMma>(tCrA_load);
999
+
1000
+ Tensor tCrB = thread_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE)
1001
+
1002
+ static constexpr int A_CPY_VEC =
1003
+ decltype(max_common_vector(tCsA, tCrA_load)){};
1004
+
1005
+ static constexpr int COVERSION_WIDTH =
1006
+ std::min(A_CPY_VEC, int(size<0>(tCrA_mma)));
1007
+
1008
+ auto load_A_to_registers = [&](int read_stage) {
1009
+ copy(create_auto_vectorizing_copy<ElementA, decltype(A_CPY_VEC)>(),
1010
+ tCsA(_, _, _, read_stage), tCrA_load(_, _, _));
1011
+ };
1012
+
1013
+ // Partition of thread -> shared and thread -> RF
1014
+ auto partitioned_extra_info =
1015
+ partition_extra_mma_info(thread_mma, shared_tensors);
1016
+ auto copy_partitions_extra_info = retile_extra_mma_info(
1017
+ tiled_mma, partitioned_extra_info, warp_group_thread_idx);
1018
+ CUTE_STATIC_ASSERT_V(size<1>(tCrA_mma) == size<1>(accum)); // MMA_M
1019
+ CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<2>(accum)); // N
1020
+ CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCsB)); // K
1021
+ CUTE_STATIC_ASSERT_V(size<3>(tCsA) == size<3>(tCsB)); // PIPE
1022
+ CUTE_STATIC_ASSERT_V(Int<DispatchPolicy::Stages>{} == size<2>(sB)); // PIPE
1023
+
1024
+ //
1025
+ // PIPELINED MAIN LOOP
1026
+ //
1027
+
1028
+ auto convert_A = [&, a_vec = Int<COVERSION_WIDTH>{}](int k_block,
1029
+ int read_stage) {
1030
+ load_extra_info_to_registers(partitioned_extra_info,
1031
+ copy_partitions_extra_info, k_block,
1032
+ read_stage);
1033
+ transform_A_kblock(tCrA_load, a_vec, tCrA_mma, partitioned_extra_info,
1034
+ k_block);
1035
+ };
1036
+
1037
+ // We release buffers to producer warps(dma load) with some mmas in flight
1038
+ PipelineState smem_pipe_release = smem_pipe_read;
1039
+
1040
+ tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
1041
+
1042
+ warpgroup_fence_operand(accum);
1043
+
1044
+ constexpr int K_BLOCK_MAX = size<2>(tCrA_load);
1045
+
1046
+ ConsumerToken barrier_token = {BarrierStatus::WaitAgain};
1047
+ // first k tile
1048
+ {
1049
+ barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
1050
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
1051
+
1052
+ int read_stage = smem_pipe_read.index();
1053
+ ++smem_pipe_read;
1054
+ barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
1055
+
1056
+ // copy smem->rmem for A operand
1057
+ load_A_to_registers(read_stage);
1058
+ convert_A(0, read_stage);
1059
+
1060
+ // Unroll the K mode manually to set scale D to 1
1061
+ CUTLASS_PRAGMA_UNROLL
1062
+ for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block) {
1063
+ if (k_block < K_BLOCK_MAX - 1) {
1064
+ convert_A(k_block + 1, smem_pipe_read.index());
1065
+ }
1066
+ warpgroup_arrive();
1067
+ // (V,M) x (V,N) => (V,M,N)
1068
+ cute::gemm(tiled_mma, tCrA_mma(_, _, k_block),
1069
+ tCrB(_, _, k_block, read_stage), accum);
1070
+ tiled_mma.accumulate_ = GMMA::ScaleOut::One;
1071
+ warpgroup_commit_batch();
1072
+ }
1073
+
1074
+ --k_tile_count;
1075
+ if (k_tile_count > 0) {
1076
+ // Wait for K_BLOCK_MAX - 1 to be in flight to ensure that it is safe to
1077
+ // overwrite the A registers for the first mma.
1078
+ warpgroup_wait<K_BLOCK_MAX - 1>();
1079
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
1080
+ load_A_to_registers(smem_pipe_read.index());
1081
+ convert_A(0, smem_pipe_read.index());
1082
+ }
1083
+ }
1084
+
1085
+ if (k_tile_count == 0) {
1086
+ return;
1087
+ }
1088
+
1089
+ warpgroup_fence_operand(accum);
1090
+ // Mainloop GMMAs
1091
+ CUTLASS_PRAGMA_NO_UNROLL
1092
+ for (; k_tile_count > 1; --k_tile_count) {
1093
+ //
1094
+ // Compute on k_tile
1095
+ //
1096
+
1097
+ int read_stage = smem_pipe_read.index();
1098
+ ++smem_pipe_read;
1099
+
1100
+ warpgroup_fence_operand(accum);
1101
+ // Unroll the K mode manually to set scale D to 1
1102
+ CUTLASS_PRAGMA_UNROLL
1103
+ for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block) {
1104
+ warpgroup_arrive();
1105
+ // (V,M) x (V,N) => (V,M,N)
1106
+ cute::gemm(tiled_mma, tCrA_mma(_, _, k_block),
1107
+ tCrB(_, _, k_block, read_stage), accum);
1108
+ tiled_mma.accumulate_ = GMMA::ScaleOut::One;
1109
+ warpgroup_commit_batch();
1110
+
1111
+ warpgroup_wait<K_BLOCK_MAX - 1>();
1112
+ if (k_block == K_BLOCK_MAX - 1) {
1113
+ // We have K_BLOCK_MAX - 1 GMMA instructions pending for this stage,
1114
+ // so we can release prior barrier
1115
+ pipeline.consumer_release(
1116
+ smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_
1117
+ // on it
1118
+ ++smem_pipe_release;
1119
+ }
1120
+
1121
+ if (k_block == 0) {
1122
+ barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
1123
+ }
1124
+
1125
+ if (k_block == K_BLOCK_MAX - 1) {
1126
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
1127
+ load_A_to_registers(smem_pipe_read.index());
1128
+ convert_A(0, smem_pipe_read.index());
1129
+ } else {
1130
+ convert_A(k_block + 1, read_stage);
1131
+ }
1132
+ }
1133
+ warpgroup_fence_operand(accum);
1134
+ }
1135
+
1136
+ warpgroup_fence_operand(accum);
1137
+
1138
+ {
1139
+ //
1140
+ // Compute on k_tile
1141
+ //
1142
+
1143
+ int read_stage = smem_pipe_read.index();
1144
+
1145
+ warpgroup_fence_operand(accum);
1146
+
1147
+ // Unroll the K mode manually to set scale D to 1
1148
+ CUTLASS_PRAGMA_UNROLL
1149
+ for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block) {
1150
+ warpgroup_arrive();
1151
+ // (V,M) x (V,N) => (V,M,N)
1152
+ cute::gemm(tiled_mma, tCrA_mma(_, _, k_block),
1153
+ tCrB(_, _, k_block, read_stage), accum);
1154
+ tiled_mma.accumulate_ = GMMA::ScaleOut::One;
1155
+ warpgroup_commit_batch();
1156
+ warpgroup_wait<K_BLOCK_MAX - 1>();
1157
+ if (k_block == K_BLOCK_MAX - 1) {
1158
+ // release prior barrier
1159
+ pipeline.consumer_release(
1160
+ smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_
1161
+ // on it
1162
+ ++smem_pipe_release;
1163
+ }
1164
+
1165
+ if (k_block < K_BLOCK_MAX - 1) {
1166
+ convert_A(k_block + 1, read_stage);
1167
+ }
1168
+ }
1169
+ }
1170
+
1171
+ warpgroup_fence_operand(accum);
1172
+ }
1173
+
1174
+ // Perform a Consumer Epilogue to release all buffers
1175
+ CUTLASS_DEVICE void mma_tail(MainloopPipeline pipeline,
1176
+ PipelineState smem_pipe_release,
1177
+ int k_tile_count) {
1178
+ // Prologue GMMAs
1179
+ int prologue_mma_count = 1;
1180
+ k_tile_count -= prologue_mma_count;
1181
+
1182
+ smem_pipe_release.advance(k_tile_count);
1183
+
1184
+ // Wait on all GMMAs to complete
1185
+ warpgroup_wait<0>();
1186
+
1187
+ for (int count = 0; count < prologue_mma_count; ++count) {
1188
+ pipeline.consumer_release(
1189
+ smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on
1190
+ // it
1191
+ ++smem_pipe_release;
1192
+ }
1193
+ }
1194
+
1195
+ private:
1196
+ // Same as upstream, should be kept the same when possible, not formatted for
1197
+ // easier comparison
1198
+ // clang-format off
1199
+ /// Utilities for any additional inputs inside of the TMA load
1200
+ template <class... Ts>
1201
+ CUTLASS_DEVICE
1202
+ auto partition_extra_tma_inputs(
1203
+ Params const& mainloop_params,
1204
+ cute::tuple<Ts...> const& load_inputs,
1205
+ TensorStorage& shared_tensors,
1206
+ uint2 const& cluster_local_block_id,
1207
+ int const m_coord,
1208
+ int const l_coord) {
1209
+
1210
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
1211
+ return cute::make_tuple();
1212
+ }
1213
+ else if constexpr (ModeHasScales) {
1214
+ Tensor sS = make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_K,PIPE)
1215
+ Tensor gS_mkl = get<2>(load_inputs);
1216
+ auto block_tma_s = mainloop_params.tma_load_scale.get_slice(cluster_local_block_id.y);
1217
+ Tensor gS = gS_mkl(_,_,m_coord,_,l_coord); // (BLK_M,BLK_K,k)
1218
+
1219
+ Tensor tSgS = block_tma_s.partition_S(gS); // (TMA,TMA_M,TMA_K,k)
1220
+ Tensor tSsS = block_tma_s.partition_D(sS); // (TMA,TMA_M,TMA_K,PIPE)
1221
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
1222
+ return cute::make_tuple(tSgS, tSsS);
1223
+ }
1224
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
1225
+ Tensor sZ = make_tensor(make_smem_ptr(shared_tensors.smem_zero.begin()), SmemLayoutScale{}); // (BLK_M,BLK_K,PIPE)
1226
+ Tensor gZ_mkl = get<3>(load_inputs);
1227
+ auto block_tma_z = mainloop_params.tma_load_zero.get_slice(cluster_local_block_id.y);
1228
+ Tensor gZ = gZ_mkl(_,_,m_coord,_,l_coord); // (BLK_M,BLK_K,k)
1229
+
1230
+ Tensor tZgZ = block_tma_z.partition_S(gZ); // (TMA,TMA_M,TMA_K,k)
1231
+ Tensor tZsZ = block_tma_z.partition_D(sZ); // (TMA,TMA_M,TMA_K,PIPE)
1232
+ return cute::make_tuple(tSgS, tSsS, tZgZ, tZsZ);
1233
+ }
1234
+ else {
1235
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled for input partitioning.");
1236
+ }
1237
+ }
1238
+ else {
1239
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled for input partitioning.");
1240
+ }
1241
+ }
1242
+ // clang-format off
1243
+
1244
+ // Same as upstream, should be kept the same when possible, not formatted for
1245
+ // easier comparison
1246
+ // clang-format off
1247
+ /// Utilities for partitioning extra inputs for loading from smem in the mainloop.
1248
+ template <class ThreadMma>
1249
+ CUTLASS_DEVICE
1250
+ auto partition_extra_mma_info(
1251
+ ThreadMma const& mma_thread_slice,
1252
+ TensorStorage& shared_tensors) {
1253
+
1254
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
1255
+ // nothing to do
1256
+ return cute::make_tuple();
1257
+ }
1258
+ else if constexpr (ModeHasScales) {
1259
+ Tensor sS = make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{});// (BLK_M,BLK_SCALE_K,PIPE)
1260
+ Tensor tCsS = mma_thread_slice.partition_A(sS);
1261
+ Tensor tCrS = make_tensor<ElementScale>(mma_thread_slice.partition_fragment_A(sS(_,_,Int<0>{})).shape());
1262
+
1263
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
1264
+ return cute::make_tuple(tCsS, tCrS);
1265
+ }
1266
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
1267
+ Tensor sZ = make_tensor(make_smem_ptr(shared_tensors.smem_zero.begin()), SmemLayoutScale{});// (BLK_M,BLK_SCALE_K,PIPE)
1268
+ Tensor tCsZ = mma_thread_slice.partition_A(sZ);
1269
+ Tensor tCrZ = make_tensor<ElementZero>(mma_thread_slice.partition_fragment_A(sZ(_,_,Int<0>{})).shape());
1270
+ return cute::make_tuple(tCsS, tCrS, tCsZ, tCrZ);
1271
+ }
1272
+ else {
1273
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in A -> RF path.");
1274
+ }
1275
+ }
1276
+ else {
1277
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in A -> RF path.");
1278
+ }
1279
+ }
1280
+ // clang-format on
1281
+
1282
+ // Same as upstream, should be kept the same when possible, not formatted for
1283
+ // easier comparison
1284
+ // clang-format off
1285
+ /// Returns the tiled copy and copy views for the extra inputs.
1286
+ template <class TiledMma, class... Ts>
1287
+ CUTLASS_DEVICE
1288
+ auto retile_extra_mma_info(
1289
+ TiledMma const& tiled_mma,
1290
+ cute::tuple<Ts...>& partitioned_extra_info,
1291
+ int const warp_group_thread_idx) {
1292
+
1293
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
1294
+ // nothing to do
1295
+ return cute::make_tuple();
1296
+ }
1297
+ else if constexpr (ModeHasScales) {
1298
+ auto smem_tiled_copy_S = make_tiled_copy_A(SmemCopyAtomScale{}, tiled_mma);
1299
+ auto smem_thr_copy_S = smem_tiled_copy_S.get_thread_slice(warp_group_thread_idx);
1300
+ Tensor tCrS_copy_view = smem_thr_copy_S.retile_D(cute::get<1>(partitioned_extra_info)); // (CPY,CPY_M,CPY_K)
1301
+
1302
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
1303
+ return cute::make_tuple(smem_tiled_copy_S, tCrS_copy_view);
1304
+ }
1305
+ else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) {
1306
+ Tensor tCrZ_copy_view = smem_thr_copy_S.retile_D(cute::get<3>(partitioned_extra_info)); // (CPY,CPY_M,CPY_K)
1307
+ return cute::make_tuple(smem_tiled_copy_S, tCrS_copy_view, tCrZ_copy_view);
1308
+ }
1309
+ else {
1310
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in A -> RF path.");
1311
+ }
1312
+ }
1313
+ else {
1314
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>, "Conversion mode not handled in A -> RF path.");
1315
+ }
1316
+ }
1317
+ // clang-format on
1318
+
1319
+ // Similar to `copy_A_and_extra_info` upstream, should be kept the same when
1320
+ // possible
1321
+ // the main differences this only loads the extra info into registers and
1322
+ // not A (since we now preload more of A in the main pipeline)
1323
+ // Load scales and zeros into registers if required
1324
+ template <class... Ts, class... Us>
1325
+ CUTLASS_DEVICE void load_extra_info_to_registers(
1326
+ cute::tuple<Ts...> const& partitioned_mma_extra_info,
1327
+ cute::tuple<Us...> const& tiled_copy_and_views, int k_block,
1328
+ int read_stage) {
1329
+ if (k_block == 0) {
1330
+ // We are starting a new k-tile so copy the scale
1331
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
1332
+ // nothing to do
1333
+ } else if constexpr (ModeHasScales) {
1334
+ auto smem_tiled_copy_S = cute::get<0>(tiled_copy_and_views);
1335
+ auto tCrS_copy_view = cute::get<1>(tiled_copy_and_views);
1336
+ auto tCsS = cute::get<0>(partitioned_mma_extra_info);
1337
+ copy(smem_tiled_copy_S, tCsS(_, _, k_block, read_stage),
1338
+ tCrS_copy_view(_, _, k_block));
1339
+ if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) {
1340
+ // Nothing extra to do
1341
+ } else if constexpr (KernelConversionMode ==
1342
+ ConversionMode::ConvertAndScaleWithZero) {
1343
+ auto tCsZ = cute::get<2>(partitioned_mma_extra_info);
1344
+ auto tCrZ_copy_view = cute::get<2>(tiled_copy_and_views);
1345
+ copy(smem_tiled_copy_S, tCsZ(_, _, k_block, read_stage),
1346
+ tCrZ_copy_view(_, _, k_block));
1347
+ } else {
1348
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
1349
+ "Conversion mode not handled in A -> RF path.");
1350
+ }
1351
+ } else {
1352
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
1353
+ "Conversion mode not handled in A -> RF path.");
1354
+ }
1355
+ }
1356
+ }
1357
+
1358
+ // Similar to upstream, should be kept the same when possible.
1359
+ // the main differences are that `convert_tensor` supports interleaved
1360
+ // layouts and bfloat16 has been optimized. `transform_internal_A` has also
1361
+ // been inlined for code simplicity.
1362
+ // Utilities to transform A.
1363
+ template <class TCrA_load, int VectorWidthA, class TCrA_mma, class... Ts>
1364
+ CUTLASS_DEVICE void transform_A_kblock(
1365
+ TCrA_load const& tCrA_load, cute::Int<VectorWidthA> vec_A,
1366
+ TCrA_mma& tCrA_mma, cute::tuple<Ts...> const& partitioned_extra_info,
1367
+ int const k_block) {
1368
+ auto in = tCrA_load(_, _, k_block);
1369
+ auto out = tCrA_mma(_, _, k_block);
1370
+
1371
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
1372
+ convert_tensor<IlvdBlkLayout>(in, out, vec_A);
1373
+ } else if constexpr (ModeHasScales) {
1374
+ auto tCrS = cute::get<1>(partitioned_extra_info);
1375
+ auto converted_inputs =
1376
+ make_fragment_like<ElementScale>(tCrA_mma)(_, _, k_block);
1377
+ auto scales = tCrS(_, _, 0);
1378
+
1379
+ // First, we upcast the inputs to the scale type
1380
+ convert_tensor<IlvdBlkLayout>(in, converted_inputs, vec_A);
1381
+ // Apply scales and broadcast across inputs, store in converted_inputs
1382
+
1383
+ // We need to cast to nv_bfloat16 for the multiply since
1384
+ // `cutlass::bfloat16_t` has an overloaded operator* that upconverts to
1385
+ // float, which nvcc will not optimize to using vectorized fma
1386
+ // instructions (i.e. hfma.bf16_v2)
1387
+ if constexpr (std::is_same_v<ElementScale, cutlass::bfloat16_t>) {
1388
+ cute::transform(
1389
+ recast<nv_bfloat16>(converted_inputs), recast<nv_bfloat16>(scales),
1390
+ recast<nv_bfloat16>(converted_inputs), cute::multiplies{});
1391
+ } else {
1392
+ cute::transform(converted_inputs, scales, converted_inputs,
1393
+ cute::multiplies{});
1394
+ }
1395
+
1396
+ // Apply zeros if required
1397
+ if constexpr (KernelConversionMode ==
1398
+ ConversionMode::ConvertAndScaleWithZero) {
1399
+ auto tCrZ = cute::get<3>(partitioned_extra_info);
1400
+ auto converted_zeros = make_fragment_like<ElementScale>(tCrZ)(_, _, 0);
1401
+
1402
+ convert_tensor<void>(tCrZ(_, _, 0), converted_zeros);
1403
+ if constexpr (std::is_same_v<ElementScale, cutlass::bfloat16_t>) {
1404
+ cute::transform(recast<nv_bfloat16>(converted_inputs),
1405
+ recast<nv_bfloat16>(converted_zeros),
1406
+ recast<nv_bfloat16>(converted_inputs), cute::plus{});
1407
+ } else {
1408
+ cute::transform(converted_inputs, converted_zeros, converted_inputs,
1409
+ cute::plus{});
1410
+ }
1411
+ }
1412
+
1413
+ // Finally, we convert the scaled inputs to the mma type.
1414
+ convert_tensor<void>(converted_inputs, out);
1415
+ } else {
1416
+ static_assert(cutlass::detail::dependent_false<KernelSchedule>,
1417
+ "No A data is loaded.");
1418
+ }
1419
+ }
1420
+
1421
+ // Modified from upstream, should be kept the same when possible
1422
+ // the main differences is that this version supports interleaved converts
1423
+ // Utilities for transforming the A operand prior to issuing tensorcore math.
1424
+ template <typename IlvdBlkLayout, class EngineIn, class EngineOut,
1425
+ class TensorLayout,
1426
+ int ConversionVectorWidth = cosize_v<TensorLayout>>
1427
+ CUTLASS_DEVICE void convert_tensor(
1428
+ Tensor<EngineIn, TensorLayout> const& in,
1429
+ Tensor<EngineOut, TensorLayout>& out,
1430
+ cute::Int<ConversionVectorWidth> width = {}) {
1431
+ // This is an element-wise conversion where we expect both tensors to have
1432
+ // the same layout. As a result, we can cast as a cutlass array to use the
1433
+ // fast numeric converters without worrying about indexing into the layout.
1434
+ constexpr int N = cosize_v<TensorLayout>;
1435
+
1436
+ // The inputs must be backed by registers & be statically sized.
1437
+ static_assert(is_rmem<EngineIn>::value,
1438
+ "Input tensor for A conversion must come from registers");
1439
+ static_assert(is_rmem<EngineOut>::value,
1440
+ "Output tensor for A conversion must come from registers");
1441
+ static_assert(is_static_v<TensorLayout>,
1442
+ "Tensor layout for the conversion must be static");
1443
+ static_assert(cosize_v<TensorLayout> == size(TensorLayout{}),
1444
+ "Cosize and size of the layout must be equal.");
1445
+ static_assert(
1446
+ N % ConversionVectorWidth == 0,
1447
+ "Conversion vector width must divide cosize of the tensor layout.");
1448
+
1449
+ using SrcType = typename EngineIn::value_type;
1450
+ using DstType = typename EngineOut::value_type;
1451
+
1452
+ using SrcArray = cutlass::Array<SrcType, ConversionVectorWidth>;
1453
+ using DstArray = cutlass::Array<DstType, ConversionVectorWidth>;
1454
+
1455
+ constexpr cutlass::FloatRoundStyle RoundStyle =
1456
+ cutlass::FloatRoundStyle::round_to_nearest;
1457
+
1458
+ using Converter = cutlass::InterleavedNumericArrayConverter<
1459
+ IlvdBlkLayout, DstType, SrcType, ConversionVectorWidth, RoundStyle>;
1460
+
1461
+ constexpr int NumIterations = N / ConversionVectorWidth;
1462
+
1463
+ for (int ii = 0; ii < NumIterations; ++ii) {
1464
+ SrcArray const* src_array_ptr =
1465
+ reinterpret_cast<SrcArray const*>(raw_pointer_cast(in.data())) + ii;
1466
+ DstArray* dst_array_ptr =
1467
+ reinterpret_cast<DstArray*>(raw_pointer_cast(out.data())) + ii;
1468
+ *dst_array_ptr = Converter::convert(*src_array_ptr);
1469
+ }
1470
+ }
1471
+};
1472
+
1473
+} // namespace machete
csrc/quantization/machete/machete_mm_kernel.cuh
new
+237
@@ -0,0 +1,237 @@
1
+#pragma once
2
+
3
+#include <ATen/cuda/CUDAContext.h>
4
+#include <c10/cuda/CUDAGuard.h>
5
+#include <torch/all.h>
6
+
7
+// clang-format off
8
+// The cutlass include order matters (annoyingly)
9
+#include "cutlass/cutlass.h"
10
+
11
+#include "cute/tensor.hpp"
12
+#include "cutlass/tensor_ref.h"
13
+#include "cutlass/epilogue/collective/default_epilogue.hpp"
14
+#include "cutlass/epilogue/thread/linear_combination.h"
15
+#include "cutlass/gemm/dispatch_policy.hpp"
16
+#include "cutlass/gemm/collective/collective_builder.hpp"
17
+#include "cutlass/epilogue/collective/collective_builder.hpp"
18
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
19
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
20
+// clang-format on
21
+
22
+#include "cutlass_extensions/cute_utils.cuh"
23
+#include "cutlass_extensions/vllm_numeric_conversion.cuh"
24
+#include "machete_collective_builder.cuh"
25
+#include "machete_prepacked_layout.cuh"
26
+#include "machete_interleaving_utils.cuh"
27
+
28
+namespace machete {
29
+
30
+using namespace cute;
31
+
32
+// NOTE This kernel computes D = alpha * A * B + beta * C by computing
33
+// D^t = alpha * B^t * A^t + beta * C^t, this is because the wgmma
34
+// instructions only support sourcing from registers for the left-hand
35
+// operand, we want to upconvert/decompress the quantized operand in
36
+// register. Since the primary use case we want to support is Y = XW^t where
37
+// W is quantized, in this situation or right-hand operand is quantized so
38
+// we compute the transpose to move it to the left-hand side.
39
+template <typename ElementA_, typename ElementB_, typename ElementD_,
40
+ typename AccumulatorT, typename ScaleT, typename ZeroT,
41
+ class KernelSchedule, typename ScheduleConfig, bool with_C,
42
+ bool with_scales, bool with_zeropoints>
43
+struct MacheteKernelTemplate {
44
+ using MmaType = ElementA_;
45
+ using ElementA = ElementA_;
46
+ using ElementB = ElementB_;
47
+ using ElementD = ElementD_;
48
+ using ElementC = cute::conditional_t<with_C, ElementD, void>;
49
+ using ElementZ = ZeroT;
50
+ using ElementS = ScaleT;
51
+
52
+ using ElementAccumulator =
53
+ AccumulatorT; // Element type for internal accumulation
54
+ using ElementCompute = AccumulatorT; // For Epilogue
55
+
56
+ using BTypeTuple = cute::conditional_t<
57
+ with_scales,
58
+ cute::conditional_t<with_zeropoints,
59
+ cute::tuple<ElementB, ElementS, ElementZ>,
60
+ cute::tuple<ElementB, ElementS>>,
61
+ ElementB>;
62
+
63
+ using LayoutA = cutlass::layout::RowMajor;
64
+ using LayoutC = cutlass::layout::RowMajor;
65
+ using LayoutD = LayoutC;
66
+ using LayoutScale = cutlass::layout::RowMajor;
67
+ // not actually used since B has the prepacked layout, but required by cutlass
68
+ using _LayoutB = cutlass::layout::ColumnMajor;
69
+
70
+ // Interface strides expected by create_arguments (will get transposed)
71
+ using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
72
+ using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
73
+ using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
74
+ using StrideS = cutlass::detail::TagToStrideA_t<LayoutScale>;
75
+ using StrideZ = StrideS;
76
+
77
+ using LayoutA_Transpose =
78
+ typename cutlass::layout::LayoutTranspose<LayoutA>::type;
79
+ using LayoutC_Transpose =
80
+ typename cutlass::layout::LayoutTranspose<LayoutC>::type;
81
+ using LayoutD_Transpose =
82
+ typename cutlass::layout::LayoutTranspose<LayoutD>::type;
83
+
84
+ using ArchTag = cutlass::arch::Sm90;
85
+ using OperatorClass = cutlass::arch::OpClassTensorOp;
86
+
87
+ using PrepackedLayoutB =
88
+ PrepackedLayoutBTemplate<ElementA_, ElementB_, ElementD_, AccumulatorT,
89
+ LayoutA_Transpose, KernelSchedule>;
90
+
91
+ static int constexpr TileShapeK =
92
+ 128 * 8 / cutlass::sizeof_bits<MmaType>::value;
93
+ static int constexpr AlignmentA = 128 / cutlass::sizeof_bits_v<ElementA>;
94
+ static int constexpr AlignmentB = 128 / cutlass::sizeof_bits_v<ElementB>;
95
+ static int constexpr AlignmentC =
96
+ (with_C) ? 128 / cutlass::sizeof_bits_v<ElementC> : 0;
97
+ static int constexpr AlignmentD = 128 / cutlass::sizeof_bits_v<ElementD>;
98
+
99
+ using TileShape = decltype(append(typename ScheduleConfig::TileShapeNM{},
100
+ cute::Int<TileShapeK>{}));
101
+ using ClusterShape = typename ScheduleConfig::ClusterShape;
102
+ using EpilogueSchedule = typename ScheduleConfig::EpilogueSchedule;
103
+ using EpilogueTileType = typename ScheduleConfig::EpilogueTileType;
104
+ using TileScheduler = typename ScheduleConfig::TileScheduler;
105
+
106
+ using CollectiveEpilogue =
107
+ typename cutlass::epilogue::collective::CollectiveBuilder<
108
+ ArchTag, OperatorClass, TileShape, ClusterShape, EpilogueTileType,
109
+ ElementAccumulator, ElementAccumulator, ElementC, LayoutC_Transpose,
110
+ AlignmentC, ElementD, LayoutD_Transpose, AlignmentD,
111
+ EpilogueSchedule>::CollectiveOp;
112
+
113
+ using CollectiveMainloop =
114
+ typename cutlass::gemm::collective::VLLMCollectiveBuilder<
115
+ cutlass::gemm::collective::MacheteKernelTag, ArchTag, OperatorClass,
116
+ BTypeTuple, PrepackedLayoutB, AlignmentB, ElementA, LayoutA_Transpose,
117
+ AlignmentA, ElementAccumulator, TileShape, ClusterShape,
118
+ cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
119
+ sizeof(typename CollectiveEpilogue::SharedStorage))>,
120
+ KernelSchedule>::CollectiveOp;
121
+
122
+ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
123
+ Shape<int, int, int, int>, // Indicates ProblemShape
124
+ CollectiveMainloop, CollectiveEpilogue, TileScheduler>;
125
+ using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
126
+
127
+ // stride_B is unused (since B is prepacked), but still required by cutlass
128
+ using _StrideB = cutlass::detail::TagToStrideB_t<_LayoutB>;
129
+
130
+ using Arguments = typename Gemm::Arguments;
131
+ using MainloopArguments = typename GemmKernel::MainloopArguments;
132
+ using EpilogueArguments = typename GemmKernel::EpilogueArguments;
133
+
134
+ template <typename ShapeA, typename ShapeC, typename ShapeD, typename ShapeS,
135
+ typename ShapeZ>
136
+ static Arguments create_arguments(
137
+ cudaStream_t stream,
138
+ ElementA const* A_ptr, // A is an MxK matrix
139
+ Layout<ShapeA, StrideA> const& layout_A,
140
+ ElementB const* B_ptr, // B is an KxN prepacked matrix
141
+ ElementD* D_ptr, // D is an MxN matrix
142
+ Layout<ShapeD, StrideD> const& layout_D,
143
+ ElementC const* C_ptr, // C is an MxN matrix
144
+ std::optional<Layout<ShapeC, StrideC>> const& layout_C,
145
+ ElementS const* S_ptr, // S is an scale_KxN matrix
146
+ std::optional<Layout<ShapeS, StrideS>> const& layout_S,
147
+ ElementZ const* Z_ptr, // Z is an scale_KxN matrix
148
+ std::optional<Layout<ShapeZ, StrideZ>> const& layout_Z,
149
+ ElementCompute alpha, ElementCompute beta,
150
+ std::optional<int> maybe_group_size) {
151
+ static_assert(!with_zeropoints || with_scales);
152
+
153
+ int M = size<0>(layout_A), N = size<1>(layout_D), K = size<1>(layout_A);
154
+
155
+ int const group_size = maybe_group_size.value_or(K);
156
+ int const scale_k = (K + group_size - 1) / group_size;
157
+
158
+ TORCH_CHECK(size<0>(layout_A) == M && size<1>(layout_A) == K);
159
+ TORCH_CHECK(size<0>(layout_D) == M && size<1>(layout_D) == N);
160
+
161
+ if constexpr (with_C) {
162
+ TORCH_CHECK(C_ptr && layout_C);
163
+ } else {
164
+ TORCH_CHECK(!C_ptr, "C not supported");
165
+ }
166
+
167
+ if constexpr (with_scales) {
168
+ TORCH_CHECK(S_ptr && layout_S);
169
+ TORCH_CHECK((size<0>(*layout_S) == scale_k && size<1>(*layout_S) == N));
170
+ } else {
171
+ TORCH_CHECK(!S_ptr, "Scales not supported");
172
+ }
173
+
174
+ if constexpr (with_zeropoints) {
175
+ TORCH_CHECK(Z_ptr && layout_Z);
176
+ TORCH_CHECK((size<0>(*layout_Z) == scale_k && size<1>(*layout_Z) == N));
177
+ TORCH_CHECK(layout_S && *layout_Z == *layout_S,
178
+ "Scales and zeros must have the same layout");
179
+ } else {
180
+ TORCH_CHECK(!Z_ptr, "Zeropoints not supported");
181
+ }
182
+
183
+ // Transpose A and D
184
+ // A doesn't need to be transposed since cutlass expects a NxK matrix
185
+ // for B (which is At)
186
+ auto stride_At = layout_A.stride();
187
+ auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
188
+ auto stride_Ct = stride_Dt;
189
+ if (layout_C) {
190
+ stride_Ct = permute_layout<1, 0, 2>(*layout_C).stride();
191
+ }
192
+
193
+ MainloopArguments mainloop_arguments{};
194
+ EpilogueArguments epilogue_arguments{
195
+ {alpha, beta}, C_ptr, stride_Ct, D_ptr, stride_Dt};
196
+
197
+ if constexpr (with_scales && with_zeropoints) {
198
+ auto stride_S = permute_layout<1, 0, 2>(*layout_S).stride();
199
+ mainloop_arguments =
200
+ MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At,
201
+ S_ptr, stride_S, group_size, Z_ptr};
202
+ } else if constexpr (with_scales) {
203
+ auto stride_S = permute_layout<1, 0, 2>(*layout_S).stride();
204
+ mainloop_arguments = MainloopArguments{
205
+ B_ptr, _StrideB{}, A_ptr, stride_At, S_ptr, stride_S, group_size};
206
+ } else {
207
+ mainloop_arguments =
208
+ MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At};
209
+ }
210
+
211
+ return Arguments{cutlass::gemm::GemmUniversalMode::kGemm,
212
+ {N, M, K, 1},
213
+ mainloop_arguments,
214
+ epilogue_arguments};
215
+ };
216
+
217
+ static size_t get_workspace_size(Arguments const& args) {
218
+ return Gemm::get_workspace_size(args);
219
+ }
220
+
221
+ static bool can_implement(Arguments const& args) {
222
+ return Gemm::can_implement(args) == cutlass::Status::kSuccess;
223
+ }
224
+
225
+ static void run(Arguments const& args, void* workspace, cudaStream_t stream) {
226
+ Gemm gemm_op;
227
+
228
+ cutlass::Status status = gemm_op.initialize(args, workspace, stream);
229
+ TORCH_CHECK(status == cutlass::Status::kSuccess,
230
+ "Machete kernel failed to initialize workspace");
231
+
232
+ status = gemm_op.run(stream);
233
+ TORCH_CHECK(status == cutlass::Status::kSuccess, "Machete kernel failed");
234
+ }
235
+};
236
+
237
+}; // namespace machete
csrc/quantization/machete/machete_mm_launcher.cuh
new
+95
@@ -0,0 +1,95 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+#include <Python.h>
5
+
6
+#include "machete_mm_kernel.cuh"
7
+#include "cutlass_extensions/torch_utils.hpp"
8
+
9
+namespace machete {
10
+
11
+struct PyTorchArguments {
12
+ torch::Tensor const& A;
13
+ torch::Tensor const& B;
14
+ c10::optional<torch::Tensor> const& scales;
15
+ c10::optional<torch::Tensor> const& zeros;
16
+ c10::optional<int64_t> group_size;
17
+ c10::optional<torch::Tensor> const& C;
18
+ c10::optional<double> alpha;
19
+ c10::optional<double> beta;
20
+ c10::optional<std::string> schedule;
21
+};
22
+
23
+template <typename MacheteKernel>
24
+torch::Tensor run_impl(PyTorchArguments args) {
25
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(args.A));
26
+
27
+ auto device = args.A.device();
28
+ auto stream = at::cuda::getCurrentCUDAStream(device.index());
29
+
30
+ using EleA = typename MacheteKernel::ElementA;
31
+ using EleB = typename MacheteKernel::ElementB;
32
+ using EleC = typename MacheteKernel::ElementC;
33
+ using EleD = typename MacheteKernel::ElementD;
34
+ using EleScale = typename MacheteKernel::ElementS;
35
+ using EleZero = typename MacheteKernel::ElementZ;
36
+
37
+ using StrideA = typename MacheteKernel::StrideA;
38
+ using StrideC = typename MacheteKernel::StrideC;
39
+ using StrideD = typename MacheteKernel::StrideD;
40
+ using StrideS = typename MacheteKernel::StrideS;
41
+ using StrideZ = typename MacheteKernel::StrideZ;
42
+
43
+ int M = args.A.size(0);
44
+ int N = args.B.size(1);
45
+ int K = args.A.size(1);
46
+
47
+ // Allocate output
48
+ torch::Tensor D =
49
+ torch::empty({M, N}, torch::TensorOptions()
50
+ .dtype(equivalent_scalar_type_v<EleD>)
51
+ .device(device));
52
+
53
+ auto const &A = args.A, &B = args.B;
54
+ auto const &C = args.C, &scales = args.scales, &zeros = args.zeros;
55
+
56
+ auto layout_A = make_cute_layout<StrideA>(A, "A");
57
+ auto layout_D = make_cute_layout<StrideD>(D, "D");
58
+ auto layout_C = maybe_make_cute_layout<StrideC>(C, "C");
59
+ auto layout_S = maybe_make_cute_layout<StrideS>(scales, "scales");
60
+ auto layout_Z = maybe_make_cute_layout<StrideZ>(zeros, "zeros");
61
+
62
+ auto A_ptr = static_cast<EleA const*>(A.const_data_ptr());
63
+ auto B_ptr = static_cast<EleB const*>(B.const_data_ptr());
64
+ auto D_ptr = static_cast<EleD*>(D.mutable_data_ptr());
65
+ auto C_ptr = static_cast<EleC const*>(C ? C->const_data_ptr() : nullptr);
66
+ auto S_ptr =
67
+ static_cast<EleScale const*>(scales ? scales->const_data_ptr() : nullptr);
68
+ auto Z_ptr =
69
+ static_cast<EleZero const*>(zeros ? zeros->const_data_ptr() : nullptr);
70
+
71
+ auto arguments = MacheteKernel::create_arguments(
72
+ stream, A_ptr, layout_A, B_ptr, D_ptr, layout_D, C_ptr, layout_C, S_ptr,
73
+ layout_S, Z_ptr, layout_Z, args.alpha.value_or(1), args.beta.value_or(0),
74
+ args.group_size.value_or(K));
75
+ TORCH_CHECK(MacheteKernel::can_implement(arguments),
76
+ "Machete kernel cannot be run with these arguments");
77
+
78
+ size_t workspace_size = MacheteKernel::get_workspace_size(arguments);
79
+ torch::Tensor workspace = torch::empty(
80
+ workspace_size, torch::TensorOptions().dtype(torch::kU8).device(device));
81
+
82
+ MacheteKernel::run(arguments, workspace.mutable_data_ptr(), stream);
83
+
84
+ return D;
85
+};
86
+
87
+template <typename ElementA, typename ElementB, typename ElementD = ElementA,
88
+ typename AccumulatorT = float, typename ScaleT = ElementA,
89
+ typename ZeroT = ElementA>
90
+struct GemmDispatcher {
91
+ static torch::Tensor dispatch(PyTorchArguments args);
92
+ static std::vector<std::string> supported_schedules();
93
+};
94
+
95
+}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepack_kernel.cuh
new
+62
@@ -0,0 +1,62 @@
1
+#pragma once
2
+
3
+#include "machete_mm_kernel.cuh"
4
+#include "cutlass_extensions/cute_utils.cuh"
5
+#include "cutlass_extensions/torch_utils.hpp"
6
+
7
+namespace machete {
8
+
9
+template <typename TileShapeNKL, typename ElementB, typename BInTensor,
10
+ typename BTiledOutTensor>
11
+static __global__ void prepack_B_kernel(BInTensor B_in,
12
+ BTiledOutTensor B_tiled_out) {
13
+ auto tB_in = local_tile(B_in, TileShapeNKL{},
14
+ make_coord(blockIdx.x, blockIdx.y, blockIdx.z));
15
+ auto tB_out = B_tiled_out(make_coord(_, _),
16
+ make_coord(blockIdx.x, blockIdx.y), blockIdx.z);
17
+
18
+ auto tiled_copy = make_tiled_copy(Copy_Atom<DefaultCopy, ElementB>{},
19
+ Layout<Shape<_4, _32>, Stride<_32, _1>>{},
20
+ Layout<Shape<_1, _2>>{});
21
+
22
+ auto thr_copy = tiled_copy.get_thread_slice(threadIdx.x);
23
+
24
+ Tensor thr_tile_S = thr_copy.partition_S(tB_in);
25
+ Tensor thr_tile_D = thr_copy.partition_D(tB_out);
26
+
27
+ // Construct a register-backed Tensor with the same shape as each thread's
28
+ // partition
29
+ auto fragment = make_tensor<ElementB>(shape(thr_tile_D));
30
+
31
+ // Copy from GMEM to RMEM and from RMEM to GMEM
32
+ copy(tiled_copy, thr_tile_S, fragment);
33
+ copy(Copy_Atom<DefaultCopy, uint8_t>{}, fragment, thr_tile_D);
34
+}
35
+
36
+template <typename PrepackedLayoutB, typename InLayout>
37
+static void prepack_B(cudaStream_t stream,
38
+ typename PrepackedLayoutB::ElementB const* B_in_ptr,
39
+ InLayout B_layout,
40
+ typename PrepackedLayoutB::ElementB* B_out_ptr) {
41
+ using TileShapeNKL =
42
+ decltype(append(typename PrepackedLayoutB::PPBlockShape_NK{}, _1{}));
43
+ auto ilvd_NKbNbKL_to_offset =
44
+ PrepackedLayoutB::ilvd_NKbNbKL_to_offset(shape(B_layout));
45
+
46
+ TORCH_CHECK(size<0>(B_layout) % size<0>(TileShapeNKL{}) == 0);
47
+ TORCH_CHECK(size<1>(B_layout) % size<1>(TileShapeNKL{}) == 0);
48
+ TORCH_CHECK(size<2>(B_layout) % size<2>(TileShapeNKL{}) == 0);
49
+
50
+ auto N_tiles = size<0>(B_layout) / size<0>(TileShapeNKL{});
51
+ auto K_tiles = size<1>(B_layout) / size<1>(TileShapeNKL{});
52
+ auto L_tiles = size<2>(B_layout) / size<2>(TileShapeNKL{});
53
+
54
+ auto B_in = make_tensor(get_logical_ptr(B_in_ptr), B_layout);
55
+ auto B_tiled_out =
56
+ make_tensor(get_logical_ptr(B_out_ptr), ilvd_NKbNbKL_to_offset);
57
+
58
+ prepack_B_kernel<TileShapeNKL, typename PrepackedLayoutB::ElementB>
59
+ <<<dim3(N_tiles, K_tiles, L_tiles), 128, 0, stream>>>(B_in, B_tiled_out);
60
+}
61
+
62
+}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepack_launcher.cuh
new
+71
@@ -0,0 +1,71 @@
1
+#pragma once
2
+
3
+#include "machete_prepack_kernel.cuh"
4
+#include "cutlass_extensions/torch_utils.hpp"
5
+
6
+namespace machete {
7
+
8
+template <typename PrepackedLayoutB>
9
+torch::Tensor prepack_impl(torch::Tensor const B) {
10
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(B));
11
+ using ElementB = typename PrepackedLayoutB::ElementB;
12
+ using PPBlockShape_NK = typename PrepackedLayoutB::PPBlockShape_NK;
13
+
14
+ auto device = B.device();
15
+ auto stream = at::cuda::getCurrentCUDAStream(device.index());
16
+ auto B_ptr = static_cast<ElementB const*>(B.const_data_ptr());
17
+ // elements per storage item for B
18
+ auto eles_per_storage =
19
+ (B.dtype().itemsize() * 8) / cute::sizeof_bits_v<ElementB>;
20
+
21
+ // torch B passed in is/should be (packed_K,N), the kernel expects (N,K,L) (to
22
+ // match cutlass using (N,K,L) for B), so we transpose B to (N,packed_K,L)
23
+ auto Bt_packed = B.t();
24
+
25
+ TORCH_CHECK(
26
+ (B.size(0) * eles_per_storage) % size<1>(PPBlockShape_NK{}) == 0,
27
+ "B.shape[0] (in terms of unpacked elements) must be a multiple of ",
28
+ size<1>(PPBlockShape_NK{}));
29
+ TORCH_CHECK(B.size(1) % size<0>(PPBlockShape_NK{}) == 0,
30
+ "B.shape[1] must be a multiple of ", size<0>(PPBlockShape_NK{}));
31
+
32
+ using StrideB = cutlass::detail::TagToStrideB_t<cutlass::layout::ColumnMajor>;
33
+ auto const l_Bt_packed = make_cute_layout<StrideB>(Bt_packed, "B");
34
+
35
+ // convert (N,packed_K,L) layout to (N,K,L) layout
36
+ // in effect we want to do: blocked_product(layout_Bt_packed,
37
+ // make_ordered_layout(make_shape(_1{}, eles_per_storage, _1{}),
38
+ // Step<_1, _0, _2>{}));
39
+ // but blocked_product does not support dynamic strides so we implement the
40
+ // equivalent manually,
41
+ // new_shape = (N, packed_K, L) * (1, eles_per_storage, 1) -> (N, K, L)
42
+ // new_stride = (s0, s1, s2) * (eles_per_storage, 1, eles_per_storage)
43
+ // when s1 == 1
44
+ TORCH_CHECK(stride<1>(l_Bt_packed) == 1);
45
+ // clang-format off
46
+ auto const layout_Bt = make_layout(
47
+ transform_with_idx(l_Bt_packed.shape(), [&](auto ele, auto idx) {
48
+ return idx == 1 ? ele * eles_per_storage : ele;
49
+ }),
50
+ transform_with_idx(l_Bt_packed.stride(), [&](auto ele, auto idx) {
51
+ return idx != 1 ? ele * eles_per_storage : ele;
52
+ }));
53
+ // clang-format on
54
+
55
+ // Allocate output
56
+ torch::Tensor D = torch::empty_like(B);
57
+
58
+ prepack_B<PrepackedLayoutB>(stream, B_ptr, layout_Bt,
59
+ static_cast<ElementB*>(D.mutable_data_ptr()));
60
+
61
+ return D;
62
+};
63
+
64
+template <typename ElementA, typename ElementB, typename ElementD,
65
+ typename AccumulatorT = float, typename ScaleT = cutlass::half_t,
66
+ typename ZeroT = cutlass::half_t>
67
+struct PrepackBDispatcher {
68
+ static torch::Tensor dispatch(torch::Tensor B);
69
+};
70
+
71
+}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepacked_layout.cuh
new
+220
@@ -0,0 +1,220 @@
1
+#pragma once
2
+
3
+#include <ATen/cuda/CUDAContext.h>
4
+#include <c10/cuda/CUDAGuard.h>
5
+#include <torch/all.h>
6
+
7
+// clang-format off
8
+// The cutlass include order matters (annoyingly)
9
+
10
+#include "cutlass/cutlass.h"
11
+
12
+#include "cute/tensor.hpp"
13
+#include "cutlass/tensor_ref.h"
14
+#include "cutlass/epilogue/collective/default_epilogue.hpp"
15
+#include "cutlass/epilogue/thread/linear_combination.h"
16
+#include "cutlass/gemm/dispatch_policy.hpp"
17
+#include "cutlass/gemm/collective/collective_builder.hpp"
18
+#include "cutlass/epilogue/collective/collective_builder.hpp"
19
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
20
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
21
+// clang-format on
22
+
23
+#include "cutlass_extensions/cute_utils.cuh"
24
+#include "machete_collective_builder.cuh"
25
+#include "machete_interleaving_utils.cuh"
26
+
27
+namespace machete {
28
+
29
+using namespace cute;
30
+
31
+struct IlvBlkLayoutAuto {};
32
+
33
+// This defines a prepacked layout for the B matrix, where the matrix is broken
34
+// up into PPBlockShape_NK blocks. The data within each block is then compactly
35
+// stored in memory such that when performing a TiledMMA operation with the same
36
+// shape as prepacked block, all the data for a given thread is contiguous in
37
+// memory. This allows us to use wider shared memory loads when loading B from
38
+// shared memory. The values within a thread are also potentially interlaeved
39
+// inorder to allow for more efficient upconverting.
40
+//
41
+// The contract here is that the `TiledMma` determined below matches the one
42
+// ultimately used in the kernel. (this is also why the other element types are
43
+// required along with the kernel schedule)
44
+template <typename ElementA_, typename ElementB_, typename ElementD_,
45
+ typename AccumulatorT, class LayoutB, class KernelSchedule,
46
+ typename IlvBlkLayout_ = IlvBlkLayoutAuto>
47
+// clang-format on
48
+struct PrepackedLayoutBTemplate {
49
+ using MmaType = ElementA_;
50
+ using ElementA = ElementA_;
51
+ using ElementB = ElementB_;
52
+ using ElementD = ElementD_;
53
+ using ElementAccumulator =
54
+ AccumulatorT; // Element type for internal accumulation
55
+ using ElementMma = MmaType;
56
+
57
+ // Only use interleaved layouts for subbyte weights, prmt instructions makes
58
+ // non-interleaved layouts for 8bit+ weights efficient enough we don't need
59
+ // iterleaved layouts
60
+ using IlvdBlkLayout = std::conditional_t<
61
+ std::is_same_v<IlvBlkLayout_, IlvBlkLayoutAuto>,
62
+ std::conditional_t<sizeof_bits_v<ElementB> <= 4,
63
+ decltype(get_interleaved_blk_layout<
64
+ ElementB, sizeof_bits_v<ElementA>, 32>()),
65
+ void>,
66
+ IlvBlkLayout_>;
67
+
68
+ // TODO (LucasWilkinson): compare the performance for other sizes
69
+ // Prepacked block shape, smallest layout atom for loading into registers
70
+ // (can contain multiple wgmma instructions worth of data in one block)
71
+ // We ideally want this to be configured such that a thread can perform 128bit
72
+ // loads, i.e. we amount of data associated with each thread within a
73
+ // prepacked block is a multiple of 128bits, when using a cooperative sechdule
74
+ // we have 256 threads working a single block at a time, this means each
75
+ // thread works on `sizeof_bits_v<ElementB> * (128*64) / 256` bits of data,
76
+ // for a 4bit type this would be 128bits
77
+ using PPBlockShape_NK = Shape<_128, _64>;
78
+
79
+ // Create the shape of the tile anticipated to be used by the GEMM kernel,
80
+ // when the kernel executes we will compute `Ct = Bt * At` since the
81
+ // quantized weights (B), must be the lhs operand so the flow through
82
+ // registers.
83
+ // The _128 here doesn't actually impact the shape of the stored tile directly
84
+ // but may impact the op selected by rs_op_selector
85
+ using GemmTileShape = decltype(make_shape(size<0>(PPBlockShape_NK{}), _128{},
86
+ size<1>(PPBlockShape_NK{})));
87
+
88
+ static constexpr cute::GMMA::Major GmmaMajorB =
89
+ gmma_rs_tag_to_major_B<LayoutB>();
90
+
91
+ // For coop schedules we have two warp groups cooperatively issuing wgmma
92
+ // instructions so we use 2 atoms along the M dim (one for each warpgroup)
93
+ using AtomLayoutMNK = cute::conditional_t<
94
+ cute::is_same_v<KernelSchedule,
95
+ KernelTmaWarpSpecializedCooperativeMixedInput>,
96
+ Layout<Shape<_2, _1, _1>>, Layout<Shape<_1, _1, _1>>>;
97
+
98
+ using TiledMma = decltype(cute::make_tiled_mma(
99
+ cute::GMMA::rs_op_selector<ElementMma, ElementMma, ElementAccumulator,
100
+ GemmTileShape, GMMA::Major::K, GmmaMajorB>(),
101
+ AtomLayoutMNK{}));
102
+
103
+ // Prepacked block, (athrid, val) -> (N,K)
104
+ // i.e. ((ThrV,(ThrN,ThrK)),(FrgV,(RestN,RestK,...))) -> (N,K)
105
+ CUTE_HOST_DEVICE static constexpr auto ppblock_TV_to_NK() {
106
+ return TiledMma{}.thrfrg_A(make_layout(PPBlockShape_NK{}));
107
+ }
108
+
109
+ // Prepacked block, (N,K) -> (athrid, val)
110
+ // i.e. (N,K) -> ((ThrV,(ThrN,ThrK)),(FrgV,(RestN,RestK,...)))
111
+ CUTE_HOST_DEVICE static constexpr auto ppblock_NK_to_TV() {
112
+ return right_inverse(ppblock_TV_to_NK()).with_shape(PPBlockShape_NK{});
113
+ }
114
+
115
+ // Prepacked block, (athrid, val) -> (storage_offset)
116
+ // i.e. ((ThrV,(ThrN,ThrK)),(FrgV,(RestN,RestK,...))) -> (storage_idx)
117
+ CUTE_HOST_DEVICE static constexpr auto ppblock_TV_to_offset() {
118
+ // Return iterleaved layout
119
+ return make_ordered_layout(shape(ppblock_TV_to_NK()), Step<_1, _0>{});
120
+ }
121
+
122
+ // Prepacked block, (athrid, val) -> (storage_offset)
123
+ // i.e. ((ThrV,(ThrM,ThrK)),(IlvdFrgV,(RestM,RestK,...))) -> (storage_idx)
124
+ CUTE_HOST_DEVICE static constexpr auto ppblock_ilvd_TV_to_offset() {
125
+ auto layout_no_interleave =
126
+ make_ordered_layout(shape(ppblock_TV_to_NK()), Step<_1, _0>{});
127
+
128
+ if constexpr (std::is_same_v<IlvdBlkLayout, void>) {
129
+ return layout_no_interleave;
130
+ } else {
131
+ // interleave by transforming FrgV into interleaved blocks where each
132
+ // block has the layout IlvdBlkLayout, for example if IlvdBlkLayout is
133
+ // (2, 2) : (2, 1) then we get: ((2, 2), size(FrgV) / 4) : ((2, 1), 4)
134
+ // if FrgV is {A, B, C, D, E, F, G, H}
135
+ // then ((IlvBlk), FrgB) is {A, C, B, D, C, G, D, H}
136
+ auto frgV = get<1, 0>(layout_no_interleave);
137
+ auto ilvdBlk = IlvdBlkLayout{};
138
+ static_assert(size(frgV) % 4 == 0, "FrgV must be divisible by 4");
139
+ auto ilvd_FrgV = make_layout(
140
+ make_shape(shape(ilvdBlk), Int<size(frgV) / size(ilvdBlk)>{}),
141
+ make_stride(stride(ilvdBlk), size(ilvdBlk)));
142
+
143
+ // Return iterleaved layout
144
+ return make_layout(
145
+ get<0>(layout_no_interleave),
146
+ make_layout(ilvd_FrgV, get<1, 1>(layout_no_interleave)));
147
+ }
148
+ }
149
+
150
+ // Prepacked block, (M,K) -> (storage_offset)
151
+ CUTE_HOST_DEVICE static constexpr auto ppblock_ilvd_NK_to_offset() {
152
+ // do (M,K) -> (athrid, val) -> (storage_idx)
153
+ return ppblock_ilvd_TV_to_offset().compose(ppblock_NK_to_TV());
154
+ }
155
+
156
+ // ((athrid, val), (BlocksN, BlocksK), L) -> (storage_idx)
157
+ template <typename Shape_NKL>
158
+ CUTE_HOST_DEVICE static constexpr auto TVbNbKL_to_offset(
159
+ Shape_NKL shape_mkl) {
160
+ constexpr auto block_layout = ppblock_TV_to_offset();
161
+
162
+ // (BlocksN, BlocksK, L)
163
+ auto blocks_shape =
164
+ cute::transform(shape_mkl, append(PPBlockShape_NK{}, _1{}),
165
+ [](auto x, auto y) { return x / y; });
166
+
167
+ // ((athrid, val), (BlocksN, BlocksK, L)) -> (storage_idx)
168
+ auto result = make_layout(
169
+ block_layout,
170
+ make_layout(blocks_shape,
171
+ compact_col_major(blocks_shape, size(block_layout))));
172
+
173
+ // ((athrid, val), (BlocksN, BlocksK, L))
174
+ // => ((athrid, val), (BlocksN, BlocksK), L)
175
+ return group<1, 3>(result(_, repeat<rank<1>(result)>(_)));
176
+ }
177
+
178
+ // ((BlockN, BlockK), (BlocksN, BlocksK), L) -> (storage_idx)
179
+ template <typename Shape_NKL>
180
+ CUTE_HOST_DEVICE static constexpr auto ilvd_NKbNbKL_to_offset(
181
+ Shape_NKL shape_mkl) {
182
+ constexpr auto block_layout = ppblock_ilvd_NK_to_offset();
183
+
184
+ // (BlocksN, BlocksK, L)
185
+ auto blocks_shape =
186
+ cute::transform(shape_mkl, append(PPBlockShape_NK{}, _1{}),
187
+ [](auto x, auto y) { return x / y; });
188
+
189
+ // ((athrid, val), (BlocksN, BlocksK, L)) -> (storage_idx)
190
+ auto result = make_layout(
191
+ block_layout,
192
+ make_layout(blocks_shape,
193
+ compact_col_major(blocks_shape, size(block_layout))));
194
+
195
+ // ((athrid, val), (BlocksN, BlocksK, L)) => ((athrid, val), (BlocksN,
196
+ // BlocksK), L)
197
+ return group<1, 3>(result(_, repeat<rank<1>(result)>(_)));
198
+ }
199
+
200
+ // ((athrid, val), (BlocksN, BlocksK, L)) -> (N, K, L)
201
+ template <class Shape_NKL>
202
+ CUTE_HOST_DEVICE static auto TVbNbK_to_NKL(Shape_NKL shape_mkl) {
203
+ auto tile = make_tile(make_layout(size<0>(PPBlockShape_NK{})),
204
+ make_layout(size<1>(PPBlockShape_NK{})));
205
+
206
+ // ((BlockN, BlockK), (BlocksN, BlocksK, L)) -> (N, K, L)
207
+ auto tiled_A = zipped_divide(make_layout(shape_mkl), tile);
208
+ return tiled_A.compose(ppblock_TV_to_NK(), _);
209
+ }
210
+
211
+ // (N, K, L) -> ((athrid, val), (BlocksN, BlocksK), L)
212
+ template <class Shape_NKL>
213
+ CUTE_HOST_DEVICE static auto NKL_to_TVbNbK(Shape_NKL shape_mkl) {
214
+ auto TVbNbK_to_NKL_layout = TVbNbK_to_NKL(shape_mkl);
215
+ return blocked_product(ppblock_NK_to_TV(),
216
+ make_layout(shape<1>(TVbNbK_to_NKL_layout)));
217
+ }
218
+};
219
+
220
+}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_pytorch.cu
new
+79
@@ -0,0 +1,79 @@
1
+#include "machete_mm_launcher.cuh"
2
+#include "machete_prepack_launcher.cuh"
3
+#include "core/scalar_type.hpp"
4
+
5
+namespace machete {
6
+
7
+using namespace vllm;
8
+
9
+//
10
+// Utils (type dispatching)
11
+//
12
+
13
+template <typename Fn>
14
+static auto scalar_type_dispatch(ScalarType const& type, Fn fn) {
15
+ if (type == vllm::kU4) {
16
+ return fn(cutlass::uint4b_t{});
17
+ } else if (type == vllm::kU8) {
18
+ return fn(cutlass::uint8_t{});
19
+ } else if (type == vllm::kU4B8) {
20
+ return fn(cutlass::vllm_uint4b8_t{});
21
+ } else if (type == vllm::kU8B128) {
22
+ return fn(cutlass::vllm_uint8b128_t{});
23
+ } else {
24
+ TORCH_CHECK(false, "Unsupported type ", type.str());
25
+ }
26
+}
27
+
28
+#define AT_DISPATCH_CASE_SUPPORTED_COMPUTE_TYPES(...) \
29
+ AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__)
30
+
31
+#define AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(TYPE, NAME, ...) \
32
+ AT_DISPATCH_SWITCH(TYPE, NAME, \
33
+ AT_DISPATCH_CASE_SUPPORTED_COMPUTE_TYPES(__VA_ARGS__))
34
+
35
+//
36
+// Interface
37
+//
38
+
39
+std::vector<std::string> supported_schedules(ScalarTypeTorchPtr const& btype) {
40
+ return scalar_type_dispatch(*btype, [&](auto BType) {
41
+ return GemmDispatcher<half_t, decltype(BType)>::supported_schedules();
42
+ });
43
+}
44
+
45
+torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
46
+ ScalarTypeTorchPtr const& btype,
47
+ c10::optional<torch::Tensor> const& scales,
48
+ c10::optional<torch::Tensor> const& zeros,
49
+ c10::optional<int64_t> group_size,
50
+ c10::optional<torch::Tensor> const& C,
51
+ c10::optional<double> alpha, c10::optional<double> beta,
52
+ c10::optional<std::string> schedule) {
53
+ auto args = PyTorchArguments{.A = A,
54
+ .B = B,
55
+ .scales = scales,
56
+ .zeros = zeros,
57
+ .group_size = group_size,
58
+ .C = C,
59
+ .alpha = alpha,
60
+ .beta = beta,
61
+ .schedule = schedule};
62
+
63
+ return scalar_type_dispatch(*btype, [&](auto BType) {
64
+ return AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(
65
+ A.scalar_type(), "machete_gemm", [&] {
66
+ using ComputeType = equivalent_cutlass_type_t<scalar_t>;
67
+ return GemmDispatcher<ComputeType, decltype(BType)>::dispatch(args);
68
+ });
69
+ });
70
+}
71
+
72
+torch::Tensor prepack_B(torch::Tensor const& B,
73
+ ScalarTypeTorchPtr const& btype) {
74
+ return scalar_type_dispatch(*btype, [&](auto BType) {
75
+ return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
76
+ });
77
+}
78
+
79
+}; // namespace machete
csrc/torch_bindings.cpp
+15
@@ -133,6 +133,21 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
133
ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
134
ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
135
136
+ // Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
137
+ ops.def("machete_supported_schedules", &machete::supported_schedules);
138
+ ops.def(
139
+ "machete_gemm(Tensor A, Tensor B,"
140
+ " __torch__.torch.classes._core_C.ScalarType btype,"
141
+ " Tensor? scales, Tensor? zeros, int? group_size,"
142
+ " Tensor? C, float? alpha, float? beta, str? schedule)"
143
+ "-> Tensor");
144
+ ops.impl("machete_gemm", torch::kCUDA, &machete::gemm);
145
+ ops.def(
146
+ "machete_prepack_B(Tensor B,"
147
+ " __torch__.torch.classes._core_C.ScalarType btype)"
148
+ "-> Tensor");
149
+ ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
150
+
151
// gptq_marlin Optimized Quantized GEMM for GPTQ.
152
ops.def("gptq_marlin_gemm", &gptq_marlin_gemm);
153
ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
tests/kernels/test_machete_gemm.py
new
+272
@@ -0,0 +1,272 @@
1
+"""Tests for the machete kernel.
2
+
3
+Run `pytest tests/kernels/test_machete_gemm.py`.
4
+"""
5
+
6
+import math
7
+from typing import Optional, Tuple
8
+
9
+import pytest
10
+import torch
11
+
12
+from vllm import _custom_ops as ops
13
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
+ pack_rows, quantize_weights)
15
+from vllm.platforms import current_platform
16
+from vllm.scalar_type import ScalarType, scalar_types
17
+
18
+CUDA_DEVICES = [
19
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
20
+]
21
+
22
+MNK_SHAPES = [
23
+ (1, 128, 128),
24
+ (1, 512, 1024),
25
+ (1, 4096, 4096),
26
+ (13, 8192, 4096),
27
+ (26, 4096, 8192),
28
+ (1, 4096, 4096),
29
+ (257, 128, 4096),
30
+ (257, 4224, 4160),
31
+ (257, 4096, 4096),
32
+ (64, 4096, 4096),
33
+]
34
+
35
+ACT_TYPES = [torch.float16, torch.bfloat16]
36
+WTYPE_ZEROPOINTS = [
37
+ # GPTQ style
38
+ (scalar_types.uint4b8, False),
39
+ (scalar_types.uint8b128, False),
40
+ # AWQ style
41
+ (scalar_types.uint4, True),
42
+ (scalar_types.uint8, True),
43
+]
44
+
45
+# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
46
+# unit tests to a common utility function. Currently the use of
47
+# `is_quant_method_supported` conflates kernels with quantization methods
48
+# an assumption which is breaking down as quantizations methods can have
49
+# have kernels and some kernels support multiple quantization methods.
50
+IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
51
+
52
+
53
+def rand_data(shape, dtype=torch.float16):
54
+ return 10 * (torch.rand(shape, dtype=dtype, device="cuda") - 0.3)
55
+
56
+
57
+def maybe_convert_zeropoints(zps: Optional[torch.Tensor], s: torch.Tensor):
58
+ return zps if zps is None else -1 * s * (zps.to(s.dtype))
59
+
60
+
61
+def machete_quantize_and_pack(w: torch.Tensor,
62
+ wtype: ScalarType,
63
+ group_size: int,
64
+ zero_points: bool = False):
65
+ assert wtype.is_integer(), "TODO: support floating point weights"
66
+
67
+ w_ref, w_q, w_s, w_zp = quantize_weights(
68
+ w,
69
+ wtype,
70
+ group_size,
71
+ zero_points=zero_points,
72
+ # to match how the kernel applies zps
73
+ ref_zero_points_after_scales=True)
74
+
75
+ w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
76
+ w_q = w_q.t().contiguous().t() # convert to col major
77
+ w_q_machete = ops.machete_prepack_B(w_q, wtype)
78
+
79
+ return w_ref, w_q_machete, w_s, w_zp
80
+
81
+
82
+def machete_gemm_test_helper(a: torch.Tensor, b: torch.Tensor,
83
+ wtype: ScalarType, group_size: int,
84
+ zero_points: bool):
85
+ w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
86
+ b, wtype, group_size, zero_points)
87
+
88
+ output_ref = torch.matmul(a, w_ref)
89
+
90
+ output = ops.machete_gemm(
91
+ a=a,
92
+ b_q=w_q_packed,
93
+ b_type=wtype,
94
+ b_scales=w_s,
95
+ b_zeros=maybe_convert_zeropoints(w_zp, w_s),
96
+ b_group_size=group_size,
97
+ )
98
+
99
+ # Relax atol as our reduction dim becomes larger (more rounding error)
100
+ # Relax atol when we have zeropoints since the way machete applies
101
+ # zeropoints (after scales) causes noise around 0
102
+ atol = 1 if zero_points else min(5e-2 * math.sqrt(a.shape[1]), 1)
103
+ torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
104
+
105
+
106
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
107
+ reason="Machete is not supported on this GPU type.")
108
+@pytest.mark.parametrize("shape",
109
+ MNK_SHAPES,
110
+ ids=lambda x: "x".join(str(v) for v in x))
111
+@pytest.mark.parametrize("atype", ACT_TYPES, ids=lambda x: str(x))
112
+@pytest.mark.parametrize("wtype_zeropoints", WTYPE_ZEROPOINTS)
113
+@pytest.mark.parametrize("group_size", [128, None])
114
+def test_machete_all_schedules(shape, atype: torch.dtype,
115
+ wtype_zeropoints: Tuple[ScalarType, bool],
116
+ group_size: Optional[int]):
117
+ m, n, k = shape
118
+ wtype, zero_points = wtype_zeropoints
119
+
120
+ if group_size is not None and k % group_size != 0:
121
+ return
122
+
123
+ print(f"MNK = {m} {n} {k}")
124
+
125
+ # Normalize group_size
126
+ if group_size is None:
127
+ group_size = k
128
+ assert group_size <= k
129
+
130
+ a = rand_data((m, k), atype)
131
+ w = rand_data((k, n), atype)
132
+
133
+ w_ref, w_q_machete, w_s, w_zp = machete_quantize_and_pack(
134
+ w, wtype, group_size, zero_points)
135
+
136
+ output_ref = torch.matmul(a, w_ref)
137
+
138
+ for schedule in ops.machete_supported_schedules(wtype):
139
+ output = ops.machete_gemm(
140
+ a,
141
+ b_q=w_q_machete,
142
+ b_type=wtype,
143
+ b_scales=w_s,
144
+ b_zeros=maybe_convert_zeropoints(w_zp, w_s),
145
+ b_group_size=group_size,
146
+ schedule=schedule,
147
+ )
148
+
149
+ # Relax atol as our reduction dim becomes larger (more rounding error)
150
+ # Relax atol when we have zeropoints since the way machete applies
151
+ # zeropoints (after scales) causes noise around 0
152
+ atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
153
+ torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol),\
154
+ f"Schedule failed {schedule}"
155
+
156
+
157
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
158
+ reason="Machete is not supported on this GPU type.")
159
+@pytest.mark.parametrize("shape",
160
+ MNK_SHAPES,
161
+ ids=lambda x: "x".join(str(v) for v in x))
162
+@pytest.mark.parametrize("atype", ACT_TYPES, ids=lambda x: str(x))
163
+@pytest.mark.parametrize("wtype_zeropoints", WTYPE_ZEROPOINTS)
164
+@pytest.mark.parametrize("group_size", [128, None])
165
+def test_machete_heuristic(shape, atype: torch.dtype,
166
+ wtype_zeropoints: Tuple[ScalarType, bool],
167
+ group_size: Optional[int]):
168
+ m, n, k = shape
169
+ wtype, zero_points = wtype_zeropoints
170
+
171
+ if group_size is not None and k % group_size != 0:
172
+ return
173
+
174
+ # Normalize group_size
175
+ if group_size is None:
176
+ group_size = k
177
+ assert group_size <= k
178
+
179
+ a = rand_data((m, k), atype)
180
+ b = rand_data((k, n), atype)
181
+
182
+ machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
183
+
184
+
185
+# Test working on other devices
186
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
187
+ reason="Machete is not supported on this GPU type.")
188
+@pytest.mark.parametrize("device", CUDA_DEVICES)
189
+def test_machete_devices(device: str):
190
+ m, n, k = 512, 4096, 4096
191
+ wtype = scalar_types.uint4b8
192
+ group_size = 128
193
+ zero_points = False
194
+
195
+ print(f"MNK = {m} {n} {k}, device = {device}")
196
+
197
+ a = rand_data((m, k), torch.float16).to(device)
198
+ b = rand_data((k, n), torch.float16).to(device)
199
+
200
+ machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
201
+
202
+
203
+# Test working with a subset of A and B
204
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
205
+ reason="Machete is not supported on this GPU type.")
206
+def test_machete_subset():
207
+ big_m, big_n, big_k = 1024, 1024, 1024
208
+ m, n, k = 512, 512, 512
209
+ wtype = scalar_types.uint4b8
210
+ group_size = 128
211
+ zero_points = False
212
+
213
+ whole_a = rand_data((big_m, big_k), torch.float16)
214
+ whole_b = rand_data((big_k, big_n), torch.float16)
215
+
216
+ a = whole_a[0:m, 0:k]
217
+ b = whole_b[0:k, 0:n]
218
+
219
+ machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
220
+
221
+
222
+# Test to make sure cuda graphs work
223
+class MacheteLayer(torch.nn.Module):
224
+
225
+ def __init__(self, **kwargs):
226
+ super().__init__()
227
+ self.kwargs = kwargs
228
+
229
+ def forward(self, a):
230
+ return ops.machete_gemm(**self.kwargs)
231
+
232
+
233
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
234
+ reason="Machete is not supported on this GPU type.")
235
+def test_machete_cuda_graph():
236
+ m, n, k = 512, 4096, 4096
237
+
238
+ a = rand_data((m, k), torch.float16)
239
+ b = rand_data((k, n), torch.float16)
240
+ wtype = scalar_types.uint4b8
241
+ group_size = 128
242
+ zero_points = False
243
+
244
+ w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
245
+ b, wtype, group_size, zero_points)
246
+
247
+ # Construct a trivial model with a single layer that calls a machete kernel
248
+ model = MacheteLayer(
249
+ a=a,
250
+ b_q=w_q_packed,
251
+ b_type=wtype,
252
+ b_scales=w_s,
253
+ b_zeros=maybe_convert_zeropoints(w_zp, w_s),
254
+ b_group_size=group_size,
255
+ )
256
+
257
+ output_ref = torch.matmul(a, w_ref)
258
+
259
+ # Run the model with a cuda graph
260
+ stream = torch.cuda.Stream()
261
+ with torch.cuda.stream(stream):
262
+ g = torch.cuda.CUDAGraph()
263
+ with torch.cuda.graph(g):
264
+ output = model(a)
265
+ output.zero_()
266
+ g.replay()
267
+
268
+ # Relax atol as our reduction dim becomes larger (more rounding error)
269
+ # Relax atol when we have zeropoints since the way machete applies
270
+ # zeropoints (after scales) causes noise around 0
271
+ atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
272
+ torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
vllm/_custom_ops.py
+26
@@ -329,6 +329,32 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
329
num_bits, size_m, size_n, size_k)
330
331
332
+# machete
333
+def machete_supported_schedules(b_type: ScalarType) -> List[str]:
334
+ return torch.ops._C.machete_supported_schedules(b_type)
335
+
336
+
337
+def machete_gemm(
338
+ a: torch.Tensor,
339
+ b_q: torch.Tensor, # Should be the tensor returned by machete_prepack_B
340
+ b_type: ScalarType,
341
+ b_scales: Optional[torch.Tensor] = None,
342
+ b_zeros: Optional[torch.Tensor] = None,
343
+ b_group_size: Optional[int] = None,
344
+ c: Optional[torch.Tensor] = None,
345
+ alpha: Optional[float] = None,
346
+ beta: Optional[float] = None,
347
+ schedule: Optional[str] = None,
348
+) -> torch.Tensor:
349
+ return torch.ops._C.machete_gemm(a, b_q, b_type, b_scales, b_zeros,
350
+ b_group_size, c, alpha, beta, schedule)
351
+
352
+
353
+def machete_prepack_B(b_q_weight: torch.Tensor,
354
+ b_type: ScalarType) -> torch.Tensor:
355
+ return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
356
+
357
+
358
# fp8
359
def scaled_fp8_quant(
360
input: torch.Tensor,
vllm/model_executor/layers/quantization/utils/quant_utils.py
+9
-2
@@ -81,7 +81,8 @@ def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
81
def quantize_weights(w: torch.Tensor,
82
quant_type: ScalarType,
83
group_size: int,
84
- zero_points: bool = False):
84
+ zero_points: bool = False,
85
+ ref_zero_points_after_scales: bool = False):
86
assert quant_type.is_integer(), \
87
"Floating point quantization may work but has not been tested"
88
@@ -126,7 +127,13 @@ def quantize_weights(w: torch.Tensor,
127
w_q = torch.clamp(w_q, min_q_val, max_q_val)
128
129
# Compute ref (dequantized)
129
- w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
130
+ # For some kernels (namely Machete) the zero-points are applied after the
131
+ # scales are applied, for this case computing the reference in similar way
132
+ # allows us to use tighter error tolerances in our unit tests.
133
+ if ref_zero_points_after_scales and zero_points:
134
+ w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s
135
+ else:
136
+ w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
137
138
if quant_type.has_bias():
139
w_q += quant_type.bias