A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support (#10995)
Co-authored-by: Faraz Shahsavan <faraz.shahsavan@gmail.com> Co-authored-by: ilmarkov <markovilya197@gmail.com> Co-authored-by: Rahul Tuli <rahul@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Dipika Sikka committed
Dec 18, 2024 at 09:57 UTC
60508ffda91c22e4cde3b18f149d222211db8886
30 files changed
+2365
-117
CMakeLists.txt
+16
-10
@@ -206,7 +206,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
206
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
207
208
# Set CUTLASS_REVISION manually -- its revision detection doesn't work in this case.
209
- set(CUTLASS_REVISION "v3.5.1" CACHE STRING "CUTLASS revision to use")
209
+ set(CUTLASS_REVISION "v3.6.0" CACHE STRING "CUTLASS revision to use")
210
211
# Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
212
if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
@@ -223,13 +223,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
223
FetchContent_Declare(
224
cutlass
225
GIT_REPOSITORY https://github.com/nvidia/cutlass.git
226
- GIT_TAG v3.5.1
226
+ GIT_TAG 8aa95dbb888be6d81c6fbf7169718c5244b53227
227
GIT_PROGRESS TRUE
228
229
# Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
230
# Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
231
# So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
232
- GIT_SHALLOW TRUE
232
+ GIT_SHALLOW FALSE
233
)
234
endif()
235
FetchContent_MakeAvailable(cutlass)
@@ -241,7 +241,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
241
"csrc/quantization/awq/gemm_kernels.cu"
242
"csrc/custom_all_reduce.cu"
243
"csrc/permute_cols.cu"
244
- "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu")
244
+ "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
245
+ "csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
246
+ "csrc/sparse/cutlass/sparse_compressor_entry.cu"
247
+ "csrc/cutlass_extensions/common.cpp")
248
249
set_gencode_flags_for_srcs(
250
SRCS "${VLLM_EXT_SRC}"
@@ -271,11 +274,14 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
274
endif()
275
276
#
274
- # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
277
+ # The cutlass_scaled_mm cutlass_scaled_sparse_mm, and cutlass_compressor kernels
278
+ # For Hopper (c3x, i.e. CUTLASS 3.x) require
279
# CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
280
cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
281
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
278
- set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
282
+ set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
283
+ "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
284
+ "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
285
set_gencode_flags_for_srcs(
286
SRCS "${SRCS}"
287
CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
@@ -284,12 +290,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
290
message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
291
else()
292
if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
287
- message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
293
+ message(STATUS "Not building cutlass_c3x kernels as CUDA Compiler version is "
294
"not >= 12.0, we recommend upgrading to CUDA 12.0 or "
289
- "later if you intend on running FP8 quantized models on "
295
+ "later if you intend on running FP8 sparse or quantized models on "
296
"Hopper.")
297
else()
292
- message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
298
+ message(STATUS "Not building cutlass_c3x as no compatible archs found "
299
"in CUDA target architectures")
300
endif()
301
@@ -404,7 +410,7 @@ define_gpu_extension_target(
410
SOURCES ${VLLM_EXT_SRC}
411
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
412
ARCHITECTURES ${VLLM_GPU_ARCHES}
407
- INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
413
+ INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
414
USE_SABI 3
415
WITH_SOABI)
416
benchmarks/cutlass_benchmarks/sparse_benchmarks.py
new
+384
@@ -0,0 +1,384 @@
1
+import argparse
2
+import copy
3
+import itertools
4
+import pickle as pkl
5
+import time
6
+from typing import Callable, Iterable, List, Tuple
7
+
8
+import torch
9
+import torch.utils.benchmark as TBenchmark
10
+from torch.utils.benchmark import Measurement as TMeasurement
11
+from utils import make_rand_sparse_tensors
12
+from weight_shapes import WEIGHT_SHAPES
13
+
14
+from vllm import _custom_ops as ops
15
+from vllm.utils import FlexibleArgumentParser
16
+
17
+DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
18
+DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
19
+DEFAULT_TP_SIZES = [1]
20
+
21
+
22
+# bench
23
+def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
24
+ **kwargs) -> TMeasurement:
25
+ min_run_time = 1
26
+
27
+ globals = {
28
+ "args": args,
29
+ "kwargs": kwargs,
30
+ "fn": fn,
31
+ }
32
+ return TBenchmark.Timer(
33
+ stmt="fn(*args, **kwargs)",
34
+ globals=globals,
35
+ label=label,
36
+ sub_label=sub_label,
37
+ description=description,
38
+ ).blocked_autorange(min_run_time=min_run_time)
39
+
40
+
41
+def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
42
+ sub_label: str) -> Iterable[TMeasurement]:
43
+ assert dtype == torch.int8
44
+ b_compressed, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
45
+ scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
46
+ scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
47
+ bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
48
+
49
+ out = ops.cutlass_scaled_sparse_mm(a, b_compressed, e, scale_a, scale_b,
50
+ torch.bfloat16)
51
+ out_ref = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16)
52
+
53
+ if not torch.allclose(out, out_ref):
54
+ print("Incorrect results")
55
+ print(out)
56
+ print(out_ref)
57
+ else:
58
+ print("Correct results")
59
+
60
+ timers = []
61
+ # pytorch impl - bfloat16
62
+ timers.append(
63
+ bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
64
+ torch.mm, a.to(dtype=torch.bfloat16),
65
+ b.to(dtype=torch.bfloat16)))
66
+
67
+ # pytorch impl - float16
68
+ timers.append(
69
+ bench_fn(label, sub_label,
70
+ "pytorch_fp16_fp16_fp16_matmul-no-scales", torch.mm,
71
+ a.to(dtype=torch.float16), b.to(dtype=torch.float16)))
72
+
73
+ # cutlass impl
74
+ timers.append(
75
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm",
76
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
77
+ torch.bfloat16))
78
+
79
+ # cutlass with bias
80
+ timers.append(
81
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_bias",
82
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
83
+ bias))
84
+
85
+ # cutlass sparse impl
86
+ timers.append(
87
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_sparse_mm",
88
+ ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
89
+ scale_b, torch.bfloat16))
90
+
91
+ # cutlass sparse with bias
92
+ timers.append(
93
+ bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_sparse_mm_bias",
94
+ ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
95
+ scale_b, torch.bfloat16, bias))
96
+
97
+ return timers
98
+
99
+
100
+def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
101
+ sub_label: str) -> Iterable[TMeasurement]:
102
+ assert dtype == torch.float8_e4m3fn
103
+ b_compressed, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n,
104
+ k)
105
+ scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
106
+ scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
107
+ bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
108
+
109
+ out = ops.cutlass_scaled_sparse_mm(a, b_compressed, e, scale_a, scale_b,
110
+ torch.bfloat16)
111
+ out_ref = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16)
112
+
113
+ if not torch.allclose(out, out_ref):
114
+ print("Incorrect results")
115
+ print(out)
116
+ print(out_ref)
117
+ else:
118
+ print("Correct results")
119
+
120
+ timers = []
121
+
122
+ # pytorch impl w. bf16
123
+ timers.append(
124
+ bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
125
+ torch.mm, a.to(dtype=torch.bfloat16, device="cuda"),
126
+ b.to(dtype=torch.bfloat16, device="cuda")))
127
+
128
+ # pytorch impl: bf16 output, without fp8 fast accum
129
+ timers.append(
130
+ bench_fn(label,
131
+ sub_label,
132
+ "pytorch_fp8_fp8_bf16_scaled_mm",
133
+ torch._scaled_mm,
134
+ a,
135
+ b,
136
+ scale_a=scale_a,
137
+ scale_b=scale_b,
138
+ out_dtype=torch.bfloat16))
139
+
140
+ # pytorch impl: bf16 output, with fp8 fast accum
141
+ timers.append(
142
+ bench_fn(label,
143
+ sub_label,
144
+ "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
145
+ torch._scaled_mm,
146
+ a,
147
+ b,
148
+ scale_a=scale_a,
149
+ scale_b=scale_b,
150
+ out_dtype=torch.bfloat16,
151
+ use_fast_accum=True))
152
+
153
+ # pytorch impl: fp16 output, without fp8 fast accum
154
+ timers.append(
155
+ bench_fn(label,
156
+ sub_label,
157
+ "pytorch_fp8_fp8_fp16_scaled_mm",
158
+ torch._scaled_mm,
159
+ a,
160
+ b,
161
+ scale_a=scale_a,
162
+ scale_b=scale_b,
163
+ out_dtype=torch.float16))
164
+
165
+ # pytorch impl: fp16 output, with fp8 fast accum
166
+ timers.append(
167
+ bench_fn(label,
168
+ sub_label,
169
+ "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
170
+ torch._scaled_mm,
171
+ a,
172
+ b,
173
+ scale_a=scale_a,
174
+ scale_b=scale_b,
175
+ out_dtype=torch.float16,
176
+ use_fast_accum=True))
177
+
178
+ # cutlass impl: bf16 output
179
+ timers.append(
180
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm",
181
+ ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
182
+ torch.bfloat16))
183
+
184
+ # cutlass impl: bf16 output
185
+ timers.append(
186
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_sparse_mm",
187
+ ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
188
+ scale_b, torch.bfloat16))
189
+
190
+ # cutlass impl: fp16 output
191
+ timers.append(
192
+ bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_sparse_mm",
193
+ ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
194
+ scale_b, torch.float16))
195
+
196
+ # cutlass impl: bf16 output, with bias
197
+ timers.append(
198
+ bench_fn(label, sub_label,
199
+ "cutlass_fp8_fp8_bf16_scaled_sparse_mm_bias",
200
+ ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
201
+ scale_b, torch.bfloat16, bias))
202
+
203
+ # cutlass impl: fp16 output, with bias
204
+ timers.append(
205
+ bench_fn(label, sub_label,
206
+ "cutlass_fp8_fp8_fp16_scaled_sparse_mm_bias",
207
+ ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
208
+ scale_b, torch.float16, bias.to(dtype=torch.float16)))
209
+
210
+ return timers
211
+
212
+
213
+def bench(dtype: torch.dtype, m: int, k: int, n: int, label: str,
214
+ sub_label: str) -> Iterable[TMeasurement]:
215
+ if dtype == torch.int8:
216
+ return bench_int8(dtype, m, k, n, label, sub_label)
217
+ if dtype == torch.float8_e4m3fn:
218
+ return bench_fp8(dtype, m, k, n, label, sub_label)
219
+ raise ValueError("unsupported type")
220
+
221
+
222
+# runner
223
+def print_timers(timers: Iterable[TMeasurement]):
224
+ compare = TBenchmark.Compare(timers)
225
+ compare.print()
226
+
227
+
228
+def run(dtype: torch.dtype,
229
+ MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
230
+ results = []
231
+ for m, k, n in MKNs:
232
+ timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
233
+ f"MKN=({m}x{k}x{n})")
234
+ print_timers(timers)
235
+ results.extend(timers)
236
+
237
+ return results
238
+
239
+
240
+# output makers
241
+def make_output(data: Iterable[TMeasurement],
242
+ MKNs: Iterable[Tuple[int, int, int]],
243
+ base_description: str,
244
+ timestamp=None):
245
+ print(f"== All Results {base_description} ====")
246
+ print_timers(data)
247
+
248
+ # pickle all the results
249
+ timestamp = int(time.time()) if timestamp is None else timestamp
250
+ with open(f"{base_description}-{timestamp}.pkl", "wb") as f:
251
+ pkl.dump(data, f)
252
+
253
+
254
+# argparse runners
255
+
256
+
257
+def run_square_bench(args):
258
+ dim_sizes = list(
259
+ range(args.dim_start, args.dim_end + 1, args.dim_increment))
260
+ MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
261
+ data = run(args.dtype, MKNs)
262
+
263
+ make_output(data, MKNs, f"square_bench-{args.dtype}")
264
+
265
+
266
+def run_range_bench(args):
267
+ dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
268
+ n = len(dim_sizes)
269
+ Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
270
+ Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
271
+ Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
272
+ MKNs = list(zip(Ms, Ks, Ns))
273
+ data = run(args.dtype, MKNs)
274
+
275
+ make_output(data, MKNs, f"range_bench-{args.dtype}")
276
+
277
+
278
+def run_model_bench(args):
279
+ print("Benchmarking models:")
280
+ for i, model in enumerate(args.models):
281
+ print(f"[{i}] {model}")
282
+
283
+ def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:
284
+ KNs = []
285
+ for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):
286
+ KN[tp_split_dim] = KN[tp_split_dim] // tp_size
287
+ KNs.append(KN)
288
+ return KNs
289
+
290
+ model_bench_data = []
291
+ models_tps = list(itertools.product(args.models, args.tp_sizes))
292
+ for model, tp_size in models_tps:
293
+ Ms = args.batch_sizes
294
+ KNs = model_shapes(model, tp_size)
295
+ MKNs = []
296
+ for m in Ms:
297
+ for k, n in KNs:
298
+ MKNs.append((m, k, n))
299
+
300
+ data = run(args.dtype, MKNs)
301
+ model_bench_data.append(data)
302
+
303
+ # Print all results
304
+ for data, model_tp in zip(model_bench_data, models_tps):
305
+ model, tp_size = model_tp
306
+ print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
307
+ print_timers(data)
308
+
309
+ timestamp = int(time.time())
310
+
311
+ all_data = []
312
+ for d in model_bench_data:
313
+ all_data.extend(d)
314
+ # pickle all data
315
+ with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
316
+ pkl.dump(all_data, f)
317
+
318
+
319
+if __name__ == '__main__':
320
+
321
+ def to_torch_dtype(dt):
322
+ if dt == "int8":
323
+ return torch.int8
324
+ if dt == "fp8":
325
+ return torch.float8_e4m3fn
326
+ raise ValueError("unsupported dtype")
327
+
328
+ parser = FlexibleArgumentParser(
329
+ description="""
330
+Benchmark Cutlass GEMM.
331
+
332
+ To run square GEMMs:
333
+ python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 square_bench --dim-start 128 --dim-end 512 --dim-increment 64
334
+
335
+ To run constant N and K and sweep M:
336
+ python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384
337
+
338
+ To run dimensions from a model:
339
+ python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1
340
+
341
+ Output:
342
+ - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs.
343
+ """, # noqa: E501
344
+ formatter_class=argparse.RawTextHelpFormatter)
345
+
346
+ parser.add_argument("--dtype",
347
+ type=to_torch_dtype,
348
+ required=True,
349
+ help="Available options are ['int8', 'fp8']")
350
+ subparsers = parser.add_subparsers(dest="cmd")
351
+
352
+ square_parser = subparsers.add_parser("square_bench")
353
+ square_parser.add_argument("--dim-start", type=int, required=True)
354
+ square_parser.add_argument("--dim-end", type=int, required=True)
355
+ square_parser.add_argument("--dim-increment", type=int, required=True)
356
+ square_parser.set_defaults(func=run_square_bench)
357
+
358
+ range_parser = subparsers.add_parser("range_bench")
359
+ range_parser.add_argument("--dim-start", type=int, required=True)
360
+ range_parser.add_argument("--dim-end", type=int, required=True)
361
+ range_parser.add_argument("--dim-increment", type=int, required=True)
362
+ range_parser.add_argument("--m-constant", type=int, default=None)
363
+ range_parser.add_argument("--n-constant", type=int, default=None)
364
+ range_parser.add_argument("--k-constant", type=int, default=None)
365
+ range_parser.set_defaults(func=run_range_bench)
366
+
367
+ model_parser = subparsers.add_parser("model_bench")
368
+ model_parser.add_argument("--models",
369
+ nargs="+",
370
+ type=str,
371
+ default=DEFAULT_MODELS,
372
+ choices=WEIGHT_SHAPES.keys())
373
+ model_parser.add_argument("--tp-sizes",
374
+ nargs="+",
375
+ type=int,
376
+ default=DEFAULT_TP_SIZES)
377
+ model_parser.add_argument("--batch-sizes",
378
+ nargs="+",
379
+ type=int,
380
+ default=DEFAULT_BATCH_SIZES)
381
+ model_parser.set_defaults(func=run_model_bench)
382
+
383
+ args = parser.parse_args()
384
+ args.func(args)
benchmarks/cutlass_benchmarks/utils.py
new
+96
@@ -0,0 +1,96 @@
1
+# Cutlass bench utils
2
+from typing import Iterable, Tuple
3
+
4
+import torch
5
+
6
+import vllm._custom_ops as ops
7
+
8
+
9
+def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
10
+ finfo = torch.finfo(torch.float8_e4m3fn)
11
+ return torch.round(tensor.clamp(
12
+ min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
13
+
14
+
15
+def to_int8(tensor: torch.Tensor) -> torch.Tensor:
16
+ return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
17
+
18
+
19
+def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
20
+ return tensor.to(dtype=torch.bfloat16)
21
+
22
+
23
+def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
24
+ return tensor.to(dtype=torch.float16)
25
+
26
+
27
+def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
28
+ k: int) -> Tuple[torch.Tensor, torch.Tensor]:
29
+ a = torch.randn((m, k), device='cuda') * 5
30
+ b = torch.randn((n, k), device='cuda').t() * 5
31
+
32
+ if dtype == torch.int8:
33
+ return to_int8(a), to_int8(b)
34
+ if dtype == torch.float8_e4m3fn:
35
+ return to_fp8(a), to_fp8(b)
36
+
37
+ raise ValueError("unsupported dtype")
38
+
39
+
40
+def prune_to_2_4(tensor):
41
+ # Reshape tensor to [N, 4] where N is number of groups of 4
42
+ original_shape = tensor.shape
43
+ reshaped = tensor.reshape(-1, 4)
44
+
45
+ # Get indices of top 2 absolute values in each group of 4
46
+ _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
47
+
48
+ # Create binary mask
49
+ mask = torch.zeros_like(reshaped)
50
+ mask.scatter_(dim=1,
51
+ index=indices,
52
+ src=torch.ones_like(indices, dtype=mask.dtype))
53
+
54
+ # Apply mask and reshape back
55
+ pruned = reshaped * mask
56
+
57
+ # Turn all -0.0 to 0.0
58
+ pruned[pruned == -0.0] = 0.0
59
+
60
+ return pruned.reshape(original_shape)
61
+
62
+
63
+def make_rand_sparse_tensors(dtype: torch.dtype, m: int, n: int,
64
+ k: int) -> Tuple[torch.Tensor, torch.Tensor]:
65
+ a = torch.randn((m, k), device='cuda') * 5
66
+ b = torch.randn((n, k), device='cuda').t() * 5
67
+
68
+ b = prune_to_2_4(b.t()).t()
69
+
70
+ if dtype == torch.int8:
71
+ a, b = to_int8(a), to_int8(b)
72
+ elif dtype == torch.float8_e4m3fn:
73
+ a, b = to_fp8(a), to_fp8(b)
74
+ elif dtype == torch.float16:
75
+ a, b = to_fp16(a), to_fp16(b)
76
+ elif dtype == torch.bfloat16:
77
+ a, b = to_bf16(a), to_bf16(b)
78
+ else:
79
+ raise ValueError("unsupported dtype")
80
+
81
+ b_compressed, e = ops.cutlass_sparse_compress(b.t())
82
+
83
+ # Compressed B, Metadata, Original A, B
84
+ return b_compressed, e, a, b
85
+
86
+
87
+def make_n_rand_sparse_tensors(num_tensors: int, dtype: torch.dtype,
88
+ m: int, n: int, k: int) -> \
89
+ Tuple[Iterable[torch.Tensor], Iterable[torch.Tensor]]:
90
+ ABs = []
91
+ for _ in range(num_tensors):
92
+ b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)
93
+ if b_comp is not None:
94
+ ABs.append(make_rand_sparse_tensors(dtype, m, n, k))
95
+ BComps, Es, As, Bs = zip(*ABs)
96
+ return list(BComps), list(Es), list(As), list(Bs)
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2
-26
@@ -8,6 +8,7 @@ from typing import Callable, Iterable, List, Tuple
8
import torch
9
import torch.utils.benchmark as TBenchmark
10
from torch.utils.benchmark import Measurement as TMeasurement
11
+from utils import make_rand_tensors
12
from weight_shapes import WEIGHT_SHAPES
13
14
from vllm import _custom_ops as ops
@@ -17,31 +18,6 @@ DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
18
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
19
DEFAULT_TP_SIZES = [1]
20
20
-# helpers
21
-
22
-
23
-def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
24
- finfo = torch.finfo(torch.float8_e4m3fn)
25
- return torch.round(tensor.clamp(
26
- min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
27
-
28
-
29
-def to_int8(tensor: torch.Tensor) -> torch.Tensor:
30
- return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
31
-
32
-
33
-def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
34
- k: int) -> Tuple[torch.Tensor, torch.Tensor]:
35
- a = torch.randn((m, k), device='cuda') * 5
36
- b = torch.randn((n, k), device='cuda').t() * 5
37
-
38
- if dtype == torch.int8:
39
- return to_int8(a), to_int8(b)
40
- if dtype == torch.float8_e4m3fn:
41
- return to_fp8(a), to_fp8(b)
42
-
43
- raise ValueError("unsupported dtype")
44
-
21
22
# bench
23
def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
@@ -386,4 +362,4 @@ Benchmark Cutlass GEMM.
362
model_parser.set_defaults(func=run_model_bench)
363
364
args = parser.parse_args()
389
- args.func(args)
365
+ args.func(args)
\ No newline at end of file
benchmarks/cutlass_benchmarks/weight_shapes.py
+1
-1
@@ -40,4 +40,4 @@ WEIGHT_SHAPES = {
40
([8192, 57344], 1),
41
([28672, 8192], 0),
42
],
43
-}
43
+}
\ No newline at end of file
csrc/core/math.hpp
new
+7
@@ -0,0 +1,7 @@
1
+#include <climits>
2
+#include <iostream>
3
+
4
+inline uint32_t next_pow_2(uint32_t const num) {
5
+ if (num <= 1) return num;
6
+ return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
7
+}
\ No newline at end of file
csrc/cutlass_extensions/common.cpp
new
+11
@@ -0,0 +1,11 @@
1
+#include "cutlass_extensions/common.hpp"
2
+
3
+int32_t get_sm_version_num() {
4
+ int32_t major_capability, minor_capability;
5
+ cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
6
+ 0);
7
+ cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
8
+ 0);
9
+ int32_t version_num = major_capability * 10 + minor_capability;
10
+ return version_num;
11
+}
\ No newline at end of file
csrc/cutlass_extensions/common.hpp
new
+35
@@ -0,0 +1,35 @@
1
+#pragma once
2
+
3
+#include "cutlass/cutlass.h"
4
+#include <climits>
5
+#include "cuda_runtime.h"
6
+#include <iostream>
7
+
8
+/**
9
+ * Helper function for checking CUTLASS errors
10
+ */
11
+#define CUTLASS_CHECK(status) \
12
+ { \
13
+ cutlass::Status error = status; \
14
+ TORCH_CHECK(error == cutlass::Status::kSuccess, \
15
+ cutlassGetStatusString(error)); \
16
+ }
17
+
18
+/**
19
+ * Panic wrapper for unwinding CUDA runtime errors
20
+ */
21
+#define CUDA_CHECK(status) \
22
+ { \
23
+ cudaError_t error = status; \
24
+ TORCH_CHECK(error == cudaSuccess, cudaGetErrorString(error)); \
25
+ }
26
+
27
+inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
28
+ int max_shared_mem_per_block_opt_in = 0;
29
+ cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
30
+ cudaDevAttrMaxSharedMemoryPerBlockOptin,
31
+ device);
32
+ return max_shared_mem_per_block_opt_in;
33
+}
34
+
35
+int32_t get_sm_version_num();
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+2
-2
@@ -36,13 +36,13 @@ struct ScaledEpilogueBase {
36
// Don't want to support nullptr by default
37
template <typename T, bool EnableNullPtr = false>
38
using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
39
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
39
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T, T,
40
Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
41
42
// Don't want to support nullptr by default
43
template <typename T, bool EnableNullPtr = false>
44
using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
45
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
45
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T, T,
46
Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
47
48
// This utility function constructs the arguments for the load descriptors
csrc/ops.h
+9
@@ -162,6 +162,15 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
162
torch::Tensor const& azp_adj,
163
c10::optional<torch::Tensor> const& azp,
164
c10::optional<torch::Tensor> const& bias);
165
+
166
+void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
167
+ torch::Tensor const& b, torch::Tensor const& e,
168
+ torch::Tensor const& a_scales,
169
+ torch::Tensor const& b_scales,
170
+ c10::optional<torch::Tensor> const& bias);
171
+
172
+bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
173
+ torch::Tensor& e, torch::Tensor const& a);
174
#endif
175
176
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/cutlass_w8a8/common.hpp
deleted
-27
@@ -1,27 +0,0 @@
1
-#pragma once
2
-
3
-#include "cutlass/cutlass.h"
4
-#include <climits>
5
-
6
-/**
7
- * Helper function for checking CUTLASS errors
8
- */
9
-#define CUTLASS_CHECK(status) \
10
- { \
11
- TORCH_CHECK(status == cutlass::Status::kSuccess, \
12
- cutlassGetStatusString(status)) \
13
- }
14
-
15
-inline uint32_t next_pow_2(uint32_t const num) {
16
- if (num <= 1) return num;
17
- return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
18
-}
19
-
20
-inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
21
- int max_shared_mem_per_block_opt_in = 0;
22
- cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
23
- cudaDevAttrMaxSharedMemoryPerBlockOptin,
24
- device);
25
- return max_shared_mem_per_block_opt_in;
26
-}
27
-
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+2
-1
@@ -21,7 +21,8 @@
21
#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
22
#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
23
24
-#include "common.hpp"
24
+#include "core/math.hpp"
25
+#include "cutlass_extensions/common.hpp"
26
// clang-format on
27
28
using namespace cute;
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+2
-1
@@ -24,7 +24,8 @@
24
#include "cutlass/gemm/collective/collective_builder.hpp"
25
26
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
27
-#include "common.hpp"
27
+#include "core/math.hpp"
28
+#include "cutlass_extensions/common.hpp"
29
// clang-format on
30
31
using namespace cute;
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+2
-10
@@ -3,6 +3,8 @@
3
#include <c10/cuda/CUDAGuard.h>
4
#include <torch/all.h>
5
6
+#include "cutlass_extensions/common.hpp"
7
+
8
void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
9
torch::Tensor const& b,
10
torch::Tensor const& a_scales,
@@ -79,16 +81,6 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
81
return false;
82
}
83
82
-int32_t get_sm_version_num() {
83
- int32_t major_capability, minor_capability;
84
- cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
85
- 0);
86
- cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
87
- 0);
88
- int32_t version_num = major_capability * 10 + minor_capability;
89
- return version_num;
90
-}
91
-
84
void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
85
torch::Tensor const& b, torch::Tensor const& a_scales,
86
torch::Tensor const& b_scales,
csrc/sparse/cutlass/sparse_compressor_c3x.cu
new
+163
@@ -0,0 +1,163 @@
1
+// clang-format will break include orders
2
+// clang-format off
3
+#include <cudaTypedefs.h>
4
+
5
+#include "sparse_scaled_mm_c3x.cuh"
6
+
7
+#include "cutlass/numeric_conversion.h"
8
+#include "cutlass/transform/device/transform_universal_adapter.hpp"
9
+#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
10
+#include "cutlass/epilogue/collective/default_epilogue.hpp"
11
+
12
+#include "cutlass/util/host_tensor.h"
13
+#include "cutlass/util/packed_stride.hpp"
14
+// clang-format on
15
+
16
+using namespace cute;
17
+using namespace vllm;
18
+
19
+/// Make A structured sparse by replacing elements with 0 and compress it
20
+template <typename ElementA_, typename ElementAcc_>
21
+bool cutlass_sparse_compress(torch::Tensor& a_nzs, torch::Tensor& a_meta,
22
+ torch::Tensor const& a) {
23
+ // Checks for conformality
24
+ TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
25
+ a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
26
+ TORCH_CHECK(a.dim() == 2)
27
+ // Check for strides and alignment
28
+ TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
29
+ TORCH_CHECK(a.stride(1) == 1)
30
+
31
+ int m = a.size(0);
32
+ int k = a.size(1);
33
+
34
+ // Sparse kernel setup; this kernel is not used for matmul,
35
+ // but just for setting up the compressor utility
36
+ // A matrix configuration
37
+ using ElementA = ElementA_;
38
+ using LayoutTagA = cutlass::layout::RowMajor;
39
+ constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
40
+ // B matrix configuration
41
+ using ElementB = ElementA;
42
+ using LayoutTagB = cutlass::layout::ColumnMajor;
43
+ constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
44
+ // C/D matrix configuration
45
+ using ElementC = float;
46
+ using LayoutTagC = cutlass::layout::ColumnMajor;
47
+ constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
48
+ // Core kernel configurations
49
+ using ElementAccumulator = ElementAcc_;
50
+ using TileShape = Shape<_128, _128, _128>;
51
+ using TileShapeRef = Shape<_128, _128, _64>;
52
+ using ClusterShape = Shape<_1, _2, _1>;
53
+ using KernelSchedule = typename std::conditional<
54
+ std::is_same_v<ElementA, cutlass::float_e4m3_t>,
55
+ cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum,
56
+ cutlass::gemm::KernelTmaWarpSpecialized>::type;
57
+
58
+ using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
59
+ using ProblemShape = Shape<int, int, int, int>;
60
+
61
+ using CollectiveEpilogue =
62
+ typename cutlass::epilogue::collective::CollectiveBuilder<
63
+ cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
64
+ ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
65
+ ElementAccumulator, ElementAccumulator, ElementC, LayoutTagC,
66
+ AlignmentC, ElementC, LayoutTagC, AlignmentC,
67
+ EpilogueSchedule>::CollectiveOp;
68
+
69
+ using CollectiveMainloop =
70
+ typename cutlass::gemm::collective::CollectiveBuilder<
71
+ cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp, ElementA,
72
+ LayoutTagA, AlignmentA, ElementB, LayoutTagB, AlignmentB,
73
+ ElementAccumulator, TileShape, ClusterShape,
74
+ cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
75
+ sizeof(typename CollectiveEpilogue::SharedStorage))>,
76
+ KernelSchedule>::CollectiveOp;
77
+
78
+ using GemmKernel =
79
+ cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop,
80
+ CollectiveEpilogue>;
81
+
82
+ using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
83
+
84
+ using StrideA = cutlass::gemm::TagToStrideA_t<LayoutTagA>;
85
+ using StrideE = StrideA;
86
+
87
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
88
+
89
+ // The n (=1) dimension does not matter for the compressor
90
+ typename GemmKernel::ProblemShape prob_shape{m, 1, k, 1};
91
+
92
+ using LayoutA = typename GemmKernel::CollectiveMainloop::LayoutA;
93
+ using LayoutE = typename GemmKernel::CollectiveMainloop::LayoutE;
94
+
95
+ using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
96
+ using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
97
+
98
+ // Offline compressor kernel
99
+ using CompressorUtility =
100
+ cutlass::transform::kernel::StructuredSparseCompressorUtility<
101
+ ProblemShape, ElementA, LayoutTagA, SparseConfig>;
102
+
103
+ using CompressorKernel =
104
+ cutlass::transform::kernel::StructuredSparseCompressor<
105
+ ProblemShape, ElementA, LayoutTagA, SparseConfig,
106
+ cutlass::arch::Sm90>;
107
+
108
+ using Compressor =
109
+ cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
110
+
111
+ auto [M, N, K, L] = prob_shape;
112
+
113
+ StrideA stride_A;
114
+ stride_A =
115
+ cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
116
+
117
+ CompressorUtility compressor_utility(prob_shape, stride_A);
118
+
119
+ int ME = compressor_utility.get_metadata_m_physical();
120
+ int KE = compressor_utility.get_metadata_k_physical();
121
+ int KC = compressor_utility.get_tensorA_k_physical();
122
+
123
+ auto a_ptr = static_cast<ElementA*>(a.data_ptr());
124
+
125
+ auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
126
+ auto a_meta_ptr = static_cast<typename Gemm::CollectiveMainloop::ElementE*>(
127
+ a_meta.data_ptr());
128
+
129
+ cutlass::KernelHardwareInfo hw_info;
130
+ hw_info.device_id = 0;
131
+ hw_info.sm_count =
132
+ cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
133
+ hw_info.device_id);
134
+ typename Compressor::Arguments arguments{
135
+ prob_shape, {a_ptr, stride_A, a_nzs_ptr, a_meta_ptr}, {hw_info}};
136
+
137
+ Compressor compressor_op;
138
+ size_t workspace_size = Compressor::get_workspace_size(arguments);
139
+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
140
+
141
+ CUTLASS_CHECK(compressor_op.can_implement(arguments));
142
+ CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.get()));
143
+ CUTLASS_CHECK(compressor_op.run());
144
+ CUDA_CHECK(cudaDeviceSynchronize());
145
+
146
+ return true;
147
+}
148
+
149
+bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
150
+ torch::Tensor const& a) {
151
+ if (a.dtype() == torch::kBFloat16) {
152
+ return cutlass_sparse_compress<cutlass::bfloat16_t, float>(a_nzs, a_meta,
153
+ a);
154
+ } else if (a.dtype() == torch::kFloat16) {
155
+ return cutlass_sparse_compress<cutlass::half_t, float>(a_nzs, a_meta, a);
156
+ } else if (a.dtype() == torch::kFloat8_e4m3fn) {
157
+ return cutlass_sparse_compress<cutlass::float_e4m3_t, float>(a_nzs, a_meta,
158
+ a);
159
+ } else if (a.dtype() == torch::kInt8) {
160
+ return cutlass_sparse_compress<int8_t, int32_t>(a_nzs, a_meta, a);
161
+ }
162
+ return false;
163
+}
\ No newline at end of file
csrc/sparse/cutlass/sparse_compressor_entry.cu
new
+42
@@ -0,0 +1,42 @@
1
+#include <cudaTypedefs.h>
2
+
3
+#include <c10/cuda/CUDAGuard.h>
4
+#include <torch/all.h>
5
+
6
+#include "cutlass_extensions/common.hpp"
7
+
8
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
9
+bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
10
+ torch::Tensor const& a);
11
+#endif
12
+
13
+bool cutlass_sparse_compress_entry(torch::Tensor& a_nzs, torch::Tensor& a_meta,
14
+ torch::Tensor const& a) {
15
+ // Checks for conformality
16
+ TORCH_CHECK(a.dim() == 2 && a_meta.dim() == 2 && a_nzs.dim() == 2);
17
+ TORCH_CHECK(a.size(0) == a_nzs.size(0) && a.size(0) == a_meta.size(0) &&
18
+ a_nzs.size(1) * 2 == a.size(1) &&
19
+ a_meta.size(1) * 2 * 4 == a.size(1));
20
+ // Considering elemsPerMetaElem = 8b / 2b_per_nz = 4
21
+
22
+ // Check for strides and alignment
23
+ TORCH_CHECK(a.stride(1) == 1 && a_nzs.stride(1) == 1 &&
24
+ a_meta.stride(1) == 1); // Row-major
25
+ TORCH_CHECK(a.stride(0) % 8 == 0); // 8 Byte Alignment for Compression
26
+
27
+ at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
28
+ int32_t version_num = get_sm_version_num();
29
+
30
+ // Guard against compilation issues for sm90 kernels
31
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
32
+ if (version_num >= 90) {
33
+ return cutlass_sparse_compress_sm90(a_nzs, a_meta, a);
34
+ }
35
+#endif
36
+
37
+ TORCH_CHECK_NOT_IMPLEMENTED(
38
+ false,
39
+ "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
40
+ "CUDA device capability: ",
41
+ version_num);
42
+}
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
new
+303
@@ -0,0 +1,303 @@
1
+// clang-format will break include orders
2
+// clang-format off
3
+#include <cudaTypedefs.h>
4
+
5
+#if defined CUDA_VERSION && CUDA_VERSION >= 12000
6
+#include "sparse_scaled_mm_c3x.cuh"
7
+// clang-format on
8
+
9
+using namespace cute;
10
+using namespace vllm;
11
+
12
+template <typename InType, typename OutType,
13
+ template <typename, typename, typename> typename Epilogue,
14
+ typename... EpilogueArgs>
15
+void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
16
+ torch::Tensor const& bt_nzs,
17
+ torch::Tensor const& bt_meta,
18
+ EpilogueArgs&&... args) {
19
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
20
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
21
+ TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
22
+ TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
23
+
24
+ using Cutlass3xGemmDefault =
25
+ typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
26
+ using Cutlass3xGemmM64 =
27
+ typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
28
+ using Cutlass3xGemmM128 =
29
+ typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
30
+ using Cutlass3xGemmM256 =
31
+ typename sm90_fp8_config_M256<InType, OutType, Epilogue>::Cutlass3xGemm;
32
+ using Cutlass3xGemmM512 =
33
+ typename sm90_fp8_config_M512<InType, OutType, Epilogue>::Cutlass3xGemm;
34
+
35
+ using Cutlass3xGemm1 =
36
+ typename sm90_fp8_config_1<InType, OutType, Epilogue>::Cutlass3xGemm;
37
+ using Cutlass3xGemm2 =
38
+ typename sm90_fp8_config_2<InType, OutType, Epilogue>::Cutlass3xGemm;
39
+ using Cutlass3xGemm3 =
40
+ typename sm90_fp8_config_3<InType, OutType, Epilogue>::Cutlass3xGemm;
41
+ using Cutlass3xGemm4 =
42
+ typename sm90_fp8_config_4<InType, OutType, Epilogue>::Cutlass3xGemm;
43
+ using Cutlass3xGemm5 =
44
+ typename sm90_fp8_config_5<InType, OutType, Epilogue>::Cutlass3xGemm;
45
+ using Cutlass3xGemm6 =
46
+ typename sm90_fp8_config_6<InType, OutType, Epilogue>::Cutlass3xGemm;
47
+ using Cutlass3xGemm7 =
48
+ typename sm90_fp8_config_7<InType, OutType, Epilogue>::Cutlass3xGemm;
49
+ using Cutlass3xGemm8 =
50
+ typename sm90_fp8_config_8<InType, OutType, Epilogue>::Cutlass3xGemm;
51
+
52
+ uint32_t const n = bt_nzs.size(0);
53
+ uint32_t const m = a.size(0); // Batch size
54
+ uint32_t const mp2 =
55
+ std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
56
+
57
+ if (mp2 <= 64) {
58
+ if (n == 28672) {
59
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm2>(
60
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
61
+ } else if (n == 4096 || n == 6144) {
62
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm1>(
63
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
64
+ }
65
+ } else if (mp2 <= 128) {
66
+ if (n == 4096) {
67
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm3>(
68
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
69
+ } else if (n == 28672) {
70
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm5>(
71
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
72
+ } else if (n == 6144) {
73
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm4>(
74
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
75
+ }
76
+ } else if (mp2 <= 256) {
77
+ if (n == 4096) {
78
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm6>(
79
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
80
+ } else if (n == 28672) {
81
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
82
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
83
+ } else if (n == 6144) {
84
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
85
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
86
+ }
87
+ } else {
88
+ if (n == 6144 || n == 28672) {
89
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
90
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
91
+ } else if (n == 4096) {
92
+ return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
93
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
94
+ }
95
+ }
96
+
97
+ // Otherwise the default heuristic
98
+ if (mp2 <= 64) {
99
+ // n in [1, 64]
100
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
101
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
102
+ } else if (mp2 <= 128) {
103
+ // n in (64, 128]
104
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
105
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
106
+ } else if (mp2 <= 256) {
107
+ // n in (128, 256]
108
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM256>(
109
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
110
+ } else {
111
+ // n in (256, inf)
112
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM512>(
113
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
114
+ }
115
+}
116
+
117
+template <typename InType, typename OutType,
118
+ template <typename, typename, typename> typename Epilogue,
119
+ typename... EpilogueArgs>
120
+void cutlass_gemm_sm90_fp16_dispatch(torch::Tensor& out, torch::Tensor const& a,
121
+ torch::Tensor const& bt_nzs,
122
+ torch::Tensor const& bt_meta,
123
+ EpilogueArgs&&... args) {
124
+ static_assert(std::is_same<InType, cutlass::half_t>());
125
+ TORCH_CHECK(a.dtype() == torch::kFloat16);
126
+ TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
127
+ TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
128
+
129
+ using Cutlass3xGemmDefault =
130
+ typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
131
+
132
+ // m in (128, inf)
133
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
134
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
135
+}
136
+
137
+template <typename InType, typename OutType,
138
+ template <typename, typename, typename> typename Epilogue,
139
+ typename... EpilogueArgs>
140
+void cutlass_gemm_sm90_bf16_dispatch(torch::Tensor& out, torch::Tensor const& a,
141
+ torch::Tensor const& bt_nzs,
142
+ torch::Tensor const& bt_meta,
143
+ EpilogueArgs&&... args) {
144
+ static_assert(std::is_same<InType, cutlass::bfloat16_t>());
145
+ TORCH_CHECK(a.dtype() == torch::kBFloat16);
146
+ TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
147
+ TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
148
+
149
+ using Cutlass3xGemmDefault =
150
+ typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
151
+
152
+ // m in (128, inf)
153
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
154
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
155
+}
156
+
157
+template <typename InType, typename OutType,
158
+ template <typename, typename, typename> typename Epilogue,
159
+ typename... EpilogueArgs>
160
+void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
161
+ torch::Tensor const& bt_nzs,
162
+ torch::Tensor const& bt_meta,
163
+ EpilogueArgs&&... args) {
164
+ static_assert(std::is_same<InType, int8_t>());
165
+ TORCH_CHECK(a.dtype() == torch::kInt8);
166
+ TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
167
+ TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
168
+
169
+ using Cutlass3xGemmDefault =
170
+ typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
171
+ using Cutlass3xGemmM128 =
172
+ typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
173
+ using Cutlass3xGemmM64 =
174
+ typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
175
+ using Cutlass3xGemmM32NBig =
176
+ typename sm90_int8_config_M32_NBig<InType, OutType,
177
+ Epilogue>::Cutlass3xGemm;
178
+ using Cutlass3xGemmM32NSmall =
179
+ typename sm90_int8_config_M32_NSmall<InType, OutType,
180
+ Epilogue>::Cutlass3xGemm;
181
+
182
+ uint32_t const n = out.size(1);
183
+ bool const is_small_n = n < 8192;
184
+
185
+ uint32_t const m = a.size(0);
186
+ uint32_t const mp2 =
187
+ std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
188
+
189
+ if (mp2 <= 32) {
190
+ // m in [1, 32]
191
+ if (is_small_n) {
192
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NSmall>(
193
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
194
+ } else {
195
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NBig>(
196
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
197
+ }
198
+ } else if (mp2 <= 64) {
199
+ // m in (32, 64]
200
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
201
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
202
+ } else if (mp2 <= 128) {
203
+ // m in (64, 128]
204
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
205
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
206
+ } else {
207
+ // m in (128, inf)
208
+ return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
209
+ out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
210
+ }
211
+}
212
+
213
+template <template <typename, typename, typename> typename Epilogue,
214
+ typename... EpilogueArgs>
215
+void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
216
+ torch::Tensor const& a,
217
+ torch::Tensor const& bt_nzs,
218
+ torch::Tensor const& bt_meta,
219
+ EpilogueArgs&&... epilogue_args) {
220
+ TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
221
+ if (a.dtype() == torch::kInt8) {
222
+ TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
223
+
224
+ if (out.dtype() == torch::kBFloat16) {
225
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
226
+ Epilogue>(
227
+ out, a, bt_nzs, bt_meta,
228
+ std::forward<EpilogueArgs>(epilogue_args)...);
229
+ } else {
230
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
231
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
232
+ out, a, bt_nzs, bt_meta,
233
+ std::forward<EpilogueArgs>(epilogue_args)...);
234
+ }
235
+ } else if (a.dtype() == torch::kFloat8_e4m3fn) {
236
+ TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
237
+
238
+ if (out.dtype() == torch::kBFloat16) {
239
+ return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
240
+ cutlass::bfloat16_t, Epilogue>(
241
+ out, a, bt_nzs, bt_meta,
242
+ std::forward<EpilogueArgs>(epilogue_args)...);
243
+ } else {
244
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
245
+ return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
246
+ cutlass::half_t, Epilogue>(
247
+ out, a, bt_nzs, bt_meta,
248
+ std::forward<EpilogueArgs>(epilogue_args)...);
249
+ }
250
+ } else if (a.dtype() == torch::kFloat16) {
251
+ TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
252
+
253
+ if (out.dtype() == torch::kBFloat16) {
254
+ return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t,
255
+ cutlass::bfloat16_t, Epilogue>(
256
+ out, a, bt_nzs, bt_meta,
257
+ std::forward<EpilogueArgs>(epilogue_args)...);
258
+ } else {
259
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
260
+ return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t, cutlass::half_t,
261
+ Epilogue>(
262
+ out, a, bt_nzs, bt_meta,
263
+ std::forward<EpilogueArgs>(epilogue_args)...);
264
+ }
265
+ } else { // a.dtype() == torch::kBFloat16
266
+ TORCH_CHECK(a.dtype() == torch::kBFloat16);
267
+ TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
268
+
269
+ if (out.dtype() == torch::kBFloat16) {
270
+ return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
271
+ cutlass::bfloat16_t, Epilogue>(
272
+ out, a, bt_nzs, bt_meta,
273
+ std::forward<EpilogueArgs>(epilogue_args)...);
274
+ } else {
275
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
276
+ return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
277
+ cutlass::half_t, Epilogue>(
278
+ out, a, bt_nzs, bt_meta,
279
+ std::forward<EpilogueArgs>(epilogue_args)...);
280
+ }
281
+ }
282
+}
283
+
284
+void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
285
+ torch::Tensor const& bt_nzs,
286
+ torch::Tensor const& bt_meta,
287
+ torch::Tensor const& a_scales,
288
+ torch::Tensor const& b_scales,
289
+ c10::optional<torch::Tensor> const& bias) {
290
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
291
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
292
+ if (bias) {
293
+ TORCH_CHECK(bias->dtype() == out.dtype(),
294
+ "currently bias dtype must match output dtype ", out.dtype());
295
+ return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
296
+ out, a, bt_nzs, bt_meta, b_scales, a_scales, *bias);
297
+ } else {
298
+ return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogue>(
299
+ out, a, bt_nzs, bt_meta, b_scales, a_scales);
300
+ }
301
+}
302
+
303
+#endif
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cuh
new
+496
@@ -0,0 +1,496 @@
1
+// clang-format will break include orders
2
+// clang-format off
3
+#include <cudaTypedefs.h>
4
+
5
+#include <torch/all.h>
6
+
7
+#include <ATen/cuda/CUDAContext.h>
8
+
9
+#include "cutlass/cutlass.h"
10
+
11
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
12
+#include "cutlass/epilogue/collective/collective_builder.hpp"
13
+#include "cutlass/gemm/collective/collective_builder.hpp"
14
+
15
+#include "core/math.hpp"
16
+#include "cutlass_extensions/cute_utils.cuh"
17
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
18
+#include "cutlass_extensions/common.hpp"
19
+#include "cutlass_extensions/torch_utils.hpp"
20
+// clang-format on
21
+
22
+using namespace cute;
23
+
24
+/*
25
+ This file defines sparse quantized GEMM operations using the CUTLASS 3.x API,
26
+ for NVIDIA GPUs with sm90a (Hopper) or later.
27
+*/
28
+
29
+namespace {
30
+
31
+// A wrapper for the GEMM kernel that is used to guard against compilation on
32
+// architectures that will never use the kernel. The purpose of this is to
33
+// reduce the size of the compiled binary.
34
+// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
35
+// into code that will be executed on the device where it is defined.
36
+template <typename Kernel>
37
+struct enable_sm90_or_later : Kernel {
38
+ template <typename... Args>
39
+ CUTLASS_DEVICE void operator()(Args&&... args) {
40
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
41
+ Kernel::operator()(std::forward<Args>(args)...);
42
+#endif
43
+ }
44
+};
45
+
46
+using GemmUniversalMode = cutlass::gemm::GemmUniversalMode;
47
+
48
+template <typename ElementAB_, typename ElementD_,
49
+ template <typename, typename, typename> typename Epilogue_,
50
+ typename TileShape, typename ClusterShape, typename KernelSchedule,
51
+ typename EpilogueSchedule, typename AccType,
52
+ typename TileSchedule = cutlass::gemm::PersistentScheduler,
53
+ GemmUniversalMode Mode_ = GemmUniversalMode::kGemm>
54
+struct cutlass_sparse_3x_gemm {
55
+ static const GemmUniversalMode Mode = Mode_;
56
+ using ElementAB = ElementAB_;
57
+ using ElementD = ElementD_;
58
+ using ElementAcc = AccType;
59
+
60
+ using EpilogueDescriptor =
61
+ cutlass::epilogue::collective::detail::EpilogueDescriptor<
62
+ TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
63
+ ElementD, EpilogueSchedule>;
64
+
65
+ using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
66
+
67
+ using ElementC = void;
68
+ using LayoutC = cutlass::layout::RowMajor;
69
+ using LayoutD = LayoutC;
70
+ using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
71
+ using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
72
+
73
+ using LayoutC_Transpose =
74
+ typename cutlass::layout::LayoutTranspose<LayoutC>::type;
75
+ using LayoutD_Transpose =
76
+ typename cutlass::layout::LayoutTranspose<LayoutD>::type;
77
+
78
+ using EVTCompute = typename Epilogue::EVTCompute;
79
+
80
+ static constexpr int AlignmentA =
81
+ 128 / cutlass::sizeof_bits<ElementAB>::value;
82
+ static constexpr int AlignmentB =
83
+ 128 / cutlass::sizeof_bits<ElementAB>::value;
84
+ static constexpr int AlignmentCD =
85
+ 128 / cutlass::sizeof_bits<ElementD>::value;
86
+
87
+ using CollectiveEpilogue =
88
+ typename cutlass::epilogue::collective::CollectiveBuilder<
89
+ cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
90
+ ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
91
+ ElementAcc, ElementAcc, ElementC, LayoutC_Transpose, AlignmentCD,
92
+ ElementD, LayoutD_Transpose, AlignmentCD, EpilogueSchedule,
93
+ EVTCompute>::CollectiveOp;
94
+
95
+ static constexpr size_t CEStorageSize =
96
+ sizeof(typename CollectiveEpilogue::SharedStorage);
97
+ using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
98
+ static_cast<int>(CEStorageSize)>;
99
+
100
+ // clang-format off
101
+ using CollectiveMainloop =
102
+ typename cutlass::gemm::collective::CollectiveBuilder<
103
+ cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
104
+ ElementAB, cutlass::layout::RowMajor, AlignmentA,
105
+ ElementAB, cutlass::layout::ColumnMajor, AlignmentB,
106
+ ElementAcc, TileShape, ClusterShape,
107
+ Stages,
108
+ KernelSchedule>::CollectiveOp;
109
+ // clang-format on
110
+
111
+ using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
112
+ cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
113
+ TileSchedule>>;
114
+
115
+ struct GemmKernel : public KernelType {};
116
+};
117
+
118
+template <typename Gemm, typename... EpilogueArgs>
119
+void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
120
+ torch::Tensor const& bt_nzs,
121
+ torch::Tensor const& bt_meta,
122
+ EpilogueArgs&&... epilogue_params) {
123
+ using ElementAB = typename Gemm::ElementAB;
124
+ using ElementD = typename Gemm::ElementD;
125
+
126
+ // Interface stride expected from the argument a (will get transposed)
127
+ // We compute C^T = B^T * A^T, but we assume B is transposed before
128
+ // compression and hence the bt_* naming
129
+ using LayoutA = cutlass::layout::RowMajor;
130
+ using LayoutB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutA;
131
+ using LayoutE = typename Gemm::GemmKernel::CollectiveMainloop::LayoutE;
132
+ using LayoutD = cutlass::layout::RowMajor;
133
+
134
+ using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
135
+ using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
136
+
137
+ auto layout_A = make_cute_layout<StrideA>(a, "A");
138
+ auto layout_D = make_cute_layout<StrideD>(out, "D");
139
+
140
+ // Transpose A and D
141
+ // A doesn't need to be transposed since cutlass expects a NxK matrix
142
+ // for B (which is At)
143
+ auto stride_At = layout_A.stride();
144
+ auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
145
+
146
+ using GemmKernel = typename Gemm::GemmKernel;
147
+ typename GemmKernel::ProblemShape prob_shape{
148
+ static_cast<int>(bt_nzs.size(0)), static_cast<int>(size<0>(layout_A)),
149
+ static_cast<int>(size<1>(layout_A)), 1};
150
+
151
+ using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
152
+ using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
153
+
154
+ LayoutB b_layout = SparseConfig::fill_layoutA(prob_shape);
155
+ LayoutE e_layout = SparseConfig::fill_layoutE(prob_shape);
156
+
157
+ auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
158
+ auto b_ptr = static_cast<ElementAB*>(bt_nzs.data_ptr());
159
+ auto e_ptr = static_cast<ElementE*>(bt_meta.data_ptr());
160
+ typename GemmKernel::MainloopArguments mainloop_args{
161
+ b_ptr, b_layout, a_ptr, stride_At, e_ptr, e_layout};
162
+
163
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
164
+ typename GemmKernel::EpilogueArguments epilogue_args{
165
+ Gemm::Epilogue::prepare_args(
166
+ std::forward<EpilogueArgs>(epilogue_params)...),
167
+ c_ptr, stride_Dt, c_ptr, stride_Dt};
168
+
169
+ typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
170
+ prob_shape, mainloop_args, epilogue_args};
171
+
172
+ // Launch the CUTLASS GEMM kernel.
173
+ using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
174
+ GemmOp gemm_op;
175
+ CUTLASS_CHECK(gemm_op.can_implement(args));
176
+
177
+ size_t workspace_size = gemm_op.get_workspace_size(args);
178
+ auto const workspace_options =
179
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
180
+ auto workspace = torch::empty(workspace_size, workspace_options);
181
+
182
+ auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
183
+
184
+ cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
185
+ CUTLASS_CHECK(status);
186
+}
187
+
188
+template <typename InType, typename OutType,
189
+ template <typename, typename, typename> typename Epilogue>
190
+struct sm90_config_default {};
191
+
192
+template <typename OutType,
193
+ template <typename, typename, typename> typename Epilogue>
194
+struct sm90_config_default<half_t, OutType, Epilogue> {
195
+ // M in (128, inf)
196
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
197
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
198
+ using TileShape = Shape<_128, _128, _128>;
199
+ using ClusterShape = Shape<_2, _1, _1>;
200
+ using Cutlass3xGemm =
201
+ cutlass_sparse_3x_gemm<half_t, OutType, Epilogue, TileShape, ClusterShape,
202
+ KernelSchedule, EpilogueSchedule, float>;
203
+};
204
+
205
+template <typename OutType,
206
+ template <typename, typename, typename> typename Epilogue>
207
+struct sm90_config_default<cutlass::bfloat16_t, OutType, Epilogue> {
208
+ // M in (128, inf)
209
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
210
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
211
+ using TileShape = Shape<_128, _128, _128>;
212
+ using ClusterShape = Shape<_2, _1, _1>;
213
+ using Cutlass3xGemm =
214
+ cutlass_sparse_3x_gemm<cutlass::bfloat16_t, OutType, Epilogue, TileShape,
215
+ ClusterShape, KernelSchedule, EpilogueSchedule,
216
+ float>;
217
+};
218
+
219
+//////////////////////// Cherry-Picking Kernels ////////////////////////
220
+template <typename InType, typename OutType,
221
+ template <typename, typename, typename> typename Epilogue>
222
+struct sm90_fp8_config_1 {
223
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
224
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
225
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
226
+ using TileShape = Shape<_64, _64, _256>;
227
+ using ClusterShape = Shape<_8, _1, _1>;
228
+ using Cutlass3xGemm =
229
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
230
+ KernelSchedule, EpilogueSchedule, float>;
231
+};
232
+
233
+template <typename InType, typename OutType,
234
+ template <typename, typename, typename> typename Epilogue>
235
+struct sm90_fp8_config_2 {
236
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
237
+ using KernelSchedule =
238
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
239
+ using EpilogueSchedule =
240
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
241
+ using TileShape = Shape<_128, _64, _256>;
242
+ using ClusterShape = Shape<_8, _1, _1>;
243
+ using Cutlass3xGemm =
244
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
245
+ KernelSchedule, EpilogueSchedule, float>;
246
+};
247
+
248
+template <typename InType, typename OutType,
249
+ template <typename, typename, typename> typename Epilogue>
250
+struct sm90_fp8_config_3 {
251
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
252
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
253
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
254
+ using TileShape = Shape<_64, _64, _256>;
255
+ using ClusterShape = Shape<_1, _2, _1>;
256
+ using Cutlass3xGemm =
257
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
258
+ KernelSchedule, EpilogueSchedule, float>;
259
+};
260
+
261
+template <typename InType, typename OutType,
262
+ template <typename, typename, typename> typename Epilogue>
263
+struct sm90_fp8_config_4 {
264
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
265
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
266
+ using EpilogueSchedule =
267
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
268
+ using TileShape = Shape<_64, _128, _256>;
269
+ using ClusterShape = Shape<_8, _1, _1>;
270
+ using Cutlass3xGemm =
271
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
272
+ KernelSchedule, EpilogueSchedule, float>;
273
+};
274
+
275
+template <typename InType, typename OutType,
276
+ template <typename, typename, typename> typename Epilogue>
277
+struct sm90_fp8_config_5 {
278
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
279
+ using KernelSchedule =
280
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
281
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
282
+ using TileShape = Shape<_128, _128, _256>;
283
+ using ClusterShape = Shape<_8, _1, _1>;
284
+ using Cutlass3xGemm =
285
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
286
+ KernelSchedule, EpilogueSchedule, float>;
287
+};
288
+
289
+template <typename InType, typename OutType,
290
+ template <typename, typename, typename> typename Epilogue>
291
+struct sm90_fp8_config_6 {
292
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
293
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
294
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
295
+ using TileShape = Shape<_64, _128, _256>;
296
+ using ClusterShape = Shape<_1, _2, _1>;
297
+ using Cutlass3xGemm =
298
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
299
+ KernelSchedule, EpilogueSchedule, float>;
300
+};
301
+
302
+template <typename InType, typename OutType,
303
+ template <typename, typename, typename> typename Epilogue>
304
+struct sm90_fp8_config_7 {
305
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
306
+ using KernelSchedule =
307
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
308
+ using EpilogueSchedule =
309
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
310
+ using TileShape = Shape<_128, _128, _256>;
311
+ using ClusterShape = Shape<_1, _1, _1>;
312
+ using Cutlass3xGemm =
313
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
314
+ KernelSchedule, EpilogueSchedule, float>;
315
+};
316
+
317
+template <typename InType, typename OutType,
318
+ template <typename, typename, typename> typename Epilogue>
319
+struct sm90_fp8_config_8 {
320
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
321
+ using KernelSchedule =
322
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
323
+ using EpilogueSchedule =
324
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
325
+ using TileShape = Shape<_128, _256, _128>;
326
+ using ClusterShape = Shape<_8, _1, _1>;
327
+ using Cutlass3xGemm =
328
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
329
+ KernelSchedule, EpilogueSchedule, float>;
330
+};
331
+////////////////////////////////////////////////////////////////////////
332
+
333
+template <typename OutType,
334
+ template <typename, typename, typename> typename Epilogue>
335
+struct sm90_config_default<cutlass::float_e4m3_t, OutType, Epilogue> {
336
+ // M in (128, inf)
337
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
338
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
339
+ using TileShape = Shape<_128, _128, _128>;
340
+ using ClusterShape = Shape<_1, _2, _1>;
341
+ using Cutlass3xGemm =
342
+ cutlass_sparse_3x_gemm<cutlass::float_e4m3_t, OutType, Epilogue,
343
+ TileShape, ClusterShape, KernelSchedule,
344
+ EpilogueSchedule, float>;
345
+};
346
+
347
+template <typename InType, typename OutType,
348
+ template <typename, typename, typename> typename Epilogue>
349
+struct sm90_fp8_config_M64 {
350
+ // M in [1, 64]
351
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
352
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
353
+ using EpilogueSchedule =
354
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
355
+ using TileShape = Shape<_64, _64, _256>;
356
+ using ClusterShape = Shape<_1, _1, _1>;
357
+
358
+ using TileSchedule = cutlass::gemm::PersistentScheduler;
359
+
360
+ using Cutlass3xGemm =
361
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
362
+ KernelSchedule, EpilogueSchedule, float,
363
+ TileSchedule>;
364
+};
365
+
366
+template <typename InType, typename OutType,
367
+ template <typename, typename, typename> typename Epilogue>
368
+struct sm90_fp8_config_M128 {
369
+ // M in (64, 128]
370
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
371
+ using KernelSchedule =
372
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
373
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
374
+ using TileShape = Shape<_64, _128, _256>;
375
+ using ClusterShape = Shape<_1, _1, _1>;
376
+
377
+ using TileSchedule = cutlass::gemm::PersistentScheduler;
378
+
379
+ using Cutlass3xGemm =
380
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
381
+ KernelSchedule, EpilogueSchedule, float,
382
+ TileSchedule>;
383
+};
384
+
385
+template <typename InType, typename OutType,
386
+ template <typename, typename, typename> typename Epilogue>
387
+struct sm90_fp8_config_M256 {
388
+ // M in (128, 256]
389
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
390
+ using KernelSchedule =
391
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
392
+ using EpilogueSchedule =
393
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
394
+ using TileShape = Shape<_128, _128, _256>;
395
+ using ClusterShape = Shape<_1, _1, _1>;
396
+
397
+ using TileSchedule = cutlass::gemm::PersistentScheduler;
398
+
399
+ using Cutlass3xGemm =
400
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
401
+ KernelSchedule, EpilogueSchedule, float,
402
+ TileSchedule>;
403
+};
404
+
405
+template <typename InType, typename OutType,
406
+ template <typename, typename, typename> typename Epilogue>
407
+struct sm90_fp8_config_M512 {
408
+ // M in (256, ]
409
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
410
+ using KernelSchedule =
411
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
412
+ using EpilogueSchedule =
413
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
414
+ using TileShape = Shape<_128, _128, _256>;
415
+ using ClusterShape = Shape<_1, _1, _1>;
416
+
417
+ using TileSchedule = cutlass::gemm::PersistentScheduler;
418
+
419
+ using Cutlass3xGemm =
420
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
421
+ KernelSchedule, EpilogueSchedule, float,
422
+ TileSchedule>;
423
+};
424
+
425
+template <typename OutType,
426
+ template <typename, typename, typename> typename Epilogue>
427
+struct sm90_config_default<int8_t, OutType, Epilogue> {
428
+ // For M > 128 and any N
429
+ using KernelSchedule =
430
+ typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
431
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
432
+ using TileShape = Shape<_128, _128, _128>;
433
+ using ClusterShape = Shape<_2, _1, _1>;
434
+ using Cutlass3xGemm =
435
+ cutlass_sparse_3x_gemm<int8_t, OutType, Epilogue, TileShape, ClusterShape,
436
+ KernelSchedule, EpilogueSchedule, int32_t>;
437
+};
438
+
439
+template <typename InType, typename OutType,
440
+ template <typename, typename, typename> typename Epilogue>
441
+struct sm90_int8_config_M128 {
442
+ // For M in (64, 128] and any N
443
+ static_assert(std::is_same<InType, int8_t>());
444
+ using KernelSchedule =
445
+ typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
446
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
447
+ using TileShape = Shape<_64, _128, _128>;
448
+ using ClusterShape = Shape<_2, _1, _1>;
449
+ using Cutlass3xGemm =
450
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
451
+ KernelSchedule, EpilogueSchedule, int32_t>;
452
+};
453
+
454
+template <typename InType, typename OutType,
455
+ template <typename, typename, typename> typename Epilogue>
456
+struct sm90_int8_config_M64 {
457
+ // For M in (32, 64] and any N
458
+ static_assert(std::is_same<InType, int8_t>());
459
+ using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
460
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
461
+ using TileShape = Shape<_64, _64, _256>;
462
+ using ClusterShape = Shape<_1, _1, _1>;
463
+ using Cutlass3xGemm =
464
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
465
+ KernelSchedule, EpilogueSchedule, int32_t>;
466
+};
467
+
468
+template <typename InType, typename OutType,
469
+ template <typename, typename, typename> typename Epilogue>
470
+struct sm90_int8_config_M32_NBig {
471
+ // For M in [1, 32] and N >= 8192
472
+ static_assert(std::is_same<InType, int8_t>());
473
+ using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
474
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
475
+ using TileShape = Shape<_64, _128, _256>;
476
+ using ClusterShape = Shape<_1, _4, _1>;
477
+ using Cutlass3xGemm =
478
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
479
+ KernelSchedule, EpilogueSchedule, int32_t>;
480
+};
481
+
482
+template <typename InType, typename OutType,
483
+ template <typename, typename, typename> typename Epilogue>
484
+struct sm90_int8_config_M32_NSmall {
485
+ // For M in [1, 32] and N < 8192
486
+ static_assert(std::is_same<InType, int8_t>());
487
+ using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
488
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
489
+ using TileShape = Shape<_64, _64, _256>;
490
+ using ClusterShape = Shape<_1, _8, _1>;
491
+ using Cutlass3xGemm =
492
+ cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
493
+ KernelSchedule, EpilogueSchedule, int32_t>;
494
+};
495
+
496
+} // namespace
\ No newline at end of file
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
new
+59
@@ -0,0 +1,59 @@
1
+#include <cudaTypedefs.h>
2
+
3
+#include <c10/cuda/CUDAGuard.h>
4
+#include <torch/all.h>
5
+
6
+#include "cutlass_extensions/common.hpp"
7
+
8
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
9
+void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
10
+ torch::Tensor const& b,
11
+ torch::Tensor const& e,
12
+ torch::Tensor const& a_scales,
13
+ torch::Tensor const& b_scales,
14
+ c10::optional<torch::Tensor> const& bias);
15
+#endif
16
+
17
+void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
18
+ torch::Tensor const& bt_nzs,
19
+ torch::Tensor const& bt_meta,
20
+ torch::Tensor const& a_scales,
21
+ torch::Tensor const& b_scales,
22
+ c10::optional<torch::Tensor> const& bias) {
23
+ // Checks for conformality
24
+ TORCH_CHECK(a.dim() == 2 && bt_nzs.dim() == 2 && c.dim() == 2);
25
+ TORCH_CHECK(c.size(1) == bt_nzs.size(0) && bt_nzs.size(1) * 2 == a.size(1) &&
26
+ a.size(0) == c.size(0));
27
+ TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
28
+ TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == bt_nzs.size(0));
29
+
30
+ // Check for strides and alignment
31
+ TORCH_CHECK(a.stride(1) == 1 && bt_nzs.stride(1) == 1 &&
32
+ c.stride(1) == 1); // Row-major
33
+ TORCH_CHECK(c.stride(0) % 16 == 0); // 16 Byte Alignment
34
+ TORCH_CHECK(bt_nzs.stride(0) % 16 == 0); // 16 Byte Alignment
35
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
36
+
37
+ if (bias) {
38
+ TORCH_CHECK(bias->numel() == bt_nzs.size(0) && bias->is_contiguous() &&
39
+ bias->dim() == 1);
40
+ }
41
+
42
+ at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
43
+ int32_t version_num = get_sm_version_num();
44
+
45
+ // Guard against compilation issues for sm90 kernels
46
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
47
+ if (version_num >= 90) {
48
+ cutlass_scaled_sparse_mm_sm90(c, a, bt_nzs, bt_meta, a_scales, b_scales,
49
+ bias);
50
+ return;
51
+ }
52
+#endif
53
+
54
+ TORCH_CHECK_NOT_IMPLEMENTED(
55
+ false,
56
+ "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
57
+ "CUDA device capability: ",
58
+ version_num);
59
+}
csrc/torch_bindings.cpp
+15
@@ -321,6 +321,21 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
321
ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
322
ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
323
324
+ // CUTLASS sparse GEMM, supporting symmetric per-tensor or per-row/column
325
+ // quantization, as well as bias
326
+ ops.def(
327
+ "cutlass_scaled_sparse_mm(Tensor! out, Tensor a,"
328
+ " Tensor bt_nzs,"
329
+ " Tensor bt_meta, Tensor a_scales,"
330
+ " Tensor b_scales, Tensor? bias) -> ()");
331
+ ops.impl("cutlass_scaled_sparse_mm", torch::kCUDA, &cutlass_scaled_sparse_mm);
332
+
333
+ // CUTLASS sparse matrix compressor
334
+ ops.def(
335
+ "cutlass_sparse_compress_entry(Tensor! a_nzs, Tensor! a_meta,"
336
+ " Tensor a) -> bool");
337
+ ops.impl("cutlass_sparse_compress_entry", &cutlass_sparse_compress_entry);
338
+
339
// Mamba selective scan kernel
340
ops.def(
341
"selective_scan_fwd(Tensor! u, Tensor! delta,"
pyproject.toml
+1
-1
@@ -83,7 +83,7 @@ exclude = [
83
]
84
85
[tool.codespell]
86
-ignore-words-list = "dout, te, indicies, subtile"
86
+ignore-words-list = "dout, te, indicies, subtile, ElementE"
87
skip = "./tests/models/fixtures,./tests/prompts,./benchmarks/sonnet.txt,./tests/lora/data,./build"
88
89
[tool.isort]
tests/kernels/test_semi_structured.py
new
+131
@@ -0,0 +1,131 @@
1
+"""Tests for sparse cutlass kernels
2
+
3
+Run `pytest tests/kernels/test_semi_structured.py`.
4
+"""
5
+from typing import Optional, Tuple, Type
6
+
7
+import pytest
8
+import torch
9
+
10
+from vllm import _custom_ops as ops
11
+from vllm.platforms import current_platform
12
+
13
+CUDA_DEVICES = [
14
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15
+]
16
+
17
+capability = current_platform.get_device_capability()
18
+capability = capability[0] * 10 + capability[1]
19
+
20
+
21
+def to_fp8(tensor: torch.Tensor):
22
+ finfo = torch.finfo(torch.float8_e4m3fn)
23
+ return torch.round(tensor.clamp(
24
+ min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
25
+
26
+
27
+def to_int8(tensor: torch.Tensor):
28
+ return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
29
+
30
+
31
+def rand_int8(shape: tuple, device: str = "cuda"):
32
+ return to_int8(torch.rand(shape, device=device) * 255 - 128)
33
+
34
+
35
+def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
36
+ return tensor.to(dtype=torch.bfloat16)
37
+
38
+
39
+def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
40
+ return tensor.to(dtype=torch.float16)
41
+
42
+
43
+def prune_to_2_4(tensor):
44
+ # Reshape tensor to [N, 4] where N is number of groups of 4
45
+ original_shape = tensor.shape
46
+ reshaped = tensor.reshape(-1, 4)
47
+
48
+ # Get indices of top 2 absolute values in each group of 4
49
+ _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
50
+
51
+ # Create binary mask
52
+ mask = torch.zeros_like(reshaped)
53
+ mask.scatter_(dim=1,
54
+ index=indices,
55
+ src=torch.ones_like(indices, dtype=mask.dtype))
56
+
57
+ # Apply mask and reshape back
58
+ pruned = reshaped * mask
59
+
60
+ # Turn all -0.0 to 0.0
61
+ pruned[pruned == -0.0] = 0.0
62
+
63
+ return pruned.reshape(original_shape)
64
+
65
+
66
+def make_rand_sparse_tensors(
67
+ dtype: torch.dtype, m: int, n: int, k: int
68
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
69
+ a = torch.randn((m, k), device='cuda') * 5
70
+ b = torch.randn((n, k), device='cuda').t() * 5
71
+
72
+ b = prune_to_2_4(b.t()).t()
73
+
74
+ if dtype == torch.int8:
75
+ a, b = to_int8(a), to_int8(b)
76
+ elif dtype == torch.float8_e4m3fn:
77
+ a, b = to_fp8(a), to_fp8(b)
78
+ elif dtype == torch.float16:
79
+ a, b = to_fp16(a), to_fp16(b)
80
+ elif dtype == torch.bfloat16:
81
+ a, b = to_bf16(a), to_bf16(b)
82
+ else:
83
+ raise ValueError("unsupported dtype")
84
+
85
+ b_compressed, e = ops.cutlass_sparse_compress(b.t())
86
+
87
+ # Compressed B, Metadata, Original A, B
88
+ return b_compressed, e, a, b
89
+
90
+
91
+def baseline_scaled_mm(a: torch.Tensor,
92
+ b: torch.Tensor,
93
+ scale_a: torch.Tensor,
94
+ scale_b: torch.Tensor,
95
+ out_dtype: Type[torch.dtype],
96
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
97
+ output = (scale_a * (scale_b * (torch.mm(
98
+ a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
99
+ if bias is not None:
100
+ output = output + bias
101
+
102
+ return output
103
+
104
+
105
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
106
+ reason="Sparse FP8 is not yet supported on this GPU type.")
107
+# Test working with a subset of A and B for sparse matmul
108
+def test_cutlass_sparse_subset():
109
+ big_m = 1024
110
+ m, n, k = 512, 512, 512
111
+
112
+ # Create tensors
113
+ b_comp, e, whole_a, b = make_rand_sparse_tensors(torch.float8_e4m3fn,
114
+ big_m, n, k)
115
+ a = whole_a[0:m, 0:k]
116
+ scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
117
+ scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
118
+
119
+ out = ops.cutlass_scaled_sparse_mm(a,
120
+ b_comp,
121
+ e,
122
+ scale_a,
123
+ scale_b,
124
+ out_dtype=torch.bfloat16)
125
+ baseline = baseline_scaled_mm(a,
126
+ b,
127
+ scale_a,
128
+ scale_b,
129
+ out_dtype=torch.bfloat16)
130
+
131
+ torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
tests/quantization/test_compressed_tensors.py
+100
-3
@@ -10,9 +10,11 @@ from compressed_tensors.quantization import QuantizationType
10
11
from tests.models.utils import check_logprobs_close
12
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
13
- CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
14
- CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
15
- CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13
+ CompressedTensors24, CompressedTensorsLinearMethod,
14
+ CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
15
+ CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
16
+ CompressedTensorsWNA16)
17
+from vllm.platforms import current_platform
18
19
20
@pytest.mark.parametrize(
@@ -208,3 +210,98 @@ def test_compressed_tensors_kv_cache(vllm_runner):
210
with vllm_runner(model_path, kv_cache_dtype="fp8") as llm:
211
output = llm.generate_greedy("Hello world!", max_tokens=20)
212
assert output
213
+
214
+
215
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
216
+ reason="Sparse FP8 is not yet supported on this GPU type.")
217
+def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
218
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
219
+ assert isinstance(qkv_proj.scheme, CompressedTensors24)
220
+
221
+ assert qkv_proj.scheme.weight_quant.strategy == weight_strategy
222
+ assert qkv_proj.scheme.input_quant.strategy == input_strategy
223
+ assert qkv_proj.scheme.quantized
224
+ assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
225
+ sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
226
+ assert sparsity_map.get("Linear").format == "dense"
227
+ assert sparsity_map.get("Linear").sparsity_structure == "2:4"
228
+
229
+
230
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
231
+ reason="Sparse FP8 is not yet supported on this GPU type.")
232
+@pytest.mark.parametrize("args_2of4", [
233
+ ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing", "channel",
234
+ "token"),
235
+ ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
236
+ "channel", "tensor"),
237
+ ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing", "tensor",
238
+ "tensor"),
239
+ ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
240
+ "tensor", "token"),
241
+])
242
+def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
243
+ model, weight_strategy, input_strategy = args_2of4
244
+ with vllm_runner(model) as llm:
245
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
246
+ layer = model.model.layers[0]
247
+
248
+ qkv_proj = layer.self_attn.qkv_proj
249
+ assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
250
+ _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
251
+
252
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
253
+ print(output)
254
+ assert output
255
+
256
+
257
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
258
+ reason="Sparse FP8 is not yet supported on this GPU type.")
259
+@pytest.mark.parametrize("args_2of4", [
260
+ ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
261
+ "channel", "token"),
262
+ ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing", "tensor",
263
+ "tensor"),
264
+ ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
265
+ "tensor", "token"),
266
+])
267
+def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
268
+ model, weight_strategy, input_strategy = args_2of4
269
+ with vllm_runner(model) as llm:
270
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
271
+ layer = model.model.layers[0]
272
+
273
+ qkv_proj = layer.self_attn.qkv_proj
274
+ assert qkv_proj.scheme.weights_dtype == torch.int8
275
+ _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
276
+
277
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
278
+ print(output)
279
+ assert output
280
+
281
+
282
+@pytest.mark.skipif(not current_platform.has_device_capability(90),
283
+ reason="Sparse FP8 is not yet supported on this GPU type.")
284
+@pytest.mark.parametrize(
285
+ "args_2of4",
286
+ [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
287
+def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
288
+ model = args_2of4
289
+ with vllm_runner(model) as llm:
290
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
291
+ layer = model.model.layers[0]
292
+
293
+ qkv_proj = layer.self_attn.qkv_proj
294
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
295
+ assert isinstance(qkv_proj.scheme, CompressedTensors24)
296
+
297
+ assert qkv_proj.scheme.weight_quant is None
298
+ assert qkv_proj.scheme.input_quant is None
299
+ assert not qkv_proj.scheme.quantized
300
+ assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
301
+ sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
302
+ assert sparsity_map.get("Linear").format == "dense"
303
+ assert sparsity_map.get("Linear").sparsity_structure == "2:4"
304
+
305
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
306
+ print(output)
307
+ assert output
tests/weight_loading/models.txt
+2
@@ -21,6 +21,8 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22
compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23
compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
24
+compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-FP8-Dynamic-testing, main, 90
25
+compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-W8A8-testing, main, 90
26
awq, casperhansen/mixtral-instruct-awq, main
27
awq_marlin, casperhansen/mixtral-instruct-awq, main
28
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
tests/weight_loading/run_model_weight_loading_test.sh
+4
@@ -26,6 +26,10 @@ do
26
export QUANTIZATION=${array[0]}
27
export MODEL_NAME=${array[1]}
28
export REVISION=${array[2]}
29
+ # If array length is larger than 3, then MIN_CAPABILITY is provided
30
+ if [ ${#array[@]} -gt 3 ]; then
31
+ export MIN_CAPABILITY=${array[3]}
32
+ fi
33
pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$?
34
35
if [[ $LOCAL_SUCCESS == 0 ]]; then
tests/weight_loading/test_weight_loading.py
+7
@@ -1,14 +1,21 @@
1
import os
2
3
+import pytest
4
import torch
5
6
+from vllm.platforms import current_platform
7
+
8
MAX_MODEL_LEN = 1024
9
MODEL_NAME = os.environ.get("MODEL_NAME",
10
"robertgshaw2/zephyr-7b-beta-channelwise-gptq")
11
REVISION = os.environ.get("REVISION", "main")
12
QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
13
+MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "89")
14
15
16
+@pytest.mark.skipif(
17
+ not current_platform.has_device_capability(int(MIN_CAPABILITY)),
18
+ reason="Current system does not have minimum capability.")
19
def test_weight_loading(vllm_runner):
20
"""
21
Test parameter weight loading with tp>1.
vllm/_custom_ops.py
+103
@@ -552,6 +552,109 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
552
return out
553
554
555
+def cutlass_sparse_compress(a: torch.Tensor) \
556
+ -> Tuple[torch.Tensor, torch.Tensor]:
557
+ """
558
+ Compresses a sparse matrix for use with Cutlass sparse operations.
559
+
560
+ This function takes a dense tensor and compresses it into two components:
561
+ non-zero elements and metadata. The compressed representation is compatible
562
+ with Cutlass sparse kernels.
563
+
564
+ Args:
565
+ a (torch.Tensor):
566
+ The input tensor to be compressed. Must have one of the following data types:
567
+ - `torch.int8`
568
+ - `torch.float8_e4m3fn`
569
+ - `torch.bfloat16`
570
+ - `torch.float16`
571
+
572
+ Returns:
573
+ Tuple[torch.Tensor, torch.Tensor]:
574
+ A tuple containing:
575
+ - `a_nzs` (torch.Tensor): A tensor containing non-zero elements of `a`.
576
+ - `a_meta` (torch.Tensor): A tensor containing metadata for the sparse representation.
577
+
578
+ Raises:
579
+ ValueError: If the compression operation fails.
580
+
581
+ Notes:
582
+ - The `a_meta` tensor has a data type of `torch.uint8`.
583
+ - Each metadata element encodes the sparsity of 4 non-zero elements (i.e., `elemsPerMetaElem = 4`).
584
+ - The shape of `a_nzs` is `(m, k // 2)`, where `m` and `k` are the dimensions of the input tensor.
585
+ - The shape of `a_meta` is `(m, k // 2 // elemsPerMetaElem)`.
586
+ """
587
+ assert (a.dtype in [
588
+ torch.int8, torch.float8_e4m3fn, torch.bfloat16, torch.float16
589
+ ])
590
+ assert (a.is_contiguous())
591
+
592
+ # a_meta.dtype: torch.uint8 so elemsPerMetaElem = 8b / 2b_per_nz = 4
593
+ elemsPerMetaElem = 4
594
+
595
+ m = a.shape[0]
596
+ k = a.shape[1]
597
+ assert (k % 2 == 0)
598
+ a_nzs = torch.empty((m, k // 2), dtype=a.dtype, device=a.device)
599
+ a_meta = torch.empty((m, k // 2 // elemsPerMetaElem),
600
+ dtype=torch.uint8,
601
+ device=a.device)
602
+
603
+ if not (torch.ops._C.cutlass_sparse_compress_entry(a_nzs, a_meta, a)):
604
+ raise ValueError
605
+
606
+ assert (a_nzs.is_contiguous())
607
+ assert (a_meta.is_contiguous())
608
+
609
+ return a_nzs, a_meta
610
+
611
+
612
+def cutlass_scaled_sparse_mm(
613
+ a: torch.Tensor,
614
+ bt_nzs: torch.Tensor,
615
+ bt_meta: torch.Tensor,
616
+ scale_a: torch.Tensor,
617
+ scale_b: torch.Tensor,
618
+ out_dtype: torch.dtype,
619
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
620
+ """
621
+ Performs a scaled sparse matrix multiplication using Cutlass.
622
+
623
+ Steps:
624
+ 1. Create a dense matrix `a` of shape (m, k) on the CUDA device:
625
+ `a = torch.randn((m, k), device='cuda')`.
626
+
627
+ 2. Create a dense matrix `b` of shape (k, n) on the CUDA device:
628
+ `b = torch.randn((k, n), device='cuda')`.
629
+
630
+ 3. Prune matrix `b` to 2:4 sparsity along the specified dimension:
631
+ `b = prune_to_2_4(b, dim=0)`.
632
+
633
+ 4. Compress the transposed sparse matrix `b.t()`:
634
+ `bt_nzs, bt_meta = cutlass_sparse_compress(b.t())`.
635
+
636
+ 5. Perform sparse matrix multiplication using the compressed matrix,
637
+ applying scaling factors for `a` and `b`, and the output data type:
638
+ `out = cutlass_scaled_sparse_mm(a, bt_nzs, bt_meta, scale_a, scale_b, out_dtype)`.
639
+
640
+ Returns:
641
+ - The result of the scaled sparse matrix multiplication.
642
+ """
643
+ assert (bt_nzs.shape[0] % 16 == 0 and bt_nzs.shape[1] % 16 == 0)
644
+ assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
645
+ assert bias is None or bias.shape[0] == bt_nzs.shape[0] \
646
+ and bias.dtype == out_dtype
647
+
648
+ m = a.shape[0]
649
+ n = bt_nzs.shape[0]
650
+ out = torch.empty((m, n), dtype=out_dtype, device=a.device)
651
+
652
+ torch.ops._C.cutlass_scaled_sparse_mm(out, a, bt_nzs, bt_meta, scale_a,
653
+ scale_b, bias)
654
+
655
+ return out
656
+
657
+
658
# aqlm
659
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
660
codebooks: torch.Tensor, scales: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+160
-27
@@ -1,7 +1,9 @@
1
-from typing import Any, Dict, List, Optional, cast
1
+from typing import Any, Dict, List, Literal, Optional, cast
2
3
import torch
4
-from compressed_tensors.config import CompressionFormat
4
+from compressed_tensors.config import (CompressionFormat,
5
+ SparsityCompressionConfig,
6
+ SparsityStructure)
7
from compressed_tensors.quantization import (QuantizationArgs,
8
QuantizationStrategy,
9
QuantizationType)
@@ -15,7 +17,7 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
17
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
18
CompressedTensorsMoEMethod)
19
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
18
- W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
20
+ W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS, CompressedTensors24,
21
CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
22
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
23
CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
@@ -27,20 +29,29 @@ from vllm.platforms import current_platform
29
30
__all__ = ["CompressedTensorsLinearMethod"]
31
32
+SPARSITY_CONFIG_NAME: Literal["sparsity_config"] = "sparsity_config"
33
+QUANTIZATION_SCHEME_MAP_TYPE = Dict[str, Optional[Dict[str, QuantizationArgs]]]
34
+
35
36
class CompressedTensorsConfig(QuantizationConfig):
37
33
- def __init__(self,
34
- target_scheme_map: Dict[str, Any],
35
- ignore: List[str],
36
- quant_format: str,
37
- kv_cache_scheme: Optional[Dict[str, Any]] = None):
38
+ def __init__(
39
+ self,
40
+ target_scheme_map: Dict[str, Any],
41
+ ignore: List[str],
42
+ quant_format: str,
43
+ sparsity_scheme_map: Dict[str, SparsityCompressionConfig],
44
+ kv_cache_scheme: Optional[Dict[str, Any]] = None,
45
+ config: Optional[Dict[str, Any]] = None,
46
+ ):
47
48
self.ignore = ignore
49
self.quant_format = quant_format
50
# Map from [target -> scheme]
51
self.target_scheme_map = target_scheme_map
52
self.kv_cache_scheme = kv_cache_scheme
53
+ self.sparsity_scheme_map = sparsity_scheme_map
54
+ self.config = config
55
56
def get_linear_method(self) -> "CompressedTensorsLinearMethod":
57
return CompressedTensorsLinearMethod(self)
@@ -78,8 +89,50 @@ class CompressedTensorsConfig(QuantizationConfig):
89
90
@classmethod
91
def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
92
+ ignore: List[str] = cast(List[str], config.get("ignore", []))
93
+ quant_format = cast(str, config.get("format"))
94
+ target_scheme_map = cls._quantization_scheme_map_from_config(
95
+ config=config)
96
+ sparsity_scheme_map = cls._sparsity_scheme_map_from_config(
97
+ config=config)
98
+
99
+ return cls(
100
+ target_scheme_map=target_scheme_map,
101
+ ignore=ignore,
102
+ quant_format=quant_format,
103
+ sparsity_scheme_map=sparsity_scheme_map,
104
+ config=config,
105
+ )
106
+
107
+ @classmethod
108
+ def _sparsity_scheme_map_from_config(
109
+ cls, config: Dict[str,
110
+ Any]) -> Dict[str, SparsityCompressionConfig]:
111
+ """
112
+ :param config: The `quantization_config` dictionary from config.json
113
+ :return: A dictionary mapping target layer names to their corresponding
114
+ sparsity compression configurations
115
+ """
116
+ if (sparsity_config := config.get(SPARSITY_CONFIG_NAME)) is None:
117
+ return dict()
118
+
119
+ sparsity_config = SparsityCompressionConfig.model_validate(
120
+ sparsity_config)
121
+ sparse_scheme_map: Dict[str, SparsityCompressionConfig] = {
122
+ target: sparsity_config
123
+ for target in sparsity_config.targets or list()
124
+ }
125
+ return sparse_scheme_map
126
+
127
+ @classmethod
128
+ def _quantization_scheme_map_from_config(
129
+ cls, config: Dict[str, Any]) -> QUANTIZATION_SCHEME_MAP_TYPE:
130
+ """
131
+ :param config: The `quantization_config` dictionary from config.json
132
+ :return: A dictionary mapping target layer names to their corresponding
133
+ quantization_args for weights and input activations
134
+ """
135
target_scheme_map: Dict[str, Any] = dict()
82
- ignore = cast(List[str], config.get("ignore"))
136
quant_format = cast(str, config.get("format"))
137
138
# The quant_config has multiple config_groups, each containing
@@ -90,12 +143,14 @@ class CompressedTensorsConfig(QuantizationConfig):
143
# details follow the structure defined by the QuantizationArgs
144
# pydantic model, which is used to verify the structure of the
145
# quant_config and also store the details for later use.
93
- for _, quant_config in config["config_groups"].items():
146
+
147
+ config_groups = config.get("config_groups", dict())
148
+ for _, quant_config in config_groups.items():
149
targets = quant_config.get("targets")
150
for target in targets:
151
target_scheme_map[target] = {}
152
target_scheme_map[target][
98
- "weights"] = QuantizationArgs.parse_obj(
153
+ "weights"] = QuantizationArgs.model_validate(
154
quant_config.get("weights"))
155
156
target_scheme_map[target]["input_activations"] = None
@@ -110,13 +165,9 @@ class CompressedTensorsConfig(QuantizationConfig):
165
"weights"].type == QuantizationType.FLOAT
166
else:
167
target_scheme_map[target][
113
- "input_activations"] = QuantizationArgs.parse_obj(
168
+ "input_activations"] = QuantizationArgs.model_validate( # noqa: E501
169
quant_config.get("input_activations"))
115
-
116
- return cls(target_scheme_map=target_scheme_map,
117
- ignore=ignore,
118
- quant_format=quant_format,
119
- kv_cache_scheme=config.get("kv_cache_scheme"))
170
+ return target_scheme_map
171
172
@classmethod
173
def get_config_filenames(cls) -> List[str]:
@@ -315,23 +366,105 @@ class CompressedTensorsConfig(QuantizationConfig):
366
# TODO (@robertgshaw): add compressed-tensors as dep
367
# so we do not have to re-write these functions
368
# need to make accelerate optional in ct to do this
318
- matched_target = find_matched_target(
319
- layer_name=layer_name,
320
- module=layer,
321
- targets=self.target_scheme_map.keys())
369
323
- # Find the quant_scheme
324
- scheme_dict = self.target_scheme_map[matched_target]
325
- scheme = self._get_scheme_from_parts(
326
- weight_quant=scheme_dict["weights"],
327
- input_quant=scheme_dict["input_activations"])
370
+ # Will be empty for models with only sparsity
371
+ if self.target_scheme_map:
372
+ matched_target = find_matched_target(
373
+ layer_name=layer_name,
374
+ module=layer,
375
+ targets=self.target_scheme_map.keys())
376
+
377
+ scheme_dict = self.target_scheme_map[matched_target]
378
+ weight_quant = scheme_dict.get("weights")
379
+ input_quant = scheme_dict.get("input_activations")
380
+ elif self.sparsity_scheme_map:
381
+ matched_target = find_matched_target(
382
+ layer_name=layer_name,
383
+ module=layer,
384
+ targets=self.sparsity_scheme_map.keys())
385
+ weight_quant = None
386
+ input_quant = None
387
+
388
+ # For models with sparsity, assumes that the sparse layers are also
389
+ # quantized for cutlass 2:4 support
390
+ sparsity_scheme: Optional[
391
+ SparsityCompressionConfig] = self.sparsity_scheme_map.get(
392
+ matched_target)
393
+
394
+ if self.supports_cutlass_24(weight_quant=weight_quant,
395
+ input_quant=input_quant,
396
+ sparsity_scheme=sparsity_scheme):
397
+ # Have a valid sparsity scheme
398
+ # Validate layer is supported by Cutlass 2:4 Kernel
399
+ scheme = CompressedTensors24(quantized=weight_quant is not None
400
+ or input_quant is not None,
401
+ weight_quant=weight_quant,
402
+ input_quant=input_quant)
403
+ else:
404
+ # Find the quant_scheme
405
+ scheme = self._get_scheme_from_parts( # type: ignore
406
+ weight_quant=weight_quant,
407
+ input_quant=input_quant,
408
+ )
409
410
# Raise error if device does not support the scheme
411
# (e.g. fp8 needs ada lovelace)
412
self._check_scheme_supported(scheme.get_min_capability())
332
-
413
return scheme
414
415
+ @staticmethod
416
+ def supports_cutlass_24(
417
+ weight_quant: Optional[QuantizationArgs],
418
+ input_quant: Optional[QuantizationArgs],
419
+ sparsity_scheme: Optional[SparsityCompressionConfig] = None
420
+ ) -> bool:
421
+ """
422
+ Check if the layer is supported by the Cutlass 2:4 Kernel
423
+ Conditions:
424
+ - Overarching condition: Sparsity Structure is 2:4
425
+ - Unquantized cases are supported
426
+ - Weight only quantization is not-supported
427
+ - Supported weight quantization strategies are TENSOR and CHANNEL
428
+ - Supported input quantization strategies are TENSOR and TOKEN
429
+ - Only 8 bit quantization is supported
430
+
431
+ :return: True if the layer is supported by the Cutlass 2:4 Kernel
432
+ False otherwise
433
+ """
434
+ is_valid_sparsity = (sparsity_scheme is not None
435
+ and sparsity_scheme.sparsity_structure
436
+ == SparsityStructure.TWO_FOUR.value
437
+ and sparsity_scheme.format == "dense")
438
+ if not is_valid_sparsity:
439
+ return False
440
+
441
+ # Unquantized cases are supported
442
+ if weight_quant is None and input_quant is None:
443
+ return True
444
+
445
+ # Weight only quantization is not-supported
446
+ if weight_quant is not None and input_quant is None:
447
+ return False
448
+
449
+ supported_weight_quant_strategies = [
450
+ QuantizationStrategy.TENSOR.value,
451
+ QuantizationStrategy.CHANNEL.value
452
+ ]
453
+
454
+ assert weight_quant is not None
455
+ assert input_quant is not None
456
+ if weight_quant.strategy not in supported_weight_quant_strategies:
457
+ return False
458
+
459
+ supported_input_quant_strategies = [
460
+ QuantizationStrategy.TENSOR.value, QuantizationStrategy.TOKEN.value
461
+ ]
462
+
463
+ if input_quant.strategy not in supported_input_quant_strategies:
464
+ return False
465
+
466
+ return weight_quant.num_bits == input_quant.num_bits == 8
467
+
468
469
class CompressedTensorsLinearMethod(LinearMethodBase):
470
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+7
-8
@@ -7,13 +7,12 @@ from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
7
from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
8
CompressedTensorsWNA16)
9
10
+from .compressed_tensors_24 import CompressedTensors24 # isort: skip
11
+
12
__all__ = [
11
- "CompressedTensorsScheme",
12
- "CompressedTensorsWNA16",
13
- "CompressedTensorsW8A16Fp8",
14
- "CompressedTensorsW4A16Sparse24",
15
- "CompressedTensorsW8A8Int8",
16
- "CompressedTensorsW8A8Fp8",
17
- "WNA16_SUPPORTED_BITS",
18
- "W4A16SPARSE24_SUPPORTED_BITS",
13
+ "CompressedTensorsScheme", "CompressedTensorsWNA16",
14
+ "CompressedTensorsW8A16Fp8", "CompressedTensorsW4A16Sparse24",
15
+ "CompressedTensorsW8A8Int8", "CompressedTensorsW8A8Fp8",
16
+ "WNA16_SUPPORTED_BITS", "W4A16SPARSE24_SUPPORTED_BITS",
17
+ "CompressedTensors24"
18
]
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
new
+203
@@ -0,0 +1,203 @@
1
+from typing import Callable, List, Optional
2
+
3
+import torch
4
+from compressed_tensors.quantization import (QuantizationArgs,
5
+ QuantizationStrategy,
6
+ QuantizationType)
7
+
8
+from vllm import _custom_ops as ops
9
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
+ CompressedTensorsScheme)
11
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
+ convert_to_channelwise)
13
+from vllm.model_executor.parameter import (BasevLLMParameter,
14
+ ChannelQuantScaleParameter,
15
+ ModelWeightParameter,
16
+ PerTensorScaleParameter)
17
+
18
+__all__ = ["CompressedTensors24"]
19
+
20
+
21
+class CompressedTensors24(CompressedTensorsScheme):
22
+
23
+ def __init__(self,
24
+ quantized: bool = False,
25
+ weight_quant: Optional[QuantizationArgs] = None,
26
+ input_quant: Optional[QuantizationArgs] = None):
27
+
28
+ self.quantized = quantized
29
+ self.weight_quant = weight_quant
30
+ self.input_quant = input_quant
31
+
32
+ @classmethod
33
+ def get_min_capability(cls) -> int:
34
+ # Only cutlass 3.x kernels are implemented so far
35
+ return 90
36
+
37
+ def create_weights(self, layer: torch.nn.Module, input_size: int,
38
+ output_partition_sizes: List[int],
39
+ input_size_per_partition: int,
40
+ params_dtype: torch.dtype, weight_loader: Callable,
41
+ **kwargs):
42
+
43
+ self.output_dtype = params_dtype
44
+ layer.logical_widths = output_partition_sizes
45
+ self.weights_dtype: torch.dtype = self._get_params_dtype(params_dtype)
46
+
47
+ # parameter to store uncompressed weight
48
+ weight = ModelWeightParameter(data=torch.empty(
49
+ sum(output_partition_sizes),
50
+ input_size_per_partition,
51
+ dtype=self.weights_dtype),
52
+ input_dim=1,
53
+ output_dim=0,
54
+ weight_loader=weight_loader)
55
+
56
+ # Check if quantized, not just 2:4 Sparse
57
+ if self.quantized:
58
+ if (self.weight_quant and self.weight_quant.strategy
59
+ == QuantizationStrategy.CHANNEL.value):
60
+ weight_scale = ChannelQuantScaleParameter(
61
+ data=torch.empty((sum(output_partition_sizes), 1),
62
+ dtype=torch.float32),
63
+ output_dim=0,
64
+ weight_loader=weight_loader)
65
+ else:
66
+ assert (self.weight_quant and self.weight_quant.strategy
67
+ == QuantizationStrategy.TENSOR.value)
68
+ weight_scale = PerTensorScaleParameter(
69
+ data=torch.empty(len(output_partition_sizes),
70
+ dtype=torch.float32),
71
+ weight_loader=weight_loader)
72
+
73
+ layer.register_parameter("weight_scale", weight_scale)
74
+
75
+ # input quant will be non-none
76
+ if self.input_quant and not self.input_quant.dynamic:
77
+ # register input quant scale
78
+ assert (self.input_quant.strategy ==
79
+ QuantizationStrategy.TENSOR.value)
80
+ input_scale = BasevLLMParameter(data=torch.empty(
81
+ 1, dtype=torch.float32),
82
+ weight_loader=weight_loader)
83
+
84
+ layer.register_parameter("input_scale", input_scale)
85
+
86
+ else:
87
+ # for sparse-only, pass in 1 for weight/input scales
88
+ weight_scale = torch.nn.Parameter(data=torch.ones(
89
+ 1, dtype=torch.float32),
90
+ requires_grad=False)
91
+ input_scale = torch.nn.Parameter(data=torch.ones(
92
+ 1, dtype=torch.float32),
93
+ requires_grad=False)
94
+ layer.register_parameter("input_scale", input_scale)
95
+ layer.register_parameter("weight_scale", weight_scale)
96
+
97
+ layer.register_parameter("weight", weight)
98
+
99
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
100
+ """
101
+ Compress weights after loading. Store compressed weight and meta
102
+ tensor
103
+
104
+ :post-condition: layer.w_compressed and layer.meta are
105
+ set to the compressed weight and meta tensor in the
106
+ format expected by the Cutlass kernels
107
+ :param layer: The layer with the weights to be processed
108
+
109
+ """
110
+ # torch.compile workaround
111
+ if hasattr(layer, "input_scale"):
112
+ layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
113
+ requires_grad=False)
114
+
115
+ if self.weight_quant:
116
+ if self.weight_quant.strategy == QuantizationStrategy.TENSOR.value:
117
+ layer.weight_scale = torch.nn.Parameter(convert_to_channelwise(
118
+ weight_scale=layer.weight_scale,
119
+ logical_widths=layer.logical_widths),
120
+ requires_grad=False)
121
+ else:
122
+ # torch.compile workaround
123
+ layer.weight_scale = torch.nn.Parameter(
124
+ layer.weight_scale.data, requires_grad=False)
125
+
126
+ w_compressed, meta = ops.cutlass_sparse_compress(layer.weight.data)
127
+ layer.weight = torch.nn.Parameter(w_compressed, requires_grad=False)
128
+ layer.meta = torch.nn.Parameter(meta, requires_grad=False)
129
+
130
+ def apply_weights(self,
131
+ layer: torch.nn.Module,
132
+ x: torch.Tensor,
133
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
134
+ """
135
+ Returns the output tensor for the layer with 2:4
136
+ sparse compressed weights, given the input tensor
137
+ and bias
138
+
139
+ :param layer: The layer with 2:4 sparse compressed
140
+ weights to be used for the computation
141
+ :param x: The input tensor to the layer
142
+ :param bias: The bias to be added to the output tensor
143
+ :return: The output tensor of the layer
144
+ """
145
+ if self.quantized:
146
+ scale = None
147
+ if hasattr(layer, "input_scale"):
148
+ scale = layer.input_scale
149
+
150
+ if self.weights_dtype == torch.int8:
151
+ ops_output = ops.scaled_int8_quant(x, scale=scale)
152
+ q_input = ops_output[0]
153
+ input_scale = ops_output[1]
154
+ else:
155
+ assert self.weights_dtype == torch.float8_e4m3fn
156
+ if scale is not None:
157
+ q_input, input_scale = ops.scaled_fp8_quant(x, scale=scale)
158
+ else:
159
+ q_input, input_scale = ops.scaled_fp8_quant(
160
+ x, use_per_token_if_dynamic=True)
161
+
162
+ else:
163
+ # Not quantized, nothing to do with the input_scales, use as is
164
+ input_scale = layer.input_scale
165
+ q_input = x
166
+
167
+ out = ops.cutlass_scaled_sparse_mm(a=q_input,
168
+ bt_nzs=layer.weight,
169
+ bt_meta=layer.meta,
170
+ scale_a=input_scale,
171
+ scale_b=layer.weight_scale,
172
+ out_dtype=self.output_dtype,
173
+ bias=bias)
174
+ assert out.is_contiguous()
175
+ return out
176
+
177
+ def _get_params_dtype(self, params_dtype: torch.dtype) -> torch.dtype:
178
+ if not self.quantized:
179
+ return params_dtype
180
+
181
+ assert self.weight_quant is not None
182
+ assert self.input_quant is not None
183
+
184
+ is_8_bits = self.weight_quant.num_bits == self.input_quant.num_bits == 8
185
+
186
+ if not is_8_bits:
187
+ raise ValueError("Cutlass only supports 8-bit quantization")
188
+
189
+ if (self.weight_quant.type == QuantizationType.FLOAT
190
+ and self.input_quant.type == QuantizationType.FLOAT):
191
+ return torch.float8_e4m3fn
192
+
193
+ if (self.weight_quant.type == QuantizationType.INT
194
+ and self.input_quant.type == QuantizationType.INT):
195
+ return torch.int8
196
+
197
+ raise ValueError("Quantization type not supported by Cutlass")
198
+
199
+
200
+def check_24(tensor):
201
+ new_tensor = tensor.view(-1, 4)
202
+ zero_counts = (new_tensor == 0).sum(dim=1)
203
+ return (zero_counts >= 2).all().item()