@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 60508ffda9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support (#10995)

Co-authored-by: Faraz Shahsavan <faraz.shahsavan@gmail.com> Co-authored-by: ilmarkov <markovilya197@gmail.com> Co-authored-by: Rahul Tuli <rahul@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>

Dipika Sikka committed Dec 18, 2024 at 09:57 UTC 60508ffda91c22e4cde3b18f149d222211db8886
30 files changed +2365 -117
CMakeLists.txt
+16 -10
@@ -206,7 +206,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
206 SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
207
208 # Set CUTLASS_REVISION manually -- its revision detection doesn't work in this case.
209 - set(CUTLASS_REVISION "v3.5.1" CACHE STRING "CUTLASS revision to use")
209 + set(CUTLASS_REVISION "v3.6.0" CACHE STRING "CUTLASS revision to use")
210
211 # Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
212 if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
@@ -223,13 +223,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
223 FetchContent_Declare(
224 cutlass
225 GIT_REPOSITORY https://github.com/nvidia/cutlass.git
226 - GIT_TAG v3.5.1
226 + GIT_TAG 8aa95dbb888be6d81c6fbf7169718c5244b53227
227 GIT_PROGRESS TRUE
228
229 # Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
230 # Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
231 # So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
232 - GIT_SHALLOW TRUE
232 + GIT_SHALLOW FALSE
233 )
234 endif()
235 FetchContent_MakeAvailable(cutlass)
@@ -241,7 +241,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
241 "csrc/quantization/awq/gemm_kernels.cu"
242 "csrc/custom_all_reduce.cu"
243 "csrc/permute_cols.cu"
244 - "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu")
244 + "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
245 + "csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
246 + "csrc/sparse/cutlass/sparse_compressor_entry.cu"
247 + "csrc/cutlass_extensions/common.cpp")
248
249 set_gencode_flags_for_srcs(
250 SRCS "${VLLM_EXT_SRC}"
@@ -271,11 +274,14 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
274 endif()
275
276 #
274 - # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
277 + # The cutlass_scaled_mm cutlass_scaled_sparse_mm, and cutlass_compressor kernels
278 + # For Hopper (c3x, i.e. CUTLASS 3.x) require
279 # CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
280 cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
281 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
278 - set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
282 + set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
283 + "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
284 + "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
285 set_gencode_flags_for_srcs(
286 SRCS "${SRCS}"
287 CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
@@ -284,12 +290,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
290 message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
291 else()
292 if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
287 - message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
293 + message(STATUS "Not building cutlass_c3x kernels as CUDA Compiler version is "
294 "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
289 - "later if you intend on running FP8 quantized models on "
295 + "later if you intend on running FP8 sparse or quantized models on "
296 "Hopper.")
297 else()
292 - message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
298 + message(STATUS "Not building cutlass_c3x as no compatible archs found "
299 "in CUDA target architectures")
300 endif()
301
@@ -404,7 +410,7 @@ define_gpu_extension_target(
410 SOURCES ${VLLM_EXT_SRC}
411 COMPILE_FLAGS ${VLLM_GPU_FLAGS}
412 ARCHITECTURES ${VLLM_GPU_ARCHES}
407 - INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
413 + INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
414 USE_SABI 3
415 WITH_SOABI)
416
benchmarks/cutlass_benchmarks/sparse_benchmarks.py new
+384
@@ -0,0 +1,384 @@
1 +import argparse
2 +import copy
3 +import itertools
4 +import pickle as pkl
5 +import time
6 +from typing import Callable, Iterable, List, Tuple
7 +
8 +import torch
9 +import torch.utils.benchmark as TBenchmark
10 +from torch.utils.benchmark import Measurement as TMeasurement
11 +from utils import make_rand_sparse_tensors
12 +from weight_shapes import WEIGHT_SHAPES
13 +
14 +from vllm import _custom_ops as ops
15 +from vllm.utils import FlexibleArgumentParser
16 +
17 +DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
18 +DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
19 +DEFAULT_TP_SIZES = [1]
20 +
21 +
22 +# bench
23 +def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
24 + **kwargs) -> TMeasurement:
25 + min_run_time = 1
26 +
27 + globals = {
28 + "args": args,
29 + "kwargs": kwargs,
30 + "fn": fn,
31 + }
32 + return TBenchmark.Timer(
33 + stmt="fn(*args, **kwargs)",
34 + globals=globals,
35 + label=label,
36 + sub_label=sub_label,
37 + description=description,
38 + ).blocked_autorange(min_run_time=min_run_time)
39 +
40 +
41 +def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
42 + sub_label: str) -> Iterable[TMeasurement]:
43 + assert dtype == torch.int8
44 + b_compressed, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
45 + scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
46 + scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
47 + bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
48 +
49 + out = ops.cutlass_scaled_sparse_mm(a, b_compressed, e, scale_a, scale_b,
50 + torch.bfloat16)
51 + out_ref = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16)
52 +
53 + if not torch.allclose(out, out_ref):
54 + print("Incorrect results")
55 + print(out)
56 + print(out_ref)
57 + else:
58 + print("Correct results")
59 +
60 + timers = []
61 + # pytorch impl - bfloat16
62 + timers.append(
63 + bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
64 + torch.mm, a.to(dtype=torch.bfloat16),
65 + b.to(dtype=torch.bfloat16)))
66 +
67 + # pytorch impl - float16
68 + timers.append(
69 + bench_fn(label, sub_label,
70 + "pytorch_fp16_fp16_fp16_matmul-no-scales", torch.mm,
71 + a.to(dtype=torch.float16), b.to(dtype=torch.float16)))
72 +
73 + # cutlass impl
74 + timers.append(
75 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm",
76 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
77 + torch.bfloat16))
78 +
79 + # cutlass with bias
80 + timers.append(
81 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_bias",
82 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
83 + bias))
84 +
85 + # cutlass sparse impl
86 + timers.append(
87 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_sparse_mm",
88 + ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
89 + scale_b, torch.bfloat16))
90 +
91 + # cutlass sparse with bias
92 + timers.append(
93 + bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_sparse_mm_bias",
94 + ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
95 + scale_b, torch.bfloat16, bias))
96 +
97 + return timers
98 +
99 +
100 +def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
101 + sub_label: str) -> Iterable[TMeasurement]:
102 + assert dtype == torch.float8_e4m3fn
103 + b_compressed, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n,
104 + k)
105 + scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
106 + scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
107 + bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
108 +
109 + out = ops.cutlass_scaled_sparse_mm(a, b_compressed, e, scale_a, scale_b,
110 + torch.bfloat16)
111 + out_ref = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16)
112 +
113 + if not torch.allclose(out, out_ref):
114 + print("Incorrect results")
115 + print(out)
116 + print(out_ref)
117 + else:
118 + print("Correct results")
119 +
120 + timers = []
121 +
122 + # pytorch impl w. bf16
123 + timers.append(
124 + bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
125 + torch.mm, a.to(dtype=torch.bfloat16, device="cuda"),
126 + b.to(dtype=torch.bfloat16, device="cuda")))
127 +
128 + # pytorch impl: bf16 output, without fp8 fast accum
129 + timers.append(
130 + bench_fn(label,
131 + sub_label,
132 + "pytorch_fp8_fp8_bf16_scaled_mm",
133 + torch._scaled_mm,
134 + a,
135 + b,
136 + scale_a=scale_a,
137 + scale_b=scale_b,
138 + out_dtype=torch.bfloat16))
139 +
140 + # pytorch impl: bf16 output, with fp8 fast accum
141 + timers.append(
142 + bench_fn(label,
143 + sub_label,
144 + "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
145 + torch._scaled_mm,
146 + a,
147 + b,
148 + scale_a=scale_a,
149 + scale_b=scale_b,
150 + out_dtype=torch.bfloat16,
151 + use_fast_accum=True))
152 +
153 + # pytorch impl: fp16 output, without fp8 fast accum
154 + timers.append(
155 + bench_fn(label,
156 + sub_label,
157 + "pytorch_fp8_fp8_fp16_scaled_mm",
158 + torch._scaled_mm,
159 + a,
160 + b,
161 + scale_a=scale_a,
162 + scale_b=scale_b,
163 + out_dtype=torch.float16))
164 +
165 + # pytorch impl: fp16 output, with fp8 fast accum
166 + timers.append(
167 + bench_fn(label,
168 + sub_label,
169 + "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
170 + torch._scaled_mm,
171 + a,
172 + b,
173 + scale_a=scale_a,
174 + scale_b=scale_b,
175 + out_dtype=torch.float16,
176 + use_fast_accum=True))
177 +
178 + # cutlass impl: bf16 output
179 + timers.append(
180 + bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm",
181 + ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
182 + torch.bfloat16))
183 +
184 + # cutlass impl: bf16 output
185 + timers.append(
186 + bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_sparse_mm",
187 + ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
188 + scale_b, torch.bfloat16))
189 +
190 + # cutlass impl: fp16 output
191 + timers.append(
192 + bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_sparse_mm",
193 + ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
194 + scale_b, torch.float16))
195 +
196 + # cutlass impl: bf16 output, with bias
197 + timers.append(
198 + bench_fn(label, sub_label,
199 + "cutlass_fp8_fp8_bf16_scaled_sparse_mm_bias",
200 + ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
201 + scale_b, torch.bfloat16, bias))
202 +
203 + # cutlass impl: fp16 output, with bias
204 + timers.append(
205 + bench_fn(label, sub_label,
206 + "cutlass_fp8_fp8_fp16_scaled_sparse_mm_bias",
207 + ops.cutlass_scaled_sparse_mm, a, b_compressed, e, scale_a,
208 + scale_b, torch.float16, bias.to(dtype=torch.float16)))
209 +
210 + return timers
211 +
212 +
213 +def bench(dtype: torch.dtype, m: int, k: int, n: int, label: str,
214 + sub_label: str) -> Iterable[TMeasurement]:
215 + if dtype == torch.int8:
216 + return bench_int8(dtype, m, k, n, label, sub_label)
217 + if dtype == torch.float8_e4m3fn:
218 + return bench_fp8(dtype, m, k, n, label, sub_label)
219 + raise ValueError("unsupported type")
220 +
221 +
222 +# runner
223 +def print_timers(timers: Iterable[TMeasurement]):
224 + compare = TBenchmark.Compare(timers)
225 + compare.print()
226 +
227 +
228 +def run(dtype: torch.dtype,
229 + MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
230 + results = []
231 + for m, k, n in MKNs:
232 + timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
233 + f"MKN=({m}x{k}x{n})")
234 + print_timers(timers)
235 + results.extend(timers)
236 +
237 + return results
238 +
239 +
240 +# output makers
241 +def make_output(data: Iterable[TMeasurement],
242 + MKNs: Iterable[Tuple[int, int, int]],
243 + base_description: str,
244 + timestamp=None):
245 + print(f"== All Results {base_description} ====")
246 + print_timers(data)
247 +
248 + # pickle all the results
249 + timestamp = int(time.time()) if timestamp is None else timestamp
250 + with open(f"{base_description}-{timestamp}.pkl", "wb") as f:
251 + pkl.dump(data, f)
252 +
253 +
254 +# argparse runners
255 +
256 +
257 +def run_square_bench(args):
258 + dim_sizes = list(
259 + range(args.dim_start, args.dim_end + 1, args.dim_increment))
260 + MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
261 + data = run(args.dtype, MKNs)
262 +
263 + make_output(data, MKNs, f"square_bench-{args.dtype}")
264 +
265 +
266 +def run_range_bench(args):
267 + dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
268 + n = len(dim_sizes)
269 + Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
270 + Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
271 + Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
272 + MKNs = list(zip(Ms, Ks, Ns))
273 + data = run(args.dtype, MKNs)
274 +
275 + make_output(data, MKNs, f"range_bench-{args.dtype}")
276 +
277 +
278 +def run_model_bench(args):
279 + print("Benchmarking models:")
280 + for i, model in enumerate(args.models):
281 + print(f"[{i}] {model}")
282 +
283 + def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:
284 + KNs = []
285 + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):
286 + KN[tp_split_dim] = KN[tp_split_dim] // tp_size
287 + KNs.append(KN)
288 + return KNs
289 +
290 + model_bench_data = []
291 + models_tps = list(itertools.product(args.models, args.tp_sizes))
292 + for model, tp_size in models_tps:
293 + Ms = args.batch_sizes
294 + KNs = model_shapes(model, tp_size)
295 + MKNs = []
296 + for m in Ms:
297 + for k, n in KNs:
298 + MKNs.append((m, k, n))
299 +
300 + data = run(args.dtype, MKNs)
301 + model_bench_data.append(data)
302 +
303 + # Print all results
304 + for data, model_tp in zip(model_bench_data, models_tps):
305 + model, tp_size = model_tp
306 + print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
307 + print_timers(data)
308 +
309 + timestamp = int(time.time())
310 +
311 + all_data = []
312 + for d in model_bench_data:
313 + all_data.extend(d)
314 + # pickle all data
315 + with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
316 + pkl.dump(all_data, f)
317 +
318 +
319 +if __name__ == '__main__':
320 +
321 + def to_torch_dtype(dt):
322 + if dt == "int8":
323 + return torch.int8
324 + if dt == "fp8":
325 + return torch.float8_e4m3fn
326 + raise ValueError("unsupported dtype")
327 +
328 + parser = FlexibleArgumentParser(
329 + description="""
330 +Benchmark Cutlass GEMM.
331 +
332 + To run square GEMMs:
333 + python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 square_bench --dim-start 128 --dim-end 512 --dim-increment 64
334 +
335 + To run constant N and K and sweep M:
336 + python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384
337 +
338 + To run dimensions from a model:
339 + python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1
340 +
341 + Output:
342 + - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs.
343 + """, # noqa: E501
344 + formatter_class=argparse.RawTextHelpFormatter)
345 +
346 + parser.add_argument("--dtype",
347 + type=to_torch_dtype,
348 + required=True,
349 + help="Available options are ['int8', 'fp8']")
350 + subparsers = parser.add_subparsers(dest="cmd")
351 +
352 + square_parser = subparsers.add_parser("square_bench")
353 + square_parser.add_argument("--dim-start", type=int, required=True)
354 + square_parser.add_argument("--dim-end", type=int, required=True)
355 + square_parser.add_argument("--dim-increment", type=int, required=True)
356 + square_parser.set_defaults(func=run_square_bench)
357 +
358 + range_parser = subparsers.add_parser("range_bench")
359 + range_parser.add_argument("--dim-start", type=int, required=True)
360 + range_parser.add_argument("--dim-end", type=int, required=True)
361 + range_parser.add_argument("--dim-increment", type=int, required=True)
362 + range_parser.add_argument("--m-constant", type=int, default=None)
363 + range_parser.add_argument("--n-constant", type=int, default=None)
364 + range_parser.add_argument("--k-constant", type=int, default=None)
365 + range_parser.set_defaults(func=run_range_bench)
366 +
367 + model_parser = subparsers.add_parser("model_bench")
368 + model_parser.add_argument("--models",
369 + nargs="+",
370 + type=str,
371 + default=DEFAULT_MODELS,
372 + choices=WEIGHT_SHAPES.keys())
373 + model_parser.add_argument("--tp-sizes",
374 + nargs="+",
375 + type=int,
376 + default=DEFAULT_TP_SIZES)
377 + model_parser.add_argument("--batch-sizes",
378 + nargs="+",
379 + type=int,
380 + default=DEFAULT_BATCH_SIZES)
381 + model_parser.set_defaults(func=run_model_bench)
382 +
383 + args = parser.parse_args()
384 + args.func(args)
benchmarks/cutlass_benchmarks/utils.py new
+96
@@ -0,0 +1,96 @@
1 +# Cutlass bench utils
2 +from typing import Iterable, Tuple
3 +
4 +import torch
5 +
6 +import vllm._custom_ops as ops
7 +
8 +
9 +def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
10 + finfo = torch.finfo(torch.float8_e4m3fn)
11 + return torch.round(tensor.clamp(
12 + min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
13 +
14 +
15 +def to_int8(tensor: torch.Tensor) -> torch.Tensor:
16 + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
17 +
18 +
19 +def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
20 + return tensor.to(dtype=torch.bfloat16)
21 +
22 +
23 +def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
24 + return tensor.to(dtype=torch.float16)
25 +
26 +
27 +def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
28 + k: int) -> Tuple[torch.Tensor, torch.Tensor]:
29 + a = torch.randn((m, k), device='cuda') * 5
30 + b = torch.randn((n, k), device='cuda').t() * 5
31 +
32 + if dtype == torch.int8:
33 + return to_int8(a), to_int8(b)
34 + if dtype == torch.float8_e4m3fn:
35 + return to_fp8(a), to_fp8(b)
36 +
37 + raise ValueError("unsupported dtype")
38 +
39 +
40 +def prune_to_2_4(tensor):
41 + # Reshape tensor to [N, 4] where N is number of groups of 4
42 + original_shape = tensor.shape
43 + reshaped = tensor.reshape(-1, 4)
44 +
45 + # Get indices of top 2 absolute values in each group of 4
46 + _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
47 +
48 + # Create binary mask
49 + mask = torch.zeros_like(reshaped)
50 + mask.scatter_(dim=1,
51 + index=indices,
52 + src=torch.ones_like(indices, dtype=mask.dtype))
53 +
54 + # Apply mask and reshape back
55 + pruned = reshaped * mask
56 +
57 + # Turn all -0.0 to 0.0
58 + pruned[pruned == -0.0] = 0.0
59 +
60 + return pruned.reshape(original_shape)
61 +
62 +
63 +def make_rand_sparse_tensors(dtype: torch.dtype, m: int, n: int,
64 + k: int) -> Tuple[torch.Tensor, torch.Tensor]:
65 + a = torch.randn((m, k), device='cuda') * 5
66 + b = torch.randn((n, k), device='cuda').t() * 5
67 +
68 + b = prune_to_2_4(b.t()).t()
69 +
70 + if dtype == torch.int8:
71 + a, b = to_int8(a), to_int8(b)
72 + elif dtype == torch.float8_e4m3fn:
73 + a, b = to_fp8(a), to_fp8(b)
74 + elif dtype == torch.float16:
75 + a, b = to_fp16(a), to_fp16(b)
76 + elif dtype == torch.bfloat16:
77 + a, b = to_bf16(a), to_bf16(b)
78 + else:
79 + raise ValueError("unsupported dtype")
80 +
81 + b_compressed, e = ops.cutlass_sparse_compress(b.t())
82 +
83 + # Compressed B, Metadata, Original A, B
84 + return b_compressed, e, a, b
85 +
86 +
87 +def make_n_rand_sparse_tensors(num_tensors: int, dtype: torch.dtype,
88 + m: int, n: int, k: int) -> \
89 + Tuple[Iterable[torch.Tensor], Iterable[torch.Tensor]]:
90 + ABs = []
91 + for _ in range(num_tensors):
92 + b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)
93 + if b_comp is not None:
94 + ABs.append(make_rand_sparse_tensors(dtype, m, n, k))
95 + BComps, Es, As, Bs = zip(*ABs)
96 + return list(BComps), list(Es), list(As), list(Bs)
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2 -26
@@ -8,6 +8,7 @@ from typing import Callable, Iterable, List, Tuple
8 import torch
9 import torch.utils.benchmark as TBenchmark
10 from torch.utils.benchmark import Measurement as TMeasurement
11 +from utils import make_rand_tensors
12 from weight_shapes import WEIGHT_SHAPES
13
14 from vllm import _custom_ops as ops
@@ -17,31 +18,6 @@ DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
18 DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
19 DEFAULT_TP_SIZES = [1]
20
20 -# helpers
21 -
22 -
23 -def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
24 - finfo = torch.finfo(torch.float8_e4m3fn)
25 - return torch.round(tensor.clamp(
26 - min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
27 -
28 -
29 -def to_int8(tensor: torch.Tensor) -> torch.Tensor:
30 - return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
31 -
32 -
33 -def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
34 - k: int) -> Tuple[torch.Tensor, torch.Tensor]:
35 - a = torch.randn((m, k), device='cuda') * 5
36 - b = torch.randn((n, k), device='cuda').t() * 5
37 -
38 - if dtype == torch.int8:
39 - return to_int8(a), to_int8(b)
40 - if dtype == torch.float8_e4m3fn:
41 - return to_fp8(a), to_fp8(b)
42 -
43 - raise ValueError("unsupported dtype")
44 -
21
22 # bench
23 def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
@@ -386,4 +362,4 @@ Benchmark Cutlass GEMM.
362 model_parser.set_defaults(func=run_model_bench)
363
364 args = parser.parse_args()
389 - args.func(args)
365 + args.func(args)
\ No newline at end of file
benchmarks/cutlass_benchmarks/weight_shapes.py
+1 -1
@@ -40,4 +40,4 @@ WEIGHT_SHAPES = {
40 ([8192, 57344], 1),
41 ([28672, 8192], 0),
42 ],
43 -}
43 +}
\ No newline at end of file
csrc/core/math.hpp new
+7
@@ -0,0 +1,7 @@
1 +#include <climits>
2 +#include <iostream>
3 +
4 +inline uint32_t next_pow_2(uint32_t const num) {
5 + if (num <= 1) return num;
6 + return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
7 +}
\ No newline at end of file
csrc/cutlass_extensions/common.cpp new
+11
@@ -0,0 +1,11 @@
1 +#include "cutlass_extensions/common.hpp"
2 +
3 +int32_t get_sm_version_num() {
4 + int32_t major_capability, minor_capability;
5 + cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
6 + 0);
7 + cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
8 + 0);
9 + int32_t version_num = major_capability * 10 + minor_capability;
10 + return version_num;
11 +}
\ No newline at end of file
csrc/cutlass_extensions/common.hpp new
+35
@@ -0,0 +1,35 @@
1 +#pragma once
2 +
3 +#include "cutlass/cutlass.h"
4 +#include <climits>
5 +#include "cuda_runtime.h"
6 +#include <iostream>
7 +
8 +/**
9 + * Helper function for checking CUTLASS errors
10 + */
11 +#define CUTLASS_CHECK(status) \
12 + { \
13 + cutlass::Status error = status; \
14 + TORCH_CHECK(error == cutlass::Status::kSuccess, \
15 + cutlassGetStatusString(error)); \
16 + }
17 +
18 +/**
19 + * Panic wrapper for unwinding CUDA runtime errors
20 + */
21 +#define CUDA_CHECK(status) \
22 + { \
23 + cudaError_t error = status; \
24 + TORCH_CHECK(error == cudaSuccess, cudaGetErrorString(error)); \
25 + }
26 +
27 +inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
28 + int max_shared_mem_per_block_opt_in = 0;
29 + cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
30 + cudaDevAttrMaxSharedMemoryPerBlockOptin,
31 + device);
32 + return max_shared_mem_per_block_opt_in;
33 +}
34 +
35 +int32_t get_sm_version_num();
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+2 -2
@@ -36,13 +36,13 @@ struct ScaledEpilogueBase {
36 // Don't want to support nullptr by default
37 template <typename T, bool EnableNullPtr = false>
38 using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
39 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
39 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T, T,
40 Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
41
42 // Don't want to support nullptr by default
43 template <typename T, bool EnableNullPtr = false>
44 using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
45 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
45 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T, T,
46 Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
47
48 // This utility function constructs the arguments for the load descriptors
csrc/ops.h
+9
@@ -162,6 +162,15 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
162 torch::Tensor const& azp_adj,
163 c10::optional<torch::Tensor> const& azp,
164 c10::optional<torch::Tensor> const& bias);
165 +
166 +void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
167 + torch::Tensor const& b, torch::Tensor const& e,
168 + torch::Tensor const& a_scales,
169 + torch::Tensor const& b_scales,
170 + c10::optional<torch::Tensor> const& bias);
171 +
172 +bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
173 + torch::Tensor& e, torch::Tensor const& a);
174 #endif
175
176 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/cutlass_w8a8/common.hpp deleted
-27
@@ -1,27 +0,0 @@
1 -#pragma once
2 -
3 -#include "cutlass/cutlass.h"
4 -#include <climits>
5 -
6 -/**
7 - * Helper function for checking CUTLASS errors
8 - */
9 -#define CUTLASS_CHECK(status) \
10 - { \
11 - TORCH_CHECK(status == cutlass::Status::kSuccess, \
12 - cutlassGetStatusString(status)) \
13 - }
14 -
15 -inline uint32_t next_pow_2(uint32_t const num) {
16 - if (num <= 1) return num;
17 - return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
18 -}
19 -
20 -inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
21 - int max_shared_mem_per_block_opt_in = 0;
22 - cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
23 - cudaDevAttrMaxSharedMemoryPerBlockOptin,
24 - device);
25 - return max_shared_mem_per_block_opt_in;
26 -}
27 -
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+2 -1
@@ -21,7 +21,8 @@
21 #include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
22 #include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
23
24 -#include "common.hpp"
24 +#include "core/math.hpp"
25 +#include "cutlass_extensions/common.hpp"
26 // clang-format on
27
28 using namespace cute;
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+2 -1
@@ -24,7 +24,8 @@
24 #include "cutlass/gemm/collective/collective_builder.hpp"
25
26 #include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
27 -#include "common.hpp"
27 +#include "core/math.hpp"
28 +#include "cutlass_extensions/common.hpp"
29 // clang-format on
30
31 using namespace cute;
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+2 -10
@@ -3,6 +3,8 @@
3 #include <c10/cuda/CUDAGuard.h>
4 #include <torch/all.h>
5
6 +#include "cutlass_extensions/common.hpp"
7 +
8 void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
9 torch::Tensor const& b,
10 torch::Tensor const& a_scales,
@@ -79,16 +81,6 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
81 return false;
82 }
83
82 -int32_t get_sm_version_num() {
83 - int32_t major_capability, minor_capability;
84 - cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
85 - 0);
86 - cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
87 - 0);
88 - int32_t version_num = major_capability * 10 + minor_capability;
89 - return version_num;
90 -}
91 -
84 void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
85 torch::Tensor const& b, torch::Tensor const& a_scales,
86 torch::Tensor const& b_scales,
csrc/sparse/cutlass/sparse_compressor_c3x.cu new
+163
@@ -0,0 +1,163 @@
1 +// clang-format will break include orders
2 +// clang-format off
3 +#include <cudaTypedefs.h>
4 +
5 +#include "sparse_scaled_mm_c3x.cuh"
6 +
7 +#include "cutlass/numeric_conversion.h"
8 +#include "cutlass/transform/device/transform_universal_adapter.hpp"
9 +#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
10 +#include "cutlass/epilogue/collective/default_epilogue.hpp"
11 +
12 +#include "cutlass/util/host_tensor.h"
13 +#include "cutlass/util/packed_stride.hpp"
14 +// clang-format on
15 +
16 +using namespace cute;
17 +using namespace vllm;
18 +
19 +/// Make A structured sparse by replacing elements with 0 and compress it
20 +template <typename ElementA_, typename ElementAcc_>
21 +bool cutlass_sparse_compress(torch::Tensor& a_nzs, torch::Tensor& a_meta,
22 + torch::Tensor const& a) {
23 + // Checks for conformality
24 + TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
25 + a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
26 + TORCH_CHECK(a.dim() == 2)
27 + // Check for strides and alignment
28 + TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
29 + TORCH_CHECK(a.stride(1) == 1)
30 +
31 + int m = a.size(0);
32 + int k = a.size(1);
33 +
34 + // Sparse kernel setup; this kernel is not used for matmul,
35 + // but just for setting up the compressor utility
36 + // A matrix configuration
37 + using ElementA = ElementA_;
38 + using LayoutTagA = cutlass::layout::RowMajor;
39 + constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
40 + // B matrix configuration
41 + using ElementB = ElementA;
42 + using LayoutTagB = cutlass::layout::ColumnMajor;
43 + constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
44 + // C/D matrix configuration
45 + using ElementC = float;
46 + using LayoutTagC = cutlass::layout::ColumnMajor;
47 + constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
48 + // Core kernel configurations
49 + using ElementAccumulator = ElementAcc_;
50 + using TileShape = Shape<_128, _128, _128>;
51 + using TileShapeRef = Shape<_128, _128, _64>;
52 + using ClusterShape = Shape<_1, _2, _1>;
53 + using KernelSchedule = typename std::conditional<
54 + std::is_same_v<ElementA, cutlass::float_e4m3_t>,
55 + cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum,
56 + cutlass::gemm::KernelTmaWarpSpecialized>::type;
57 +
58 + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
59 + using ProblemShape = Shape<int, int, int, int>;
60 +
61 + using CollectiveEpilogue =
62 + typename cutlass::epilogue::collective::CollectiveBuilder<
63 + cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
64 + ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
65 + ElementAccumulator, ElementAccumulator, ElementC, LayoutTagC,
66 + AlignmentC, ElementC, LayoutTagC, AlignmentC,
67 + EpilogueSchedule>::CollectiveOp;
68 +
69 + using CollectiveMainloop =
70 + typename cutlass::gemm::collective::CollectiveBuilder<
71 + cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp, ElementA,
72 + LayoutTagA, AlignmentA, ElementB, LayoutTagB, AlignmentB,
73 + ElementAccumulator, TileShape, ClusterShape,
74 + cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
75 + sizeof(typename CollectiveEpilogue::SharedStorage))>,
76 + KernelSchedule>::CollectiveOp;
77 +
78 + using GemmKernel =
79 + cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop,
80 + CollectiveEpilogue>;
81 +
82 + using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
83 +
84 + using StrideA = cutlass::gemm::TagToStrideA_t<LayoutTagA>;
85 + using StrideE = StrideA;
86 +
87 + using StrideA = Stride<int64_t, Int<1>, int64_t>;
88 +
89 + // The n (=1) dimension does not matter for the compressor
90 + typename GemmKernel::ProblemShape prob_shape{m, 1, k, 1};
91 +
92 + using LayoutA = typename GemmKernel::CollectiveMainloop::LayoutA;
93 + using LayoutE = typename GemmKernel::CollectiveMainloop::LayoutE;
94 +
95 + using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
96 + using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
97 +
98 + // Offline compressor kernel
99 + using CompressorUtility =
100 + cutlass::transform::kernel::StructuredSparseCompressorUtility<
101 + ProblemShape, ElementA, LayoutTagA, SparseConfig>;
102 +
103 + using CompressorKernel =
104 + cutlass::transform::kernel::StructuredSparseCompressor<
105 + ProblemShape, ElementA, LayoutTagA, SparseConfig,
106 + cutlass::arch::Sm90>;
107 +
108 + using Compressor =
109 + cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
110 +
111 + auto [M, N, K, L] = prob_shape;
112 +
113 + StrideA stride_A;
114 + stride_A =
115 + cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
116 +
117 + CompressorUtility compressor_utility(prob_shape, stride_A);
118 +
119 + int ME = compressor_utility.get_metadata_m_physical();
120 + int KE = compressor_utility.get_metadata_k_physical();
121 + int KC = compressor_utility.get_tensorA_k_physical();
122 +
123 + auto a_ptr = static_cast<ElementA*>(a.data_ptr());
124 +
125 + auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
126 + auto a_meta_ptr = static_cast<typename Gemm::CollectiveMainloop::ElementE*>(
127 + a_meta.data_ptr());
128 +
129 + cutlass::KernelHardwareInfo hw_info;
130 + hw_info.device_id = 0;
131 + hw_info.sm_count =
132 + cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
133 + hw_info.device_id);
134 + typename Compressor::Arguments arguments{
135 + prob_shape, {a_ptr, stride_A, a_nzs_ptr, a_meta_ptr}, {hw_info}};
136 +
137 + Compressor compressor_op;
138 + size_t workspace_size = Compressor::get_workspace_size(arguments);
139 + cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
140 +
141 + CUTLASS_CHECK(compressor_op.can_implement(arguments));
142 + CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.get()));
143 + CUTLASS_CHECK(compressor_op.run());
144 + CUDA_CHECK(cudaDeviceSynchronize());
145 +
146 + return true;
147 +}
148 +
149 +bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
150 + torch::Tensor const& a) {
151 + if (a.dtype() == torch::kBFloat16) {
152 + return cutlass_sparse_compress<cutlass::bfloat16_t, float>(a_nzs, a_meta,
153 + a);
154 + } else if (a.dtype() == torch::kFloat16) {
155 + return cutlass_sparse_compress<cutlass::half_t, float>(a_nzs, a_meta, a);
156 + } else if (a.dtype() == torch::kFloat8_e4m3fn) {
157 + return cutlass_sparse_compress<cutlass::float_e4m3_t, float>(a_nzs, a_meta,
158 + a);
159 + } else if (a.dtype() == torch::kInt8) {
160 + return cutlass_sparse_compress<int8_t, int32_t>(a_nzs, a_meta, a);
161 + }
162 + return false;
163 +}
\ No newline at end of file
csrc/sparse/cutlass/sparse_compressor_entry.cu new
+42
@@ -0,0 +1,42 @@
1 +#include <cudaTypedefs.h>
2 +
3 +#include <c10/cuda/CUDAGuard.h>
4 +#include <torch/all.h>
5 +
6 +#include "cutlass_extensions/common.hpp"
7 +
8 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
9 +bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
10 + torch::Tensor const& a);
11 +#endif
12 +
13 +bool cutlass_sparse_compress_entry(torch::Tensor& a_nzs, torch::Tensor& a_meta,
14 + torch::Tensor const& a) {
15 + // Checks for conformality
16 + TORCH_CHECK(a.dim() == 2 && a_meta.dim() == 2 && a_nzs.dim() == 2);
17 + TORCH_CHECK(a.size(0) == a_nzs.size(0) && a.size(0) == a_meta.size(0) &&
18 + a_nzs.size(1) * 2 == a.size(1) &&
19 + a_meta.size(1) * 2 * 4 == a.size(1));
20 + // Considering elemsPerMetaElem = 8b / 2b_per_nz = 4
21 +
22 + // Check for strides and alignment
23 + TORCH_CHECK(a.stride(1) == 1 && a_nzs.stride(1) == 1 &&
24 + a_meta.stride(1) == 1); // Row-major
25 + TORCH_CHECK(a.stride(0) % 8 == 0); // 8 Byte Alignment for Compression
26 +
27 + at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
28 + int32_t version_num = get_sm_version_num();
29 +
30 + // Guard against compilation issues for sm90 kernels
31 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
32 + if (version_num >= 90) {
33 + return cutlass_sparse_compress_sm90(a_nzs, a_meta, a);
34 + }
35 +#endif
36 +
37 + TORCH_CHECK_NOT_IMPLEMENTED(
38 + false,
39 + "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
40 + "CUDA device capability: ",
41 + version_num);
42 +}
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu new
+303
@@ -0,0 +1,303 @@
1 +// clang-format will break include orders
2 +// clang-format off
3 +#include <cudaTypedefs.h>
4 +
5 +#if defined CUDA_VERSION && CUDA_VERSION >= 12000
6 +#include "sparse_scaled_mm_c3x.cuh"
7 +// clang-format on
8 +
9 +using namespace cute;
10 +using namespace vllm;
11 +
12 +template <typename InType, typename OutType,
13 + template <typename, typename, typename> typename Epilogue,
14 + typename... EpilogueArgs>
15 +void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
16 + torch::Tensor const& bt_nzs,
17 + torch::Tensor const& bt_meta,
18 + EpilogueArgs&&... args) {
19 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
20 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
21 + TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
22 + TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
23 +
24 + using Cutlass3xGemmDefault =
25 + typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
26 + using Cutlass3xGemmM64 =
27 + typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
28 + using Cutlass3xGemmM128 =
29 + typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
30 + using Cutlass3xGemmM256 =
31 + typename sm90_fp8_config_M256<InType, OutType, Epilogue>::Cutlass3xGemm;
32 + using Cutlass3xGemmM512 =
33 + typename sm90_fp8_config_M512<InType, OutType, Epilogue>::Cutlass3xGemm;
34 +
35 + using Cutlass3xGemm1 =
36 + typename sm90_fp8_config_1<InType, OutType, Epilogue>::Cutlass3xGemm;
37 + using Cutlass3xGemm2 =
38 + typename sm90_fp8_config_2<InType, OutType, Epilogue>::Cutlass3xGemm;
39 + using Cutlass3xGemm3 =
40 + typename sm90_fp8_config_3<InType, OutType, Epilogue>::Cutlass3xGemm;
41 + using Cutlass3xGemm4 =
42 + typename sm90_fp8_config_4<InType, OutType, Epilogue>::Cutlass3xGemm;
43 + using Cutlass3xGemm5 =
44 + typename sm90_fp8_config_5<InType, OutType, Epilogue>::Cutlass3xGemm;
45 + using Cutlass3xGemm6 =
46 + typename sm90_fp8_config_6<InType, OutType, Epilogue>::Cutlass3xGemm;
47 + using Cutlass3xGemm7 =
48 + typename sm90_fp8_config_7<InType, OutType, Epilogue>::Cutlass3xGemm;
49 + using Cutlass3xGemm8 =
50 + typename sm90_fp8_config_8<InType, OutType, Epilogue>::Cutlass3xGemm;
51 +
52 + uint32_t const n = bt_nzs.size(0);
53 + uint32_t const m = a.size(0); // Batch size
54 + uint32_t const mp2 =
55 + std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
56 +
57 + if (mp2 <= 64) {
58 + if (n == 28672) {
59 + return cutlass_sparse_gemm_caller<Cutlass3xGemm2>(
60 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
61 + } else if (n == 4096 || n == 6144) {
62 + return cutlass_sparse_gemm_caller<Cutlass3xGemm1>(
63 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
64 + }
65 + } else if (mp2 <= 128) {
66 + if (n == 4096) {
67 + return cutlass_sparse_gemm_caller<Cutlass3xGemm3>(
68 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
69 + } else if (n == 28672) {
70 + return cutlass_sparse_gemm_caller<Cutlass3xGemm5>(
71 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
72 + } else if (n == 6144) {
73 + return cutlass_sparse_gemm_caller<Cutlass3xGemm4>(
74 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
75 + }
76 + } else if (mp2 <= 256) {
77 + if (n == 4096) {
78 + return cutlass_sparse_gemm_caller<Cutlass3xGemm6>(
79 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
80 + } else if (n == 28672) {
81 + return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
82 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
83 + } else if (n == 6144) {
84 + return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
85 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
86 + }
87 + } else {
88 + if (n == 6144 || n == 28672) {
89 + return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
90 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
91 + } else if (n == 4096) {
92 + return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
93 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
94 + }
95 + }
96 +
97 + // Otherwise the default heuristic
98 + if (mp2 <= 64) {
99 + // n in [1, 64]
100 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
101 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
102 + } else if (mp2 <= 128) {
103 + // n in (64, 128]
104 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
105 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
106 + } else if (mp2 <= 256) {
107 + // n in (128, 256]
108 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM256>(
109 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
110 + } else {
111 + // n in (256, inf)
112 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM512>(
113 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
114 + }
115 +}
116 +
117 +template <typename InType, typename OutType,
118 + template <typename, typename, typename> typename Epilogue,
119 + typename... EpilogueArgs>
120 +void cutlass_gemm_sm90_fp16_dispatch(torch::Tensor& out, torch::Tensor const& a,
121 + torch::Tensor const& bt_nzs,
122 + torch::Tensor const& bt_meta,
123 + EpilogueArgs&&... args) {
124 + static_assert(std::is_same<InType, cutlass::half_t>());
125 + TORCH_CHECK(a.dtype() == torch::kFloat16);
126 + TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
127 + TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
128 +
129 + using Cutlass3xGemmDefault =
130 + typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
131 +
132 + // m in (128, inf)
133 + return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
134 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
135 +}
136 +
137 +template <typename InType, typename OutType,
138 + template <typename, typename, typename> typename Epilogue,
139 + typename... EpilogueArgs>
140 +void cutlass_gemm_sm90_bf16_dispatch(torch::Tensor& out, torch::Tensor const& a,
141 + torch::Tensor const& bt_nzs,
142 + torch::Tensor const& bt_meta,
143 + EpilogueArgs&&... args) {
144 + static_assert(std::is_same<InType, cutlass::bfloat16_t>());
145 + TORCH_CHECK(a.dtype() == torch::kBFloat16);
146 + TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
147 + TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
148 +
149 + using Cutlass3xGemmDefault =
150 + typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
151 +
152 + // m in (128, inf)
153 + return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
154 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
155 +}
156 +
157 +template <typename InType, typename OutType,
158 + template <typename, typename, typename> typename Epilogue,
159 + typename... EpilogueArgs>
160 +void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
161 + torch::Tensor const& bt_nzs,
162 + torch::Tensor const& bt_meta,
163 + EpilogueArgs&&... args) {
164 + static_assert(std::is_same<InType, int8_t>());
165 + TORCH_CHECK(a.dtype() == torch::kInt8);
166 + TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
167 + TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
168 +
169 + using Cutlass3xGemmDefault =
170 + typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
171 + using Cutlass3xGemmM128 =
172 + typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
173 + using Cutlass3xGemmM64 =
174 + typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
175 + using Cutlass3xGemmM32NBig =
176 + typename sm90_int8_config_M32_NBig<InType, OutType,
177 + Epilogue>::Cutlass3xGemm;
178 + using Cutlass3xGemmM32NSmall =
179 + typename sm90_int8_config_M32_NSmall<InType, OutType,
180 + Epilogue>::Cutlass3xGemm;
181 +
182 + uint32_t const n = out.size(1);
183 + bool const is_small_n = n < 8192;
184 +
185 + uint32_t const m = a.size(0);
186 + uint32_t const mp2 =
187 + std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
188 +
189 + if (mp2 <= 32) {
190 + // m in [1, 32]
191 + if (is_small_n) {
192 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NSmall>(
193 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
194 + } else {
195 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NBig>(
196 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
197 + }
198 + } else if (mp2 <= 64) {
199 + // m in (32, 64]
200 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
201 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
202 + } else if (mp2 <= 128) {
203 + // m in (64, 128]
204 + return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
205 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
206 + } else {
207 + // m in (128, inf)
208 + return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
209 + out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
210 + }
211 +}
212 +
213 +template <template <typename, typename, typename> typename Epilogue,
214 + typename... EpilogueArgs>
215 +void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
216 + torch::Tensor const& a,
217 + torch::Tensor const& bt_nzs,
218 + torch::Tensor const& bt_meta,
219 + EpilogueArgs&&... epilogue_args) {
220 + TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
221 + if (a.dtype() == torch::kInt8) {
222 + TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
223 +
224 + if (out.dtype() == torch::kBFloat16) {
225 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
226 + Epilogue>(
227 + out, a, bt_nzs, bt_meta,
228 + std::forward<EpilogueArgs>(epilogue_args)...);
229 + } else {
230 + TORCH_CHECK(out.dtype() == torch::kFloat16);
231 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
232 + out, a, bt_nzs, bt_meta,
233 + std::forward<EpilogueArgs>(epilogue_args)...);
234 + }
235 + } else if (a.dtype() == torch::kFloat8_e4m3fn) {
236 + TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
237 +
238 + if (out.dtype() == torch::kBFloat16) {
239 + return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
240 + cutlass::bfloat16_t, Epilogue>(
241 + out, a, bt_nzs, bt_meta,
242 + std::forward<EpilogueArgs>(epilogue_args)...);
243 + } else {
244 + TORCH_CHECK(out.dtype() == torch::kFloat16);
245 + return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
246 + cutlass::half_t, Epilogue>(
247 + out, a, bt_nzs, bt_meta,
248 + std::forward<EpilogueArgs>(epilogue_args)...);
249 + }
250 + } else if (a.dtype() == torch::kFloat16) {
251 + TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
252 +
253 + if (out.dtype() == torch::kBFloat16) {
254 + return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t,
255 + cutlass::bfloat16_t, Epilogue>(
256 + out, a, bt_nzs, bt_meta,
257 + std::forward<EpilogueArgs>(epilogue_args)...);
258 + } else {
259 + TORCH_CHECK(out.dtype() == torch::kFloat16);
260 + return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t, cutlass::half_t,
261 + Epilogue>(
262 + out, a, bt_nzs, bt_meta,
263 + std::forward<EpilogueArgs>(epilogue_args)...);
264 + }
265 + } else { // a.dtype() == torch::kBFloat16
266 + TORCH_CHECK(a.dtype() == torch::kBFloat16);
267 + TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
268 +
269 + if (out.dtype() == torch::kBFloat16) {
270 + return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
271 + cutlass::bfloat16_t, Epilogue>(
272 + out, a, bt_nzs, bt_meta,
273 + std::forward<EpilogueArgs>(epilogue_args)...);
274 + } else {
275 + TORCH_CHECK(out.dtype() == torch::kFloat16);
276 + return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
277 + cutlass::half_t, Epilogue>(
278 + out, a, bt_nzs, bt_meta,
279 + std::forward<EpilogueArgs>(epilogue_args)...);
280 + }
281 + }
282 +}
283 +
284 +void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
285 + torch::Tensor const& bt_nzs,
286 + torch::Tensor const& bt_meta,
287 + torch::Tensor const& a_scales,
288 + torch::Tensor const& b_scales,
289 + c10::optional<torch::Tensor> const& bias) {
290 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
291 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
292 + if (bias) {
293 + TORCH_CHECK(bias->dtype() == out.dtype(),
294 + "currently bias dtype must match output dtype ", out.dtype());
295 + return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
296 + out, a, bt_nzs, bt_meta, b_scales, a_scales, *bias);
297 + } else {
298 + return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogue>(
299 + out, a, bt_nzs, bt_meta, b_scales, a_scales);
300 + }
301 +}
302 +
303 +#endif
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cuh new
+496
@@ -0,0 +1,496 @@
1 +// clang-format will break include orders
2 +// clang-format off
3 +#include <cudaTypedefs.h>
4 +
5 +#include <torch/all.h>
6 +
7 +#include <ATen/cuda/CUDAContext.h>
8 +
9 +#include "cutlass/cutlass.h"
10 +
11 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
12 +#include "cutlass/epilogue/collective/collective_builder.hpp"
13 +#include "cutlass/gemm/collective/collective_builder.hpp"
14 +
15 +#include "core/math.hpp"
16 +#include "cutlass_extensions/cute_utils.cuh"
17 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
18 +#include "cutlass_extensions/common.hpp"
19 +#include "cutlass_extensions/torch_utils.hpp"
20 +// clang-format on
21 +
22 +using namespace cute;
23 +
24 +/*
25 + This file defines sparse quantized GEMM operations using the CUTLASS 3.x API,
26 + for NVIDIA GPUs with sm90a (Hopper) or later.
27 +*/
28 +
29 +namespace {
30 +
31 +// A wrapper for the GEMM kernel that is used to guard against compilation on
32 +// architectures that will never use the kernel. The purpose of this is to
33 +// reduce the size of the compiled binary.
34 +// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
35 +// into code that will be executed on the device where it is defined.
36 +template <typename Kernel>
37 +struct enable_sm90_or_later : Kernel {
38 + template <typename... Args>
39 + CUTLASS_DEVICE void operator()(Args&&... args) {
40 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
41 + Kernel::operator()(std::forward<Args>(args)...);
42 +#endif
43 + }
44 +};
45 +
46 +using GemmUniversalMode = cutlass::gemm::GemmUniversalMode;
47 +
48 +template <typename ElementAB_, typename ElementD_,
49 + template <typename, typename, typename> typename Epilogue_,
50 + typename TileShape, typename ClusterShape, typename KernelSchedule,
51 + typename EpilogueSchedule, typename AccType,
52 + typename TileSchedule = cutlass::gemm::PersistentScheduler,
53 + GemmUniversalMode Mode_ = GemmUniversalMode::kGemm>
54 +struct cutlass_sparse_3x_gemm {
55 + static const GemmUniversalMode Mode = Mode_;
56 + using ElementAB = ElementAB_;
57 + using ElementD = ElementD_;
58 + using ElementAcc = AccType;
59 +
60 + using EpilogueDescriptor =
61 + cutlass::epilogue::collective::detail::EpilogueDescriptor<
62 + TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
63 + ElementD, EpilogueSchedule>;
64 +
65 + using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
66 +
67 + using ElementC = void;
68 + using LayoutC = cutlass::layout::RowMajor;
69 + using LayoutD = LayoutC;
70 + using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
71 + using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
72 +
73 + using LayoutC_Transpose =
74 + typename cutlass::layout::LayoutTranspose<LayoutC>::type;
75 + using LayoutD_Transpose =
76 + typename cutlass::layout::LayoutTranspose<LayoutD>::type;
77 +
78 + using EVTCompute = typename Epilogue::EVTCompute;
79 +
80 + static constexpr int AlignmentA =
81 + 128 / cutlass::sizeof_bits<ElementAB>::value;
82 + static constexpr int AlignmentB =
83 + 128 / cutlass::sizeof_bits<ElementAB>::value;
84 + static constexpr int AlignmentCD =
85 + 128 / cutlass::sizeof_bits<ElementD>::value;
86 +
87 + using CollectiveEpilogue =
88 + typename cutlass::epilogue::collective::CollectiveBuilder<
89 + cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
90 + ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
91 + ElementAcc, ElementAcc, ElementC, LayoutC_Transpose, AlignmentCD,
92 + ElementD, LayoutD_Transpose, AlignmentCD, EpilogueSchedule,
93 + EVTCompute>::CollectiveOp;
94 +
95 + static constexpr size_t CEStorageSize =
96 + sizeof(typename CollectiveEpilogue::SharedStorage);
97 + using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
98 + static_cast<int>(CEStorageSize)>;
99 +
100 + // clang-format off
101 + using CollectiveMainloop =
102 + typename cutlass::gemm::collective::CollectiveBuilder<
103 + cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
104 + ElementAB, cutlass::layout::RowMajor, AlignmentA,
105 + ElementAB, cutlass::layout::ColumnMajor, AlignmentB,
106 + ElementAcc, TileShape, ClusterShape,
107 + Stages,
108 + KernelSchedule>::CollectiveOp;
109 + // clang-format on
110 +
111 + using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
112 + cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
113 + TileSchedule>>;
114 +
115 + struct GemmKernel : public KernelType {};
116 +};
117 +
118 +template <typename Gemm, typename... EpilogueArgs>
119 +void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
120 + torch::Tensor const& bt_nzs,
121 + torch::Tensor const& bt_meta,
122 + EpilogueArgs&&... epilogue_params) {
123 + using ElementAB = typename Gemm::ElementAB;
124 + using ElementD = typename Gemm::ElementD;
125 +
126 + // Interface stride expected from the argument a (will get transposed)
127 + // We compute C^T = B^T * A^T, but we assume B is transposed before
128 + // compression and hence the bt_* naming
129 + using LayoutA = cutlass::layout::RowMajor;
130 + using LayoutB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutA;
131 + using LayoutE = typename Gemm::GemmKernel::CollectiveMainloop::LayoutE;
132 + using LayoutD = cutlass::layout::RowMajor;
133 +
134 + using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
135 + using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
136 +
137 + auto layout_A = make_cute_layout<StrideA>(a, "A");
138 + auto layout_D = make_cute_layout<StrideD>(out, "D");
139 +
140 + // Transpose A and D
141 + // A doesn't need to be transposed since cutlass expects a NxK matrix
142 + // for B (which is At)
143 + auto stride_At = layout_A.stride();
144 + auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
145 +
146 + using GemmKernel = typename Gemm::GemmKernel;
147 + typename GemmKernel::ProblemShape prob_shape{
148 + static_cast<int>(bt_nzs.size(0)), static_cast<int>(size<0>(layout_A)),
149 + static_cast<int>(size<1>(layout_A)), 1};
150 +
151 + using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
152 + using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
153 +
154 + LayoutB b_layout = SparseConfig::fill_layoutA(prob_shape);
155 + LayoutE e_layout = SparseConfig::fill_layoutE(prob_shape);
156 +
157 + auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
158 + auto b_ptr = static_cast<ElementAB*>(bt_nzs.data_ptr());
159 + auto e_ptr = static_cast<ElementE*>(bt_meta.data_ptr());
160 + typename GemmKernel::MainloopArguments mainloop_args{
161 + b_ptr, b_layout, a_ptr, stride_At, e_ptr, e_layout};
162 +
163 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
164 + typename GemmKernel::EpilogueArguments epilogue_args{
165 + Gemm::Epilogue::prepare_args(
166 + std::forward<EpilogueArgs>(epilogue_params)...),
167 + c_ptr, stride_Dt, c_ptr, stride_Dt};
168 +
169 + typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
170 + prob_shape, mainloop_args, epilogue_args};
171 +
172 + // Launch the CUTLASS GEMM kernel.
173 + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
174 + GemmOp gemm_op;
175 + CUTLASS_CHECK(gemm_op.can_implement(args));
176 +
177 + size_t workspace_size = gemm_op.get_workspace_size(args);
178 + auto const workspace_options =
179 + torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
180 + auto workspace = torch::empty(workspace_size, workspace_options);
181 +
182 + auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
183 +
184 + cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
185 + CUTLASS_CHECK(status);
186 +}
187 +
188 +template <typename InType, typename OutType,
189 + template <typename, typename, typename> typename Epilogue>
190 +struct sm90_config_default {};
191 +
192 +template <typename OutType,
193 + template <typename, typename, typename> typename Epilogue>
194 +struct sm90_config_default<half_t, OutType, Epilogue> {
195 + // M in (128, inf)
196 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
197 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
198 + using TileShape = Shape<_128, _128, _128>;
199 + using ClusterShape = Shape<_2, _1, _1>;
200 + using Cutlass3xGemm =
201 + cutlass_sparse_3x_gemm<half_t, OutType, Epilogue, TileShape, ClusterShape,
202 + KernelSchedule, EpilogueSchedule, float>;
203 +};
204 +
205 +template <typename OutType,
206 + template <typename, typename, typename> typename Epilogue>
207 +struct sm90_config_default<cutlass::bfloat16_t, OutType, Epilogue> {
208 + // M in (128, inf)
209 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
210 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
211 + using TileShape = Shape<_128, _128, _128>;
212 + using ClusterShape = Shape<_2, _1, _1>;
213 + using Cutlass3xGemm =
214 + cutlass_sparse_3x_gemm<cutlass::bfloat16_t, OutType, Epilogue, TileShape,
215 + ClusterShape, KernelSchedule, EpilogueSchedule,
216 + float>;
217 +};
218 +
219 +//////////////////////// Cherry-Picking Kernels ////////////////////////
220 +template <typename InType, typename OutType,
221 + template <typename, typename, typename> typename Epilogue>
222 +struct sm90_fp8_config_1 {
223 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
224 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
225 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
226 + using TileShape = Shape<_64, _64, _256>;
227 + using ClusterShape = Shape<_8, _1, _1>;
228 + using Cutlass3xGemm =
229 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
230 + KernelSchedule, EpilogueSchedule, float>;
231 +};
232 +
233 +template <typename InType, typename OutType,
234 + template <typename, typename, typename> typename Epilogue>
235 +struct sm90_fp8_config_2 {
236 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
237 + using KernelSchedule =
238 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
239 + using EpilogueSchedule =
240 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
241 + using TileShape = Shape<_128, _64, _256>;
242 + using ClusterShape = Shape<_8, _1, _1>;
243 + using Cutlass3xGemm =
244 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
245 + KernelSchedule, EpilogueSchedule, float>;
246 +};
247 +
248 +template <typename InType, typename OutType,
249 + template <typename, typename, typename> typename Epilogue>
250 +struct sm90_fp8_config_3 {
251 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
252 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
253 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
254 + using TileShape = Shape<_64, _64, _256>;
255 + using ClusterShape = Shape<_1, _2, _1>;
256 + using Cutlass3xGemm =
257 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
258 + KernelSchedule, EpilogueSchedule, float>;
259 +};
260 +
261 +template <typename InType, typename OutType,
262 + template <typename, typename, typename> typename Epilogue>
263 +struct sm90_fp8_config_4 {
264 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
265 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
266 + using EpilogueSchedule =
267 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
268 + using TileShape = Shape<_64, _128, _256>;
269 + using ClusterShape = Shape<_8, _1, _1>;
270 + using Cutlass3xGemm =
271 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
272 + KernelSchedule, EpilogueSchedule, float>;
273 +};
274 +
275 +template <typename InType, typename OutType,
276 + template <typename, typename, typename> typename Epilogue>
277 +struct sm90_fp8_config_5 {
278 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
279 + using KernelSchedule =
280 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
281 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
282 + using TileShape = Shape<_128, _128, _256>;
283 + using ClusterShape = Shape<_8, _1, _1>;
284 + using Cutlass3xGemm =
285 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
286 + KernelSchedule, EpilogueSchedule, float>;
287 +};
288 +
289 +template <typename InType, typename OutType,
290 + template <typename, typename, typename> typename Epilogue>
291 +struct sm90_fp8_config_6 {
292 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
293 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
294 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
295 + using TileShape = Shape<_64, _128, _256>;
296 + using ClusterShape = Shape<_1, _2, _1>;
297 + using Cutlass3xGemm =
298 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
299 + KernelSchedule, EpilogueSchedule, float>;
300 +};
301 +
302 +template <typename InType, typename OutType,
303 + template <typename, typename, typename> typename Epilogue>
304 +struct sm90_fp8_config_7 {
305 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
306 + using KernelSchedule =
307 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
308 + using EpilogueSchedule =
309 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
310 + using TileShape = Shape<_128, _128, _256>;
311 + using ClusterShape = Shape<_1, _1, _1>;
312 + using Cutlass3xGemm =
313 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
314 + KernelSchedule, EpilogueSchedule, float>;
315 +};
316 +
317 +template <typename InType, typename OutType,
318 + template <typename, typename, typename> typename Epilogue>
319 +struct sm90_fp8_config_8 {
320 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
321 + using KernelSchedule =
322 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
323 + using EpilogueSchedule =
324 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
325 + using TileShape = Shape<_128, _256, _128>;
326 + using ClusterShape = Shape<_8, _1, _1>;
327 + using Cutlass3xGemm =
328 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
329 + KernelSchedule, EpilogueSchedule, float>;
330 +};
331 +////////////////////////////////////////////////////////////////////////
332 +
333 +template <typename OutType,
334 + template <typename, typename, typename> typename Epilogue>
335 +struct sm90_config_default<cutlass::float_e4m3_t, OutType, Epilogue> {
336 + // M in (128, inf)
337 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
338 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
339 + using TileShape = Shape<_128, _128, _128>;
340 + using ClusterShape = Shape<_1, _2, _1>;
341 + using Cutlass3xGemm =
342 + cutlass_sparse_3x_gemm<cutlass::float_e4m3_t, OutType, Epilogue,
343 + TileShape, ClusterShape, KernelSchedule,
344 + EpilogueSchedule, float>;
345 +};
346 +
347 +template <typename InType, typename OutType,
348 + template <typename, typename, typename> typename Epilogue>
349 +struct sm90_fp8_config_M64 {
350 + // M in [1, 64]
351 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
352 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
353 + using EpilogueSchedule =
354 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
355 + using TileShape = Shape<_64, _64, _256>;
356 + using ClusterShape = Shape<_1, _1, _1>;
357 +
358 + using TileSchedule = cutlass::gemm::PersistentScheduler;
359 +
360 + using Cutlass3xGemm =
361 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
362 + KernelSchedule, EpilogueSchedule, float,
363 + TileSchedule>;
364 +};
365 +
366 +template <typename InType, typename OutType,
367 + template <typename, typename, typename> typename Epilogue>
368 +struct sm90_fp8_config_M128 {
369 + // M in (64, 128]
370 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
371 + using KernelSchedule =
372 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
373 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
374 + using TileShape = Shape<_64, _128, _256>;
375 + using ClusterShape = Shape<_1, _1, _1>;
376 +
377 + using TileSchedule = cutlass::gemm::PersistentScheduler;
378 +
379 + using Cutlass3xGemm =
380 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
381 + KernelSchedule, EpilogueSchedule, float,
382 + TileSchedule>;
383 +};
384 +
385 +template <typename InType, typename OutType,
386 + template <typename, typename, typename> typename Epilogue>
387 +struct sm90_fp8_config_M256 {
388 + // M in (128, 256]
389 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
390 + using KernelSchedule =
391 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
392 + using EpilogueSchedule =
393 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
394 + using TileShape = Shape<_128, _128, _256>;
395 + using ClusterShape = Shape<_1, _1, _1>;
396 +
397 + using TileSchedule = cutlass::gemm::PersistentScheduler;
398 +
399 + using Cutlass3xGemm =
400 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
401 + KernelSchedule, EpilogueSchedule, float,
402 + TileSchedule>;
403 +};
404 +
405 +template <typename InType, typename OutType,
406 + template <typename, typename, typename> typename Epilogue>
407 +struct sm90_fp8_config_M512 {
408 + // M in (256, ]
409 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
410 + using KernelSchedule =
411 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
412 + using EpilogueSchedule =
413 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
414 + using TileShape = Shape<_128, _128, _256>;
415 + using ClusterShape = Shape<_1, _1, _1>;
416 +
417 + using TileSchedule = cutlass::gemm::PersistentScheduler;
418 +
419 + using Cutlass3xGemm =
420 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
421 + KernelSchedule, EpilogueSchedule, float,
422 + TileSchedule>;
423 +};
424 +
425 +template <typename OutType,
426 + template <typename, typename, typename> typename Epilogue>
427 +struct sm90_config_default<int8_t, OutType, Epilogue> {
428 + // For M > 128 and any N
429 + using KernelSchedule =
430 + typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
431 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
432 + using TileShape = Shape<_128, _128, _128>;
433 + using ClusterShape = Shape<_2, _1, _1>;
434 + using Cutlass3xGemm =
435 + cutlass_sparse_3x_gemm<int8_t, OutType, Epilogue, TileShape, ClusterShape,
436 + KernelSchedule, EpilogueSchedule, int32_t>;
437 +};
438 +
439 +template <typename InType, typename OutType,
440 + template <typename, typename, typename> typename Epilogue>
441 +struct sm90_int8_config_M128 {
442 + // For M in (64, 128] and any N
443 + static_assert(std::is_same<InType, int8_t>());
444 + using KernelSchedule =
445 + typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
446 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
447 + using TileShape = Shape<_64, _128, _128>;
448 + using ClusterShape = Shape<_2, _1, _1>;
449 + using Cutlass3xGemm =
450 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
451 + KernelSchedule, EpilogueSchedule, int32_t>;
452 +};
453 +
454 +template <typename InType, typename OutType,
455 + template <typename, typename, typename> typename Epilogue>
456 +struct sm90_int8_config_M64 {
457 + // For M in (32, 64] and any N
458 + static_assert(std::is_same<InType, int8_t>());
459 + using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
460 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
461 + using TileShape = Shape<_64, _64, _256>;
462 + using ClusterShape = Shape<_1, _1, _1>;
463 + using Cutlass3xGemm =
464 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
465 + KernelSchedule, EpilogueSchedule, int32_t>;
466 +};
467 +
468 +template <typename InType, typename OutType,
469 + template <typename, typename, typename> typename Epilogue>
470 +struct sm90_int8_config_M32_NBig {
471 + // For M in [1, 32] and N >= 8192
472 + static_assert(std::is_same<InType, int8_t>());
473 + using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
474 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
475 + using TileShape = Shape<_64, _128, _256>;
476 + using ClusterShape = Shape<_1, _4, _1>;
477 + using Cutlass3xGemm =
478 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
479 + KernelSchedule, EpilogueSchedule, int32_t>;
480 +};
481 +
482 +template <typename InType, typename OutType,
483 + template <typename, typename, typename> typename Epilogue>
484 +struct sm90_int8_config_M32_NSmall {
485 + // For M in [1, 32] and N < 8192
486 + static_assert(std::is_same<InType, int8_t>());
487 + using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
488 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
489 + using TileShape = Shape<_64, _64, _256>;
490 + using ClusterShape = Shape<_1, _8, _1>;
491 + using Cutlass3xGemm =
492 + cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
493 + KernelSchedule, EpilogueSchedule, int32_t>;
494 +};
495 +
496 +} // namespace
\ No newline at end of file
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu new
+59
@@ -0,0 +1,59 @@
1 +#include <cudaTypedefs.h>
2 +
3 +#include <c10/cuda/CUDAGuard.h>
4 +#include <torch/all.h>
5 +
6 +#include "cutlass_extensions/common.hpp"
7 +
8 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
9 +void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
10 + torch::Tensor const& b,
11 + torch::Tensor const& e,
12 + torch::Tensor const& a_scales,
13 + torch::Tensor const& b_scales,
14 + c10::optional<torch::Tensor> const& bias);
15 +#endif
16 +
17 +void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
18 + torch::Tensor const& bt_nzs,
19 + torch::Tensor const& bt_meta,
20 + torch::Tensor const& a_scales,
21 + torch::Tensor const& b_scales,
22 + c10::optional<torch::Tensor> const& bias) {
23 + // Checks for conformality
24 + TORCH_CHECK(a.dim() == 2 && bt_nzs.dim() == 2 && c.dim() == 2);
25 + TORCH_CHECK(c.size(1) == bt_nzs.size(0) && bt_nzs.size(1) * 2 == a.size(1) &&
26 + a.size(0) == c.size(0));
27 + TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
28 + TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == bt_nzs.size(0));
29 +
30 + // Check for strides and alignment
31 + TORCH_CHECK(a.stride(1) == 1 && bt_nzs.stride(1) == 1 &&
32 + c.stride(1) == 1); // Row-major
33 + TORCH_CHECK(c.stride(0) % 16 == 0); // 16 Byte Alignment
34 + TORCH_CHECK(bt_nzs.stride(0) % 16 == 0); // 16 Byte Alignment
35 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
36 +
37 + if (bias) {
38 + TORCH_CHECK(bias->numel() == bt_nzs.size(0) && bias->is_contiguous() &&
39 + bias->dim() == 1);
40 + }
41 +
42 + at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
43 + int32_t version_num = get_sm_version_num();
44 +
45 + // Guard against compilation issues for sm90 kernels
46 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
47 + if (version_num >= 90) {
48 + cutlass_scaled_sparse_mm_sm90(c, a, bt_nzs, bt_meta, a_scales, b_scales,
49 + bias);
50 + return;
51 + }
52 +#endif
53 +
54 + TORCH_CHECK_NOT_IMPLEMENTED(
55 + false,
56 + "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
57 + "CUDA device capability: ",
58 + version_num);
59 +}
csrc/torch_bindings.cpp
+15
@@ -321,6 +321,21 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
321 ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
322 ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
323
324 + // CUTLASS sparse GEMM, supporting symmetric per-tensor or per-row/column
325 + // quantization, as well as bias
326 + ops.def(
327 + "cutlass_scaled_sparse_mm(Tensor! out, Tensor a,"
328 + " Tensor bt_nzs,"
329 + " Tensor bt_meta, Tensor a_scales,"
330 + " Tensor b_scales, Tensor? bias) -> ()");
331 + ops.impl("cutlass_scaled_sparse_mm", torch::kCUDA, &cutlass_scaled_sparse_mm);
332 +
333 + // CUTLASS sparse matrix compressor
334 + ops.def(
335 + "cutlass_sparse_compress_entry(Tensor! a_nzs, Tensor! a_meta,"
336 + " Tensor a) -> bool");
337 + ops.impl("cutlass_sparse_compress_entry", &cutlass_sparse_compress_entry);
338 +
339 // Mamba selective scan kernel
340 ops.def(
341 "selective_scan_fwd(Tensor! u, Tensor! delta,"
pyproject.toml
+1 -1
@@ -83,7 +83,7 @@ exclude = [
83 ]
84
85 [tool.codespell]
86 -ignore-words-list = "dout, te, indicies, subtile"
86 +ignore-words-list = "dout, te, indicies, subtile, ElementE"
87 skip = "./tests/models/fixtures,./tests/prompts,./benchmarks/sonnet.txt,./tests/lora/data,./build"
88
89 [tool.isort]
tests/kernels/test_semi_structured.py new
+131
@@ -0,0 +1,131 @@
1 +"""Tests for sparse cutlass kernels
2 +
3 +Run `pytest tests/kernels/test_semi_structured.py`.
4 +"""
5 +from typing import Optional, Tuple, Type
6 +
7 +import pytest
8 +import torch
9 +
10 +from vllm import _custom_ops as ops
11 +from vllm.platforms import current_platform
12 +
13 +CUDA_DEVICES = [
14 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15 +]
16 +
17 +capability = current_platform.get_device_capability()
18 +capability = capability[0] * 10 + capability[1]
19 +
20 +
21 +def to_fp8(tensor: torch.Tensor):
22 + finfo = torch.finfo(torch.float8_e4m3fn)
23 + return torch.round(tensor.clamp(
24 + min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
25 +
26 +
27 +def to_int8(tensor: torch.Tensor):
28 + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
29 +
30 +
31 +def rand_int8(shape: tuple, device: str = "cuda"):
32 + return to_int8(torch.rand(shape, device=device) * 255 - 128)
33 +
34 +
35 +def to_bf16(tensor: torch.Tensor) -> torch.Tensor:
36 + return tensor.to(dtype=torch.bfloat16)
37 +
38 +
39 +def to_fp16(tensor: torch.Tensor) -> torch.Tensor:
40 + return tensor.to(dtype=torch.float16)
41 +
42 +
43 +def prune_to_2_4(tensor):
44 + # Reshape tensor to [N, 4] where N is number of groups of 4
45 + original_shape = tensor.shape
46 + reshaped = tensor.reshape(-1, 4)
47 +
48 + # Get indices of top 2 absolute values in each group of 4
49 + _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
50 +
51 + # Create binary mask
52 + mask = torch.zeros_like(reshaped)
53 + mask.scatter_(dim=1,
54 + index=indices,
55 + src=torch.ones_like(indices, dtype=mask.dtype))
56 +
57 + # Apply mask and reshape back
58 + pruned = reshaped * mask
59 +
60 + # Turn all -0.0 to 0.0
61 + pruned[pruned == -0.0] = 0.0
62 +
63 + return pruned.reshape(original_shape)
64 +
65 +
66 +def make_rand_sparse_tensors(
67 + dtype: torch.dtype, m: int, n: int, k: int
68 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
69 + a = torch.randn((m, k), device='cuda') * 5
70 + b = torch.randn((n, k), device='cuda').t() * 5
71 +
72 + b = prune_to_2_4(b.t()).t()
73 +
74 + if dtype == torch.int8:
75 + a, b = to_int8(a), to_int8(b)
76 + elif dtype == torch.float8_e4m3fn:
77 + a, b = to_fp8(a), to_fp8(b)
78 + elif dtype == torch.float16:
79 + a, b = to_fp16(a), to_fp16(b)
80 + elif dtype == torch.bfloat16:
81 + a, b = to_bf16(a), to_bf16(b)
82 + else:
83 + raise ValueError("unsupported dtype")
84 +
85 + b_compressed, e = ops.cutlass_sparse_compress(b.t())
86 +
87 + # Compressed B, Metadata, Original A, B
88 + return b_compressed, e, a, b
89 +
90 +
91 +def baseline_scaled_mm(a: torch.Tensor,
92 + b: torch.Tensor,
93 + scale_a: torch.Tensor,
94 + scale_b: torch.Tensor,
95 + out_dtype: Type[torch.dtype],
96 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
97 + output = (scale_a * (scale_b * (torch.mm(
98 + a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
99 + if bias is not None:
100 + output = output + bias
101 +
102 + return output
103 +
104 +
105 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
106 + reason="Sparse FP8 is not yet supported on this GPU type.")
107 +# Test working with a subset of A and B for sparse matmul
108 +def test_cutlass_sparse_subset():
109 + big_m = 1024
110 + m, n, k = 512, 512, 512
111 +
112 + # Create tensors
113 + b_comp, e, whole_a, b = make_rand_sparse_tensors(torch.float8_e4m3fn,
114 + big_m, n, k)
115 + a = whole_a[0:m, 0:k]
116 + scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
117 + scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
118 +
119 + out = ops.cutlass_scaled_sparse_mm(a,
120 + b_comp,
121 + e,
122 + scale_a,
123 + scale_b,
124 + out_dtype=torch.bfloat16)
125 + baseline = baseline_scaled_mm(a,
126 + b,
127 + scale_a,
128 + scale_b,
129 + out_dtype=torch.bfloat16)
130 +
131 + torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
tests/quantization/test_compressed_tensors.py
+100 -3
@@ -10,9 +10,11 @@ from compressed_tensors.quantization import QuantizationType
10
11 from tests.models.utils import check_logprobs_close
12 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
13 - CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
14 - CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
15 - CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13 + CompressedTensors24, CompressedTensorsLinearMethod,
14 + CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
15 + CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
16 + CompressedTensorsWNA16)
17 +from vllm.platforms import current_platform
18
19
20 @pytest.mark.parametrize(
@@ -208,3 +210,98 @@ def test_compressed_tensors_kv_cache(vllm_runner):
210 with vllm_runner(model_path, kv_cache_dtype="fp8") as llm:
211 output = llm.generate_greedy("Hello world!", max_tokens=20)
212 assert output
213 +
214 +
215 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
216 + reason="Sparse FP8 is not yet supported on this GPU type.")
217 +def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
218 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
219 + assert isinstance(qkv_proj.scheme, CompressedTensors24)
220 +
221 + assert qkv_proj.scheme.weight_quant.strategy == weight_strategy
222 + assert qkv_proj.scheme.input_quant.strategy == input_strategy
223 + assert qkv_proj.scheme.quantized
224 + assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
225 + sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
226 + assert sparsity_map.get("Linear").format == "dense"
227 + assert sparsity_map.get("Linear").sparsity_structure == "2:4"
228 +
229 +
230 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
231 + reason="Sparse FP8 is not yet supported on this GPU type.")
232 +@pytest.mark.parametrize("args_2of4", [
233 + ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing", "channel",
234 + "token"),
235 + ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
236 + "channel", "tensor"),
237 + ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing", "tensor",
238 + "tensor"),
239 + ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
240 + "tensor", "token"),
241 +])
242 +def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
243 + model, weight_strategy, input_strategy = args_2of4
244 + with vllm_runner(model) as llm:
245 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
246 + layer = model.model.layers[0]
247 +
248 + qkv_proj = layer.self_attn.qkv_proj
249 + assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
250 + _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
251 +
252 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
253 + print(output)
254 + assert output
255 +
256 +
257 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
258 + reason="Sparse FP8 is not yet supported on this GPU type.")
259 +@pytest.mark.parametrize("args_2of4", [
260 + ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
261 + "channel", "token"),
262 + ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing", "tensor",
263 + "tensor"),
264 + ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
265 + "tensor", "token"),
266 +])
267 +def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
268 + model, weight_strategy, input_strategy = args_2of4
269 + with vllm_runner(model) as llm:
270 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
271 + layer = model.model.layers[0]
272 +
273 + qkv_proj = layer.self_attn.qkv_proj
274 + assert qkv_proj.scheme.weights_dtype == torch.int8
275 + _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
276 +
277 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
278 + print(output)
279 + assert output
280 +
281 +
282 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
283 + reason="Sparse FP8 is not yet supported on this GPU type.")
284 +@pytest.mark.parametrize(
285 + "args_2of4",
286 + [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
287 +def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
288 + model = args_2of4
289 + with vllm_runner(model) as llm:
290 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
291 + layer = model.model.layers[0]
292 +
293 + qkv_proj = layer.self_attn.qkv_proj
294 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
295 + assert isinstance(qkv_proj.scheme, CompressedTensors24)
296 +
297 + assert qkv_proj.scheme.weight_quant is None
298 + assert qkv_proj.scheme.input_quant is None
299 + assert not qkv_proj.scheme.quantized
300 + assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
301 + sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
302 + assert sparsity_map.get("Linear").format == "dense"
303 + assert sparsity_map.get("Linear").sparsity_structure == "2:4"
304 +
305 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
306 + print(output)
307 + assert output
tests/weight_loading/models.txt
+2
@@ -21,6 +21,8 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22 compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23 compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
24 +compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-FP8-Dynamic-testing, main, 90
25 +compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-W8A8-testing, main, 90
26 awq, casperhansen/mixtral-instruct-awq, main
27 awq_marlin, casperhansen/mixtral-instruct-awq, main
28 fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
tests/weight_loading/run_model_weight_loading_test.sh
+4
@@ -26,6 +26,10 @@ do
26 export QUANTIZATION=${array[0]}
27 export MODEL_NAME=${array[1]}
28 export REVISION=${array[2]}
29 + # If array length is larger than 3, then MIN_CAPABILITY is provided
30 + if [ ${#array[@]} -gt 3 ]; then
31 + export MIN_CAPABILITY=${array[3]}
32 + fi
33 pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$?
34
35 if [[ $LOCAL_SUCCESS == 0 ]]; then
tests/weight_loading/test_weight_loading.py
+7
@@ -1,14 +1,21 @@
1 import os
2
3 +import pytest
4 import torch
5
6 +from vllm.platforms import current_platform
7 +
8 MAX_MODEL_LEN = 1024
9 MODEL_NAME = os.environ.get("MODEL_NAME",
10 "robertgshaw2/zephyr-7b-beta-channelwise-gptq")
11 REVISION = os.environ.get("REVISION", "main")
12 QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
13 +MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "89")
14
15
16 +@pytest.mark.skipif(
17 + not current_platform.has_device_capability(int(MIN_CAPABILITY)),
18 + reason="Current system does not have minimum capability.")
19 def test_weight_loading(vllm_runner):
20 """
21 Test parameter weight loading with tp>1.
vllm/_custom_ops.py
+103
@@ -552,6 +552,109 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
552 return out
553
554
555 +def cutlass_sparse_compress(a: torch.Tensor) \
556 + -> Tuple[torch.Tensor, torch.Tensor]:
557 + """
558 + Compresses a sparse matrix for use with Cutlass sparse operations.
559 +
560 + This function takes a dense tensor and compresses it into two components:
561 + non-zero elements and metadata. The compressed representation is compatible
562 + with Cutlass sparse kernels.
563 +
564 + Args:
565 + a (torch.Tensor):
566 + The input tensor to be compressed. Must have one of the following data types:
567 + - `torch.int8`
568 + - `torch.float8_e4m3fn`
569 + - `torch.bfloat16`
570 + - `torch.float16`
571 +
572 + Returns:
573 + Tuple[torch.Tensor, torch.Tensor]:
574 + A tuple containing:
575 + - `a_nzs` (torch.Tensor): A tensor containing non-zero elements of `a`.
576 + - `a_meta` (torch.Tensor): A tensor containing metadata for the sparse representation.
577 +
578 + Raises:
579 + ValueError: If the compression operation fails.
580 +
581 + Notes:
582 + - The `a_meta` tensor has a data type of `torch.uint8`.
583 + - Each metadata element encodes the sparsity of 4 non-zero elements (i.e., `elemsPerMetaElem = 4`).
584 + - The shape of `a_nzs` is `(m, k // 2)`, where `m` and `k` are the dimensions of the input tensor.
585 + - The shape of `a_meta` is `(m, k // 2 // elemsPerMetaElem)`.
586 + """
587 + assert (a.dtype in [
588 + torch.int8, torch.float8_e4m3fn, torch.bfloat16, torch.float16
589 + ])
590 + assert (a.is_contiguous())
591 +
592 + # a_meta.dtype: torch.uint8 so elemsPerMetaElem = 8b / 2b_per_nz = 4
593 + elemsPerMetaElem = 4
594 +
595 + m = a.shape[0]
596 + k = a.shape[1]
597 + assert (k % 2 == 0)
598 + a_nzs = torch.empty((m, k // 2), dtype=a.dtype, device=a.device)
599 + a_meta = torch.empty((m, k // 2 // elemsPerMetaElem),
600 + dtype=torch.uint8,
601 + device=a.device)
602 +
603 + if not (torch.ops._C.cutlass_sparse_compress_entry(a_nzs, a_meta, a)):
604 + raise ValueError
605 +
606 + assert (a_nzs.is_contiguous())
607 + assert (a_meta.is_contiguous())
608 +
609 + return a_nzs, a_meta
610 +
611 +
612 +def cutlass_scaled_sparse_mm(
613 + a: torch.Tensor,
614 + bt_nzs: torch.Tensor,
615 + bt_meta: torch.Tensor,
616 + scale_a: torch.Tensor,
617 + scale_b: torch.Tensor,
618 + out_dtype: torch.dtype,
619 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
620 + """
621 + Performs a scaled sparse matrix multiplication using Cutlass.
622 +
623 + Steps:
624 + 1. Create a dense matrix `a` of shape (m, k) on the CUDA device:
625 + `a = torch.randn((m, k), device='cuda')`.
626 +
627 + 2. Create a dense matrix `b` of shape (k, n) on the CUDA device:
628 + `b = torch.randn((k, n), device='cuda')`.
629 +
630 + 3. Prune matrix `b` to 2:4 sparsity along the specified dimension:
631 + `b = prune_to_2_4(b, dim=0)`.
632 +
633 + 4. Compress the transposed sparse matrix `b.t()`:
634 + `bt_nzs, bt_meta = cutlass_sparse_compress(b.t())`.
635 +
636 + 5. Perform sparse matrix multiplication using the compressed matrix,
637 + applying scaling factors for `a` and `b`, and the output data type:
638 + `out = cutlass_scaled_sparse_mm(a, bt_nzs, bt_meta, scale_a, scale_b, out_dtype)`.
639 +
640 + Returns:
641 + - The result of the scaled sparse matrix multiplication.
642 + """
643 + assert (bt_nzs.shape[0] % 16 == 0 and bt_nzs.shape[1] % 16 == 0)
644 + assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
645 + assert bias is None or bias.shape[0] == bt_nzs.shape[0] \
646 + and bias.dtype == out_dtype
647 +
648 + m = a.shape[0]
649 + n = bt_nzs.shape[0]
650 + out = torch.empty((m, n), dtype=out_dtype, device=a.device)
651 +
652 + torch.ops._C.cutlass_scaled_sparse_mm(out, a, bt_nzs, bt_meta, scale_a,
653 + scale_b, bias)
654 +
655 + return out
656 +
657 +
658 # aqlm
659 def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
660 codebooks: torch.Tensor, scales: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+160 -27
@@ -1,7 +1,9 @@
1 -from typing import Any, Dict, List, Optional, cast
1 +from typing import Any, Dict, List, Literal, Optional, cast
2
3 import torch
4 -from compressed_tensors.config import CompressionFormat
4 +from compressed_tensors.config import (CompressionFormat,
5 + SparsityCompressionConfig,
6 + SparsityStructure)
7 from compressed_tensors.quantization import (QuantizationArgs,
8 QuantizationStrategy,
9 QuantizationType)
@@ -15,7 +17,7 @@ from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
17 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
18 CompressedTensorsMoEMethod)
19 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
18 - W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
20 + W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS, CompressedTensors24,
21 CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
22 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
23 CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
@@ -27,20 +29,29 @@ from vllm.platforms import current_platform
29
30 __all__ = ["CompressedTensorsLinearMethod"]
31
32 +SPARSITY_CONFIG_NAME: Literal["sparsity_config"] = "sparsity_config"
33 +QUANTIZATION_SCHEME_MAP_TYPE = Dict[str, Optional[Dict[str, QuantizationArgs]]]
34 +
35
36 class CompressedTensorsConfig(QuantizationConfig):
37
33 - def __init__(self,
34 - target_scheme_map: Dict[str, Any],
35 - ignore: List[str],
36 - quant_format: str,
37 - kv_cache_scheme: Optional[Dict[str, Any]] = None):
38 + def __init__(
39 + self,
40 + target_scheme_map: Dict[str, Any],
41 + ignore: List[str],
42 + quant_format: str,
43 + sparsity_scheme_map: Dict[str, SparsityCompressionConfig],
44 + kv_cache_scheme: Optional[Dict[str, Any]] = None,
45 + config: Optional[Dict[str, Any]] = None,
46 + ):
47
48 self.ignore = ignore
49 self.quant_format = quant_format
50 # Map from [target -> scheme]
51 self.target_scheme_map = target_scheme_map
52 self.kv_cache_scheme = kv_cache_scheme
53 + self.sparsity_scheme_map = sparsity_scheme_map
54 + self.config = config
55
56 def get_linear_method(self) -> "CompressedTensorsLinearMethod":
57 return CompressedTensorsLinearMethod(self)
@@ -78,8 +89,50 @@ class CompressedTensorsConfig(QuantizationConfig):
89
90 @classmethod
91 def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
92 + ignore: List[str] = cast(List[str], config.get("ignore", []))
93 + quant_format = cast(str, config.get("format"))
94 + target_scheme_map = cls._quantization_scheme_map_from_config(
95 + config=config)
96 + sparsity_scheme_map = cls._sparsity_scheme_map_from_config(
97 + config=config)
98 +
99 + return cls(
100 + target_scheme_map=target_scheme_map,
101 + ignore=ignore,
102 + quant_format=quant_format,
103 + sparsity_scheme_map=sparsity_scheme_map,
104 + config=config,
105 + )
106 +
107 + @classmethod
108 + def _sparsity_scheme_map_from_config(
109 + cls, config: Dict[str,
110 + Any]) -> Dict[str, SparsityCompressionConfig]:
111 + """
112 + :param config: The `quantization_config` dictionary from config.json
113 + :return: A dictionary mapping target layer names to their corresponding
114 + sparsity compression configurations
115 + """
116 + if (sparsity_config := config.get(SPARSITY_CONFIG_NAME)) is None:
117 + return dict()
118 +
119 + sparsity_config = SparsityCompressionConfig.model_validate(
120 + sparsity_config)
121 + sparse_scheme_map: Dict[str, SparsityCompressionConfig] = {
122 + target: sparsity_config
123 + for target in sparsity_config.targets or list()
124 + }
125 + return sparse_scheme_map
126 +
127 + @classmethod
128 + def _quantization_scheme_map_from_config(
129 + cls, config: Dict[str, Any]) -> QUANTIZATION_SCHEME_MAP_TYPE:
130 + """
131 + :param config: The `quantization_config` dictionary from config.json
132 + :return: A dictionary mapping target layer names to their corresponding
133 + quantization_args for weights and input activations
134 + """
135 target_scheme_map: Dict[str, Any] = dict()
82 - ignore = cast(List[str], config.get("ignore"))
136 quant_format = cast(str, config.get("format"))
137
138 # The quant_config has multiple config_groups, each containing
@@ -90,12 +143,14 @@ class CompressedTensorsConfig(QuantizationConfig):
143 # details follow the structure defined by the QuantizationArgs
144 # pydantic model, which is used to verify the structure of the
145 # quant_config and also store the details for later use.
93 - for _, quant_config in config["config_groups"].items():
146 +
147 + config_groups = config.get("config_groups", dict())
148 + for _, quant_config in config_groups.items():
149 targets = quant_config.get("targets")
150 for target in targets:
151 target_scheme_map[target] = {}
152 target_scheme_map[target][
98 - "weights"] = QuantizationArgs.parse_obj(
153 + "weights"] = QuantizationArgs.model_validate(
154 quant_config.get("weights"))
155
156 target_scheme_map[target]["input_activations"] = None
@@ -110,13 +165,9 @@ class CompressedTensorsConfig(QuantizationConfig):
165 "weights"].type == QuantizationType.FLOAT
166 else:
167 target_scheme_map[target][
113 - "input_activations"] = QuantizationArgs.parse_obj(
168 + "input_activations"] = QuantizationArgs.model_validate( # noqa: E501
169 quant_config.get("input_activations"))
115 -
116 - return cls(target_scheme_map=target_scheme_map,
117 - ignore=ignore,
118 - quant_format=quant_format,
119 - kv_cache_scheme=config.get("kv_cache_scheme"))
170 + return target_scheme_map
171
172 @classmethod
173 def get_config_filenames(cls) -> List[str]:
@@ -315,23 +366,105 @@ class CompressedTensorsConfig(QuantizationConfig):
366 # TODO (@robertgshaw): add compressed-tensors as dep
367 # so we do not have to re-write these functions
368 # need to make accelerate optional in ct to do this
318 - matched_target = find_matched_target(
319 - layer_name=layer_name,
320 - module=layer,
321 - targets=self.target_scheme_map.keys())
369
323 - # Find the quant_scheme
324 - scheme_dict = self.target_scheme_map[matched_target]
325 - scheme = self._get_scheme_from_parts(
326 - weight_quant=scheme_dict["weights"],
327 - input_quant=scheme_dict["input_activations"])
370 + # Will be empty for models with only sparsity
371 + if self.target_scheme_map:
372 + matched_target = find_matched_target(
373 + layer_name=layer_name,
374 + module=layer,
375 + targets=self.target_scheme_map.keys())
376 +
377 + scheme_dict = self.target_scheme_map[matched_target]
378 + weight_quant = scheme_dict.get("weights")
379 + input_quant = scheme_dict.get("input_activations")
380 + elif self.sparsity_scheme_map:
381 + matched_target = find_matched_target(
382 + layer_name=layer_name,
383 + module=layer,
384 + targets=self.sparsity_scheme_map.keys())
385 + weight_quant = None
386 + input_quant = None
387 +
388 + # For models with sparsity, assumes that the sparse layers are also
389 + # quantized for cutlass 2:4 support
390 + sparsity_scheme: Optional[
391 + SparsityCompressionConfig] = self.sparsity_scheme_map.get(
392 + matched_target)
393 +
394 + if self.supports_cutlass_24(weight_quant=weight_quant,
395 + input_quant=input_quant,
396 + sparsity_scheme=sparsity_scheme):
397 + # Have a valid sparsity scheme
398 + # Validate layer is supported by Cutlass 2:4 Kernel
399 + scheme = CompressedTensors24(quantized=weight_quant is not None
400 + or input_quant is not None,
401 + weight_quant=weight_quant,
402 + input_quant=input_quant)
403 + else:
404 + # Find the quant_scheme
405 + scheme = self._get_scheme_from_parts( # type: ignore
406 + weight_quant=weight_quant,
407 + input_quant=input_quant,
408 + )
409
410 # Raise error if device does not support the scheme
411 # (e.g. fp8 needs ada lovelace)
412 self._check_scheme_supported(scheme.get_min_capability())
332 -
413 return scheme
414
415 + @staticmethod
416 + def supports_cutlass_24(
417 + weight_quant: Optional[QuantizationArgs],
418 + input_quant: Optional[QuantizationArgs],
419 + sparsity_scheme: Optional[SparsityCompressionConfig] = None
420 + ) -> bool:
421 + """
422 + Check if the layer is supported by the Cutlass 2:4 Kernel
423 + Conditions:
424 + - Overarching condition: Sparsity Structure is 2:4
425 + - Unquantized cases are supported
426 + - Weight only quantization is not-supported
427 + - Supported weight quantization strategies are TENSOR and CHANNEL
428 + - Supported input quantization strategies are TENSOR and TOKEN
429 + - Only 8 bit quantization is supported
430 +
431 + :return: True if the layer is supported by the Cutlass 2:4 Kernel
432 + False otherwise
433 + """
434 + is_valid_sparsity = (sparsity_scheme is not None
435 + and sparsity_scheme.sparsity_structure
436 + == SparsityStructure.TWO_FOUR.value
437 + and sparsity_scheme.format == "dense")
438 + if not is_valid_sparsity:
439 + return False
440 +
441 + # Unquantized cases are supported
442 + if weight_quant is None and input_quant is None:
443 + return True
444 +
445 + # Weight only quantization is not-supported
446 + if weight_quant is not None and input_quant is None:
447 + return False
448 +
449 + supported_weight_quant_strategies = [
450 + QuantizationStrategy.TENSOR.value,
451 + QuantizationStrategy.CHANNEL.value
452 + ]
453 +
454 + assert weight_quant is not None
455 + assert input_quant is not None
456 + if weight_quant.strategy not in supported_weight_quant_strategies:
457 + return False
458 +
459 + supported_input_quant_strategies = [
460 + QuantizationStrategy.TENSOR.value, QuantizationStrategy.TOKEN.value
461 + ]
462 +
463 + if input_quant.strategy not in supported_input_quant_strategies:
464 + return False
465 +
466 + return weight_quant.num_bits == input_quant.num_bits == 8
467 +
468
469 class CompressedTensorsLinearMethod(LinearMethodBase):
470
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+7 -8
@@ -7,13 +7,12 @@ from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
7 from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
8 CompressedTensorsWNA16)
9
10 +from .compressed_tensors_24 import CompressedTensors24 # isort: skip
11 +
12 __all__ = [
11 - "CompressedTensorsScheme",
12 - "CompressedTensorsWNA16",
13 - "CompressedTensorsW8A16Fp8",
14 - "CompressedTensorsW4A16Sparse24",
15 - "CompressedTensorsW8A8Int8",
16 - "CompressedTensorsW8A8Fp8",
17 - "WNA16_SUPPORTED_BITS",
18 - "W4A16SPARSE24_SUPPORTED_BITS",
13 + "CompressedTensorsScheme", "CompressedTensorsWNA16",
14 + "CompressedTensorsW8A16Fp8", "CompressedTensorsW4A16Sparse24",
15 + "CompressedTensorsW8A8Int8", "CompressedTensorsW8A8Fp8",
16 + "WNA16_SUPPORTED_BITS", "W4A16SPARSE24_SUPPORTED_BITS",
17 + "CompressedTensors24"
18 ]
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py new
+203
@@ -0,0 +1,203 @@
1 +from typing import Callable, List, Optional
2 +
3 +import torch
4 +from compressed_tensors.quantization import (QuantizationArgs,
5 + QuantizationStrategy,
6 + QuantizationType)
7 +
8 +from vllm import _custom_ops as ops
9 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 + CompressedTensorsScheme)
11 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 + convert_to_channelwise)
13 +from vllm.model_executor.parameter import (BasevLLMParameter,
14 + ChannelQuantScaleParameter,
15 + ModelWeightParameter,
16 + PerTensorScaleParameter)
17 +
18 +__all__ = ["CompressedTensors24"]
19 +
20 +
21 +class CompressedTensors24(CompressedTensorsScheme):
22 +
23 + def __init__(self,
24 + quantized: bool = False,
25 + weight_quant: Optional[QuantizationArgs] = None,
26 + input_quant: Optional[QuantizationArgs] = None):
27 +
28 + self.quantized = quantized
29 + self.weight_quant = weight_quant
30 + self.input_quant = input_quant
31 +
32 + @classmethod
33 + def get_min_capability(cls) -> int:
34 + # Only cutlass 3.x kernels are implemented so far
35 + return 90
36 +
37 + def create_weights(self, layer: torch.nn.Module, input_size: int,
38 + output_partition_sizes: List[int],
39 + input_size_per_partition: int,
40 + params_dtype: torch.dtype, weight_loader: Callable,
41 + **kwargs):
42 +
43 + self.output_dtype = params_dtype
44 + layer.logical_widths = output_partition_sizes
45 + self.weights_dtype: torch.dtype = self._get_params_dtype(params_dtype)
46 +
47 + # parameter to store uncompressed weight
48 + weight = ModelWeightParameter(data=torch.empty(
49 + sum(output_partition_sizes),
50 + input_size_per_partition,
51 + dtype=self.weights_dtype),
52 + input_dim=1,
53 + output_dim=0,
54 + weight_loader=weight_loader)
55 +
56 + # Check if quantized, not just 2:4 Sparse
57 + if self.quantized:
58 + if (self.weight_quant and self.weight_quant.strategy
59 + == QuantizationStrategy.CHANNEL.value):
60 + weight_scale = ChannelQuantScaleParameter(
61 + data=torch.empty((sum(output_partition_sizes), 1),
62 + dtype=torch.float32),
63 + output_dim=0,
64 + weight_loader=weight_loader)
65 + else:
66 + assert (self.weight_quant and self.weight_quant.strategy
67 + == QuantizationStrategy.TENSOR.value)
68 + weight_scale = PerTensorScaleParameter(
69 + data=torch.empty(len(output_partition_sizes),
70 + dtype=torch.float32),
71 + weight_loader=weight_loader)
72 +
73 + layer.register_parameter("weight_scale", weight_scale)
74 +
75 + # input quant will be non-none
76 + if self.input_quant and not self.input_quant.dynamic:
77 + # register input quant scale
78 + assert (self.input_quant.strategy ==
79 + QuantizationStrategy.TENSOR.value)
80 + input_scale = BasevLLMParameter(data=torch.empty(
81 + 1, dtype=torch.float32),
82 + weight_loader=weight_loader)
83 +
84 + layer.register_parameter("input_scale", input_scale)
85 +
86 + else:
87 + # for sparse-only, pass in 1 for weight/input scales
88 + weight_scale = torch.nn.Parameter(data=torch.ones(
89 + 1, dtype=torch.float32),
90 + requires_grad=False)
91 + input_scale = torch.nn.Parameter(data=torch.ones(
92 + 1, dtype=torch.float32),
93 + requires_grad=False)
94 + layer.register_parameter("input_scale", input_scale)
95 + layer.register_parameter("weight_scale", weight_scale)
96 +
97 + layer.register_parameter("weight", weight)
98 +
99 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
100 + """
101 + Compress weights after loading. Store compressed weight and meta
102 + tensor
103 +
104 + :post-condition: layer.w_compressed and layer.meta are
105 + set to the compressed weight and meta tensor in the
106 + format expected by the Cutlass kernels
107 + :param layer: The layer with the weights to be processed
108 +
109 + """
110 + # torch.compile workaround
111 + if hasattr(layer, "input_scale"):
112 + layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
113 + requires_grad=False)
114 +
115 + if self.weight_quant:
116 + if self.weight_quant.strategy == QuantizationStrategy.TENSOR.value:
117 + layer.weight_scale = torch.nn.Parameter(convert_to_channelwise(
118 + weight_scale=layer.weight_scale,
119 + logical_widths=layer.logical_widths),
120 + requires_grad=False)
121 + else:
122 + # torch.compile workaround
123 + layer.weight_scale = torch.nn.Parameter(
124 + layer.weight_scale.data, requires_grad=False)
125 +
126 + w_compressed, meta = ops.cutlass_sparse_compress(layer.weight.data)
127 + layer.weight = torch.nn.Parameter(w_compressed, requires_grad=False)
128 + layer.meta = torch.nn.Parameter(meta, requires_grad=False)
129 +
130 + def apply_weights(self,
131 + layer: torch.nn.Module,
132 + x: torch.Tensor,
133 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
134 + """
135 + Returns the output tensor for the layer with 2:4
136 + sparse compressed weights, given the input tensor
137 + and bias
138 +
139 + :param layer: The layer with 2:4 sparse compressed
140 + weights to be used for the computation
141 + :param x: The input tensor to the layer
142 + :param bias: The bias to be added to the output tensor
143 + :return: The output tensor of the layer
144 + """
145 + if self.quantized:
146 + scale = None
147 + if hasattr(layer, "input_scale"):
148 + scale = layer.input_scale
149 +
150 + if self.weights_dtype == torch.int8:
151 + ops_output = ops.scaled_int8_quant(x, scale=scale)
152 + q_input = ops_output[0]
153 + input_scale = ops_output[1]
154 + else:
155 + assert self.weights_dtype == torch.float8_e4m3fn
156 + if scale is not None:
157 + q_input, input_scale = ops.scaled_fp8_quant(x, scale=scale)
158 + else:
159 + q_input, input_scale = ops.scaled_fp8_quant(
160 + x, use_per_token_if_dynamic=True)
161 +
162 + else:
163 + # Not quantized, nothing to do with the input_scales, use as is
164 + input_scale = layer.input_scale
165 + q_input = x
166 +
167 + out = ops.cutlass_scaled_sparse_mm(a=q_input,
168 + bt_nzs=layer.weight,
169 + bt_meta=layer.meta,
170 + scale_a=input_scale,
171 + scale_b=layer.weight_scale,
172 + out_dtype=self.output_dtype,
173 + bias=bias)
174 + assert out.is_contiguous()
175 + return out
176 +
177 + def _get_params_dtype(self, params_dtype: torch.dtype) -> torch.dtype:
178 + if not self.quantized:
179 + return params_dtype
180 +
181 + assert self.weight_quant is not None
182 + assert self.input_quant is not None
183 +
184 + is_8_bits = self.weight_quant.num_bits == self.input_quant.num_bits == 8
185 +
186 + if not is_8_bits:
187 + raise ValueError("Cutlass only supports 8-bit quantization")
188 +
189 + if (self.weight_quant.type == QuantizationType.FLOAT
190 + and self.input_quant.type == QuantizationType.FLOAT):
191 + return torch.float8_e4m3fn
192 +
193 + if (self.weight_quant.type == QuantizationType.INT
194 + and self.input_quant.type == QuantizationType.INT):
195 + return torch.int8
196 +
197 + raise ValueError("Quantization type not supported by Cutlass")
198 +
199 +
200 +def check_24(tensor):
201 + new_tensor = tensor.view(-1, 4)
202 + zero_counts = (new_tensor == 0).sum(dim=1)
203 + return (zero_counts >= 2).all().item()