@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9798b2fb00
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling (#11868)

Lucas Wilkinson committed Jan 30, 2025 at 21:33 UTC 9798b2fb0052092a6420172e41c0c8a307eedfa6
25 files changed +1924 -346
CMakeLists.txt
+7 -2
@@ -245,7 +245,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
245 FetchContent_Declare(
246 cutlass
247 GIT_REPOSITORY https://github.com/nvidia/cutlass.git
248 - GIT_TAG v3.6.0
248 + GIT_TAG v3.7.0
249 GIT_PROGRESS TRUE
250
251 # Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
@@ -299,7 +299,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
299 # CUDA 12.0 or later (and only work on Hopper, 9.0a for now).
300 cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0a" "${CUDA_ARCHS}")
301 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
302 - set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
302 + set(SRCS
303 + "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
304 + "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu"
305 + "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu"
306 + "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu"
307 + "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8.cu")
308 set_gencode_flags_for_srcs(
309 SRCS "${SRCS}"
310 CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+142 -148
@@ -3,7 +3,7 @@ import copy
3 import itertools
4 import pickle as pkl
5 import time
6 -from typing import Callable, Iterable, List, Tuple
6 +from typing import Callable, Iterable, List, Optional, Tuple
7
8 import torch
9 import torch.utils.benchmark as TBenchmark
@@ -12,6 +12,8 @@ from utils import make_rand_tensors
12 from weight_shapes import WEIGHT_SHAPES
13
14 from vllm import _custom_ops as ops
15 +from vllm.model_executor.layers.quantization.utils.fp8_utils import (
16 + w8a8_block_fp8_matmul)
17 from vllm.utils import FlexibleArgumentParser
18
19 DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
@@ -38,8 +40,15 @@ def bench_fn(label: str, sub_label: str, description: str, fn: Callable, *args,
40 ).blocked_autorange(min_run_time=min_run_time)
41
42
41 -def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
42 - sub_label: str) -> Iterable[TMeasurement]:
43 +def bench_int8(
44 + dtype: torch.dtype,
45 + m: int,
46 + k: int,
47 + n: int,
48 + label: str,
49 + sub_label: str,
50 + bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
51 + """Benchmark INT8-based kernels."""
52 assert dtype == torch.int8
53 a, b = make_rand_tensors(torch.int8, m, n, k)
54 scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
@@ -48,155 +57,132 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
57 azp = torch.zeros((m, ), device="cuda", dtype=torch.int32)
58 azp_adj = torch.zeros((n, ), device="cuda", dtype=torch.int32)
59
60 + bench_fns = {
61 + "pytorch_bf16_bf16_bf16_matmul-no-scales":
62 + lambda: torch.mm(a.to(dtype=torch.bfloat16), b.to(dtype=torch.bfloat16)
63 + ),
64 + "pytorch_fp16_fp16_fp16_matmul-no-scales":
65 + lambda: torch.mm(a.to(dtype=torch.float16), b.to(dtype=torch.float16)),
66 + "cutlass_i8_i8_bf16_scaled_mm":
67 + lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16),
68 + "cutlass_i8_i8_bf16_scaled_mm_bias":
69 + lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16,
70 + bias),
71 + "cutlass_i8_i8_bf16_scaled_mm_azp":
72 + lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
73 + bfloat16, azp_adj),
74 + "cutlass_i8_i8_bf16_scaled_mm_azp_bias":
75 + lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
76 + bfloat16, azp_adj, None, bias),
77 + "cutlass_i8_i8_bf16_scaled_mm_azp_pt":
78 + lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
79 + bfloat16, azp_adj, azp),
80 + "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias":
81 + lambda: ops.cutlass_scaled_mm_azp(a, b, scale_a, scale_b, torch.
82 + bfloat16, azp_adj, azp, bias),
83 + }
84 +
85 timers = []
52 - # pytorch impl - bfloat16
53 - timers.append(
54 - bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
55 - torch.mm, a.to(dtype=torch.bfloat16),
56 - b.to(dtype=torch.bfloat16)))
57 -
58 - # pytorch impl - float16
59 - timers.append(
60 - bench_fn(label, sub_label,
61 - "pytorch_fp16_fp16_fp16_matmul-no-scales", torch.mm,
62 - a.to(dtype=torch.float16), b.to(dtype=torch.float16)))
63 -
64 - # cutlass impl
65 - timers.append(
66 - bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm",
67 - ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
68 - torch.bfloat16))
69 -
70 - # cutlass with bias
71 - timers.append(
72 - bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_bias",
73 - ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
74 - bias))
75 -
76 - # cutlass with azp per-tensor
77 - timers.append(
78 - bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp",
79 - ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
80 - torch.bfloat16, azp_adj))
81 -
82 - # cutlass with azp per-tensor + bias
83 - timers.append(
84 - bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_bias",
85 - ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
86 - torch.bfloat16, azp_adj, None, bias))
87 -
88 - # cutlass with azp per-token
89 - timers.append(
90 - bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt",
91 - ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
92 - torch.bfloat16, azp_adj, azp))
93 -
94 - # cutlass with azp per-token + bias
95 - timers.append(
96 - bench_fn(label, sub_label, "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias",
97 - ops.cutlass_scaled_mm_azp, a, b, scale_a, scale_b,
98 - torch.bfloat16, azp_adj, azp, bias))
86 + for name, fn in bench_fns.items():
87 + # If bench_kernels is None, run all. Otherwise, run only exact matches.
88 + if bench_kernels is None or name in bench_kernels:
89 + print(f"Running {name}")
90 + timers.append(bench_fn(label, sub_label, name, fn))
91
92 return timers
93
94
103 -def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
104 - sub_label: str) -> Iterable[TMeasurement]:
95 +def bench_fp8(
96 + dtype: torch.dtype,
97 + m: int,
98 + k: int,
99 + n: int,
100 + label: str,
101 + sub_label: str,
102 + bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
103 + """Benchmark FP8-based kernels."""
104 assert dtype == torch.float8_e4m3fn
105 a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)
106 + a_cont = a.contiguous()
107 scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
108 scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
109 + block_scale_a = torch.rand((m, k // 128),
110 + device="cuda",
111 + dtype=torch.float32)
112 + block_scale_b = torch.rand((k // 128, n // 128),
113 + device="cuda",
114 + dtype=torch.float32)
115 + block_scale_a_M_major = block_scale_a.t().contiguous().t()
116 + block_scale_b_K_major = block_scale_b.t().contiguous().t()
117 bias = torch.zeros((n, ), device="cuda", dtype=torch.bfloat16)
118
111 - timers = []
119 + print(m, k, n)
120 +
121 + bench_fns = {
122 + "pytorch_bf16_bf16_bf16_matmul-no-scales":
123 + lambda: torch.mm(a.to(dtype=torch.bfloat16), b.to(dtype=torch.bfloat16)
124 + ),
125 + "pytorch_fp16_fp16_fp16_matmul-no-scales":
126 + lambda: torch.mm(a.to(dtype=torch.float16), b.to(dtype=torch.float16)),
127 + "pytorch_fp8_fp8_fp16_scaled_mm":
128 + lambda: torch._scaled_mm(
129 + a, b, scale_a, scale_b, out_dtype=torch.float16),
130 + "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum":
131 + lambda: torch._scaled_mm(a,
132 + b,
133 + scale_a,
134 + scale_b,
135 + out_dtype=torch.float16,
136 + use_fast_accum=True),
137 + "pytorch_fp8_fp8_bf16_scaled_mm":
138 + lambda: torch._scaled_mm(
139 + a, b, scale_a, scale_b, out_dtype=torch.bfloat16),
140 + "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum":
141 + lambda: torch._scaled_mm(a,
142 + b,
143 + scale_a,
144 + scale_b,
145 + out_dtype=torch.bfloat16,
146 + use_fast_accum=True),
147 + "cutlass_fp8_fp8_bf16_scaled_mm":
148 + lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16),
149 + "cutlass_fp8_fp8_fp16_scaled_mm":
150 + lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.float16),
151 + "cutlass_fp8_fp8_bf16_scaled_mm_bias":
152 + lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16,
153 + bias),
154 + "cutlass_fp8_fp8_fp16_scaled_mm_bias":
155 + lambda: ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.float16,
156 + bias.to(dtype=torch.float16)),
157 + "triton_fp8_fp8_fp16_scaled_mm_blockwise":
158 + lambda: w8a8_block_fp8_matmul(a_cont, b.t(), block_scale_a,
159 + block_scale_b.t(), (128, 128)),
160 + "cutlass_fp8_fp8_fp16_scaled_mm_blockwise":
161 + lambda: ops.cutlass_scaled_mm(a, b, block_scale_a_M_major,
162 + block_scale_b_K_major, torch.float16),
163 + }
164
113 - # pytorch impl w. bf16
114 - timers.append(
115 - bench_fn(label, sub_label, "pytorch_bf16_bf16_bf16_matmul-no-scales",
116 - torch.mm, a.to(dtype=torch.bfloat16, device="cuda"),
117 - b.to(dtype=torch.bfloat16, device="cuda")))
118 -
119 - # pytorch impl: bf16 output, without fp8 fast accum
120 - timers.append(
121 - bench_fn(label,
122 - sub_label,
123 - "pytorch_fp8_fp8_bf16_scaled_mm",
124 - torch._scaled_mm,
125 - a,
126 - b,
127 - scale_a=scale_a,
128 - scale_b=scale_b,
129 - out_dtype=torch.bfloat16))
130 -
131 - # pytorch impl: bf16 output, with fp8 fast accum
132 - timers.append(
133 - bench_fn(label,
134 - sub_label,
135 - "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
136 - torch._scaled_mm,
137 - a,
138 - b,
139 - scale_a=scale_a,
140 - scale_b=scale_b,
141 - out_dtype=torch.bfloat16,
142 - use_fast_accum=True))
143 -
144 - # pytorch impl: fp16 output, without fp8 fast accum
145 - timers.append(
146 - bench_fn(label,
147 - sub_label,
148 - "pytorch_fp8_fp8_fp16_scaled_mm",
149 - torch._scaled_mm,
150 - a,
151 - b,
152 - scale_a=scale_a,
153 - scale_b=scale_b,
154 - out_dtype=torch.float16))
155 -
156 - # pytorch impl: fp16 output, with fp8 fast accum
157 - timers.append(
158 - bench_fn(label,
159 - sub_label,
160 - "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
161 - torch._scaled_mm,
162 - a,
163 - b,
164 - scale_a=scale_a,
165 - scale_b=scale_b,
166 - out_dtype=torch.float16,
167 - use_fast_accum=True))
168 -
169 - # cutlass impl: bf16 output
170 - timers.append(
171 - bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm",
172 - ops.cutlass_scaled_mm, a, b, scale_a, scale_b,
173 - torch.bfloat16))
174 - # cutlass impl: fp16 output
175 - timers.append(
176 - bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm",
177 - ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16))
178 -
179 - # cutlass impl: bf16 output, with bias
180 - timers.append(
181 - bench_fn(label, sub_label, "cutlass_fp8_fp8_bf16_scaled_mm_bias",
182 - ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.bfloat16,
183 - bias))
184 -
185 - # cutlass impl: fp16 output, with bias
186 - timers.append(
187 - bench_fn(label, sub_label, "cutlass_fp8_fp8_fp16_scaled_mm_bias",
188 - ops.cutlass_scaled_mm, a, b, scale_a, scale_b, torch.float16,
189 - bias.to(dtype=torch.float16)))
165 + timers = []
166 + for name, fn in bench_fns.items():
167 + # If bench_kernels is None, run all. Otherwise, run only exact matches.
168 + if bench_kernels is None or name in bench_kernels:
169 + print(f"Running {name}")
170 + timers.append(bench_fn(label, sub_label, name, fn))
171
172 return timers
173
174
194 -def bench(dtype: torch.dtype, m: int, k: int, n: int, label: str,
195 - sub_label: str) -> Iterable[TMeasurement]:
175 +def bench(dtype: torch.dtype,
176 + m: int,
177 + k: int,
178 + n: int,
179 + label: str,
180 + sub_label: str,
181 + bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
182 if dtype == torch.int8:
197 - return bench_int8(dtype, m, k, n, label, sub_label)
183 + return bench_int8(dtype, m, k, n, label, sub_label, bench_kernels)
184 if dtype == torch.float8_e4m3fn:
199 - return bench_fp8(dtype, m, k, n, label, sub_label)
185 + return bench_fp8(dtype, m, k, n, label, sub_label, bench_kernels)
186 raise ValueError("unsupported type")
187
188
@@ -207,18 +193,22 @@ def print_timers(timers: Iterable[TMeasurement]):
193
194
195 def run(dtype: torch.dtype,
210 - MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
196 + MKNs: Iterable[Tuple[int, int, int]],
197 + bench_kernels: Optional[List[str]] = None) -> Iterable[TMeasurement]:
198 results = []
199 for m, k, n in MKNs:
213 - timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
214 - f"MKN=({m}x{k}x{n})")
200 + timers = bench(dtype,
201 + m,
202 + k,
203 + n,
204 + f"scaled-{dtype}-gemm",
205 + f"MKN=({m}x{k}x{n})",
206 + bench_kernels=bench_kernels)
207 print_timers(timers)
208 results.extend(timers)
217 -
209 return results
210
211
221 -# output makers
212 def make_output(data: Iterable[TMeasurement],
213 MKNs: Iterable[Tuple[int, int, int]],
214 base_description: str,
@@ -232,15 +222,11 @@ def make_output(data: Iterable[TMeasurement],
222 pkl.dump(data, f)
223
224
235 -# argparse runners
236 -
237 -
225 def run_square_bench(args):
226 dim_sizes = list(
227 range(args.dim_start, args.dim_end + 1, args.dim_increment))
228 MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
242 - data = run(args.dtype, MKNs)
243 -
229 + data = run(args.dtype, MKNs, bench_kernels=args.kernels)
230 make_output(data, MKNs, f"square_bench-{args.dtype}")
231
232
@@ -251,8 +237,7 @@ def run_range_bench(args):
237 Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
238 Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
239 MKNs = list(zip(Ms, Ks, Ns))
254 - data = run(args.dtype, MKNs)
255 -
240 + data = run(args.dtype, MKNs, bench_kernels=args.kernels)
241 make_output(data, MKNs, f"range_bench-{args.dtype}")
242
243
@@ -278,7 +263,7 @@ def run_model_bench(args):
263 for k, n in KNs:
264 MKNs.append((m, k, n))
265
281 - data = run(args.dtype, MKNs)
266 + data = run(args.dtype, MKNs, bench_kernels=args.kernels)
267 model_bench_data.append(data)
268
269 # Print all results
@@ -328,6 +313,15 @@ Benchmark Cutlass GEMM.
313 type=to_torch_dtype,
314 required=True,
315 help="Available options are ['int8', 'fp8']")
316 + parser.add_argument(
317 + "--kernels",
318 + nargs="+",
319 + type=str,
320 + default=None,
321 + help=
322 + "Exact names of the kernels to benchmark. If not set, runs all kernels."
323 + )
324 +
325 subparsers = parser.add_subparsers(dest="cmd")
326
327 square_parser = subparsers.add_parser("square_bench")
@@ -362,4 +356,4 @@ Benchmark Cutlass GEMM.
356 model_parser.set_defaults(func=run_model_bench)
357
358 args = parser.parse_args()
365 - args.func(args)
\ No newline at end of file
359 + args.func(args)
csrc/core/math.hpp
+8 -1
@@ -1,7 +1,14 @@
1 +#pragma once
2 +
3 #include <climits>
4 #include <iostream>
5
4 -inline uint32_t next_pow_2(uint32_t const num) {
6 +inline constexpr uint32_t next_pow_2(uint32_t const num) {
7 if (num <= 1) return num;
8 return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
9 +}
10 +
11 +template <typename T>
12 +inline constexpr std::enable_if_t<std::is_integral_v<T>, T> ceil_div(T a, T b) {
13 + return (a + b - 1) / b;
14 }
\ No newline at end of file
csrc/cutlass_extensions/common.hpp
+17
@@ -32,3 +32,20 @@ inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
32 }
33
34 int32_t get_sm_version_num();
35 +
36 +/**
37 + * A wrapper for a kernel that is used to guard against compilation on
38 + * architectures that will never use the kernel. The purpose of this is to
39 + * reduce the size of the compiled binary.
40 + * __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
41 + * into code that will be executed on the device where it is defined.
42 + */
43 +template <typename Kernel>
44 +struct enable_sm90_or_later : Kernel {
45 + template <typename... Args>
46 + CUTLASS_DEVICE void operator()(Args&&... args) {
47 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
48 + Kernel::operator()(std::forward<Args>(args)...);
49 +#endif
50 + }
51 +};
\ No newline at end of file
csrc/cutlass_extensions/gemm/collective/collective_builder.hpp new
+123
@@ -0,0 +1,123 @@
1 +// Modified from: cutlass/gemm/collective/builders/sm90_gmma_builder.inl
2 +// clang-format off
3 +#pragma once
4 +
5 +#include "cutlass/gemm/collective/builders/sm90_gmma_builder.inl"
6 +
7 +#include "cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp"
8 +
9 +
10 +/////////////////////////////////////////////////////////////////////////////////////////////////
11 +
12 +namespace cutlass::gemm::collective {
13 +
14 +/////////////////////////////////////////////////////////////////////////////////////////////////
15 +
16 +// GMMA_TMA_WS_SS (BlockScaled Builders)
17 +template <
18 + class ElementA,
19 + class GmemLayoutATag,
20 + int AlignmentA,
21 + class ElementB,
22 + class GmemLayoutBTag,
23 + int AlignmentB,
24 + class ElementAccumulator,
25 + class TileShape_MNK,
26 + class ClusterShape_MNK,
27 + class StageCountType,
28 + int ScaleGranularityM
29 +>
30 +struct CollectiveBuilder<
31 + arch::Sm90,
32 + arch::OpClassTensorOp,
33 + ElementA,
34 + GmemLayoutATag,
35 + AlignmentA,
36 + ElementB,
37 + GmemLayoutBTag,
38 + AlignmentB,
39 + ElementAccumulator,
40 + TileShape_MNK,
41 + ClusterShape_MNK,
42 + StageCountType,
43 + KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<ScaleGranularityM>,
44 + cute::enable_if_t<
45 + not detail::is_use_rmem_A<ElementA, GmemLayoutATag, ElementB, GmemLayoutBTag>()>
46 +> {
47 + using KernelScheduleType = KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<ScaleGranularityM>;
48 +
49 + static_assert(is_static<TileShape_MNK>::value);
50 + static_assert(is_static<ClusterShape_MNK>::value);
51 +#ifndef CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED
52 + static_assert(cutlass::detail::dependent_false<ElementA>, "Unsupported Toolkit for SM90 Collective Builder\n");
53 +#endif
54 + static_assert(detail::is_aligned<ElementA, AlignmentA, ElementB, AlignmentB, detail::tma_alignment_bytes>(),
55 + "Should meet TMA alignment requirement\n");
56 +
57 + static constexpr bool IsArrayOfPointersGemm = (cute::is_any_of_v<KernelScheduleType,
58 + KernelPtrArrayTmaWarpSpecializedCooperative,
59 + KernelPtrArrayTmaWarpSpecializedPingpong>);
60 + static constexpr bool IsFP8Input = detail::is_input_fp8<ElementA, ElementB>();
61 + static_assert((!IsFP8Input || !IsArrayOfPointersGemm),
62 + "KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum is only compatible with FP8 Blocked Scaled version right now.");
63 +
64 + // For fp32 types, map to tf32 MMA value type
65 + using ElementAMma = cute::conditional_t<cute::is_same_v<ElementA, float>, tfloat32_t, ElementA>;
66 + using ElementBMma = cute::conditional_t<cute::is_same_v<ElementB, float>, tfloat32_t, ElementB>;
67 +
68 + static constexpr cute::GMMA::Major GmmaMajorA = detail::gmma_ss_tag_to_major_A<ElementAMma, GmemLayoutATag>();
69 + static constexpr cute::GMMA::Major GmmaMajorB = detail::gmma_ss_tag_to_major_B<ElementBMma, GmemLayoutBTag>();
70 +
71 + static constexpr bool IsCooperative = cute::is_any_of_v<KernelScheduleType,
72 + KernelTmaWarpSpecializedCooperative,
73 + KernelPtrArrayTmaWarpSpecializedCooperative,
74 + KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<ScaleGranularityM>>;
75 + using AtomLayoutMNK = cute::conditional_t<IsCooperative,
76 + Layout<Shape<_2,_1,_1>>, Layout<Shape<_1,_1,_1>>>;
77 +
78 + using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::ss_op_selector<
79 + ElementAMma, ElementBMma, ElementAccumulator, TileShape_MNK, GmmaMajorA, GmmaMajorB>(), AtomLayoutMNK{}));
80 +
81 + using GmemTiledCopyA = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
82 + using GmemTiledCopyB = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
83 +
84 + using SmemLayoutAtomA = decltype(detail::ss_smem_selector<
85 + GmmaMajorA, ElementAMma, decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
86 + using SmemLayoutAtomB = decltype(detail::ss_smem_selector<
87 + GmmaMajorB, ElementBMma, decltype(cute::get<1>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
88 +
89 + static constexpr size_t TensorMapStorage = IsArrayOfPointersGemm ? sizeof(cute::TmaDescriptor) * 2 /* for A and B */ : 0;
90 + static constexpr int KernelSmemCarveout = static_cast<int>(TensorMapStorage);
91 +
92 + static constexpr int PipelineStages = detail::compute_stage_count_or_override<detail::sm90_smem_capacity_bytes - KernelSmemCarveout,
93 + ElementAMma, ElementBMma, TileShape_MNK>(StageCountType{});
94 + using DispatchPolicy = MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8<PipelineStages, ClusterShape_MNK, KernelScheduleType, ScaleGranularityM>;
95 +
96 + using SmemCopyAtomA = void;
97 + using SmemCopyAtomB = void;
98 +
99 + using CollectiveOp = CollectiveMma<
100 + DispatchPolicy,
101 + TileShape_MNK,
102 + ElementA,
103 + TagToStrideA_t<GmemLayoutATag>,
104 + ElementB,
105 + TagToStrideB_t<GmemLayoutBTag>,
106 + TiledMma,
107 + GmemTiledCopyA,
108 + SmemLayoutAtomA,
109 + SmemCopyAtomA,
110 + cute::identity,
111 + GmemTiledCopyB,
112 + SmemLayoutAtomB,
113 + SmemCopyAtomB,
114 + cute::identity
115 + >;
116 +};
117 +
118 +
119 +/////////////////////////////////////////////////////////////////////////////////////////////////
120 +
121 +} // namespace cutlass::gemm::collective
122 +
123 +/////////////////////////////////////////////////////////////////////////////////////////////////
csrc/cutlass_extensions/gemm/collective/fp8_accumulation.hpp new
+183
@@ -0,0 +1,183 @@
1 +// clang-format off
2 +// adapted from: https://github.com/soundOfDestiny/cutlass/blob/a4208aa6958864923505cade9c63eb2a6daf16e5/include/cutlass/gemm/collective/fp8_accumulation.hpp
3 +
4 +/***************************************************************************************************
5 + * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6 + * SPDX-License-Identifier: BSD-3-Clause
7 + *
8 + * Redistribution and use in source and binary forms, with or without
9 + * modification, are permitted provided that the following conditions are met:
10 + *
11 + * 1. Redistributions of source code must retain the above copyright notice, this
12 + * list of conditions and the following disclaimer.
13 + *
14 + * 2. Redistributions in binary form must reproduce the above copyright notice,
15 + * this list of conditions and the following disclaimer in the documentation
16 + * and/or other materials provided with the distribution.
17 + *
18 + * 3. Neither the name of the copyright holder nor the names of its
19 + * contributors may be used to endorse or promote products derived from
20 + * this software without specific prior written permission.
21 + *
22 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25 + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
26 + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
28 + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 + *
33 + **************************************************************************************************/
34 +
35 +#pragma once
36 +
37 +#include "cute/algorithm/clear.hpp"
38 +#include "cute/tensor.hpp"
39 +
40 +//////////////////////////////////////////////////////////////////////////////
41 +///////////////////////////////////FP8 Accumulation///////////////////////////
42 +//////////////////////////////////////////////////////////////////////////////
43 +/// This class provides API to promote (add) or scale (multiply_add) the results
44 +/// from the tensor core accumulators to the main accumulators when the number
45 +/// of MMAs reaches the max number of MMA interval specified by user, after that
46 +/// the tensor core accumulators are zeroed.
47 +//////////////////////////////////////////////////////////////////////////////
48 +
49 +namespace cutlass::gemm::collective {
50 +
51 +template <
52 + class EngineAccum,
53 + class LayoutAccum>
54 +struct GmmaFP8AccumulationWithScale {
55 + using TensorAccum = cute::Tensor<EngineAccum, LayoutAccum>;
56 + using ElementAccumulator = typename EngineAccum::value_type;
57 +
58 + static_assert(is_static<LayoutAccum>::value, "Accumulator Layout should be static");
59 + static_assert(is_rmem<TensorAccum>::value , "Accumulator tensor must be rmem resident.");
60 +
61 +private:
62 + TensorAccum& accum_;
63 + TensorAccum accum_temp_;
64 +
65 + uint32_t accum_promotion_interval_; // defines the max num of executed MMAs after which accum should be promoted.
66 + uint32_t mma_count_per_mainloop_iteration_; // num of MMAs per k_tile of mainloop
67 + uint32_t mma_count_; // current executed MMAs
68 + uint32_t reset_accum_flag_; // accum needs to be zeroed or not.
69 +
70 + // promote or `add` the partial accumulators to main accumulator (FADD).
71 + CUTLASS_DEVICE
72 + void promote_core() {
73 + warpgroup_wait<0>();
74 + CUTLASS_PRAGMA_UNROLL
75 + for (int i = 0; i < size(accum_); ++i) {
76 + accum_(i) += accum_temp_(i);
77 + }
78 + }
79 +
80 + // `multiply` scale the partial accumulators and `add` to main accumulator (FFMA).
81 + template <
82 + class EngineScale,
83 + class LayoutScale>
84 + CUTLASS_DEVICE
85 + void scale_core(const cute::Tensor<EngineScale, LayoutScale> &scale) {
86 + using TensorScale = cute::Tensor<EngineScale, LayoutScale>;
87 +
88 + static_assert(is_static<LayoutScale>::value, "Scale Layout should be static");
89 + static_assert(is_rmem<TensorScale>::value , "Scale tensor must be rmem resident.");
90 +
91 + static_assert(LayoutAccum{}.shape() == LayoutScale{}.shape(), "Accumulator and scale must have same shape.");
92 +
93 + warpgroup_wait<0>();
94 + CUTLASS_PRAGMA_UNROLL
95 + for (int i = 0; i < size(accum_); ++i) {
96 + accum_(i) += accum_temp_(i) * scale(i);
97 + }
98 + }
99 +
100 +public:
101 + CUTLASS_DEVICE
102 + GmmaFP8AccumulationWithScale(
103 + TensorAccum &accum,
104 + uint32_t accum_promotion_interval,
105 + uint32_t mma_count_per_mainloop_iteration)
106 + : accum_(accum),
107 + accum_promotion_interval_(accum_promotion_interval),
108 + mma_count_per_mainloop_iteration_(mma_count_per_mainloop_iteration),
109 + mma_count_(0),
110 + reset_accum_flag_(0)
111 + {
112 + accum_temp_ = cute::make_fragment_like(accum);
113 + }
114 +
115 + //
116 + // Methods (Common)
117 + //
118 +
119 + CUTLASS_DEVICE
120 + TensorAccum& operator()() {
121 + return accum_temp_;
122 + }
123 +
124 + /// prepare the MMA accumulators when initialization or zeroing is required.
125 + CUTLASS_DEVICE
126 + bool prepare_if_needed() {
127 + return reset_accum_flag_;
128 + }
129 +
130 + //
131 + // Methods (for FADD version)
132 + //
133 +
134 + /// promote (add) the results from the MMA accumulators to main accumulator if needed.
135 + CUTLASS_DEVICE
136 + void promote_if_needed() {
137 + mma_count_ += mma_count_per_mainloop_iteration_;
138 + reset_accum_flag_ = __shfl_sync(0xffffffff, mma_count_ == accum_promotion_interval_, 0);
139 + if (reset_accum_flag_) {
140 + promote_core();
141 + mma_count_ = 0;
142 + }
143 + }
144 +
145 + /// promote (add) the residue results from the MMA accumulators to main accumulator if needed.
146 + CUTLASS_DEVICE
147 + void promote_residue_if_needed() {
148 + if (__shfl_sync(0xffffffff, mma_count_ > 0, 0)) {
149 + promote_core();
150 + }
151 + }
152 +
153 + //
154 + // Methods (for FFMA version)
155 + //
156 +
157 + /// scale (multiply_add) the results from the MMA accumulators to main accumulator if needed.
158 + template <
159 + class EngineScale,
160 + class LayoutScale>
161 + CUTLASS_DEVICE
162 + void scale_if_needed(const cute::Tensor<EngineScale, LayoutScale> &scale) {
163 + mma_count_ += mma_count_per_mainloop_iteration_;
164 + reset_accum_flag_ = __shfl_sync(0xffffffff, mma_count_ == accum_promotion_interval_, 0);
165 + if (reset_accum_flag_) {
166 + scale_core(scale);
167 + mma_count_ = 0;
168 + }
169 + }
170 +
171 + /// scale (multiply_add) the residue results from the MMA accumulators to main accumulator if needed.
172 + template <
173 + class EngineScale,
174 + class LayoutScale>
175 + CUTLASS_DEVICE
176 + void scale_residue_if_needed(const cute::Tensor<EngineScale, LayoutScale> &scale) {
177 + if (__shfl_sync(0xffffffff, mma_count_ > 0, 0)) {
178 + scale_core(scale);
179 + }
180 + }
181 +};
182 +
183 +} // namespace cutlass::gemm::collective
csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp new
+730
@@ -0,0 +1,730 @@
1 +// clang-format off
2 +// Adapted (Heavily) from: https://github.com/soundOfDestiny/cutlass/blob/9d997ce0dea4c5fa1a617db6b7ff29aa9235822c/include/cutlass/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp
3 +
4 +/***************************************************************************************************
5 + * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6 + * SPDX-License-Identifier: BSD-3-Clause
7 + *
8 + * Redistribution and use in source and binary forms, with or without
9 + * modification, are permitted provided that the following conditions are met:
10 + *
11 + * 1. Redistributions of source code must retain the above copyright notice, this
12 + * list of conditions and the following disclaimer.
13 + *
14 + * 2. Redistributions in binary form must reproduce the above copyright notice,
15 + * this list of conditions and the following disclaimer in the documentation
16 + * and/or other materials provided with the distribution.
17 + *
18 + * 3. Neither the name of the copyright holder nor the names of its
19 + * contributors may be used to endorse or promote products derived from
20 + * this software without specific prior written permission.
21 + *
22 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25 + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
26 + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
28 + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 + *
33 + **************************************************************************************************/
34 +
35 +#pragma once
36 +
37 +#include "cutlass/cutlass.h"
38 +#include "cutlass/gemm/dispatch_policy.hpp"
39 +#include "cutlass/trace.h"
40 +#include "cutlass/numeric_types.h"
41 +
42 +#include "cute/arch/cluster_sm90.hpp"
43 +#include "cute/arch/copy_sm80.hpp"
44 +#include "cute/arch/copy_sm90.hpp"
45 +#include "cute/algorithm/functional.hpp"
46 +#include "cute/atom/mma_atom.hpp"
47 +#include "cute/algorithm/gemm.hpp"
48 +#include "cute/tensor_predicate.hpp"
49 +#include "cute/numeric/arithmetic_tuple.hpp"
50 +
51 +#include "cutlass_extensions/gemm/dispatch_policy.hpp"
52 +#include "cutlass_extensions/gemm/collective/fp8_accumulation.hpp"
53 +
54 +/////////////////////////////////////////////////////////////////////////////////////////////////
55 +
56 +namespace cutlass::gemm::collective {
57 +using namespace cute;
58 +
59 +/////////////////////////////////////////////////////////////////////////////////////////////////
60 +
61 +// WarpSpecialized Mainloop
62 +template <
63 + int Stages,
64 + class ClusterShape,
65 + class KernelSchedule,
66 + int ScaleGranularityM_,
67 + class TileShape_,
68 + class ElementA_,
69 + class StrideA_,
70 + class ElementB_,
71 + class StrideB_,
72 + class TiledMma_,
73 + class GmemTiledCopyA_,
74 + class SmemLayoutAtomA_,
75 + class SmemCopyAtomA_,
76 + class TransformA_,
77 + class GmemTiledCopyB_,
78 + class SmemLayoutAtomB_,
79 + class SmemCopyAtomB_,
80 + class TransformB_>
81 +struct CollectiveMma<
82 + MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8<Stages, ClusterShape, KernelSchedule, ScaleGranularityM_>,
83 + TileShape_,
84 + ElementA_,
85 + StrideA_,
86 + ElementB_,
87 + StrideB_,
88 + TiledMma_,
89 + GmemTiledCopyA_,
90 + SmemLayoutAtomA_,
91 + SmemCopyAtomA_,
92 + TransformA_,
93 + GmemTiledCopyB_,
94 + SmemLayoutAtomB_,
95 + SmemCopyAtomB_,
96 + TransformB_>
97 +{
98 + //
99 + // Type Aliases
100 + //
101 + using DispatchPolicy = MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8<Stages, ClusterShape, KernelSchedule, ScaleGranularityM_>;
102 + using TileShape = TileShape_;
103 + using ElementA = ElementA_;
104 + using StrideA = StrideA_;
105 + using ElementB = ElementB_;
106 + using StrideB = StrideB_;
107 + using TiledMma = TiledMma_;
108 + using ElementAccumulator = typename TiledMma::ValTypeC;
109 + using ElementBlockScale = ElementAccumulator;
110 + using GmemTiledCopyA = GmemTiledCopyA_;
111 + using GmemTiledCopyB = GmemTiledCopyB_;
112 + using SmemLayoutAtomA = SmemLayoutAtomA_;
113 + using SmemLayoutAtomB = SmemLayoutAtomB_;
114 + using SmemCopyAtomA = SmemCopyAtomA_;
115 + using SmemCopyAtomB = SmemCopyAtomB_;
116 + using TransformA = TransformA_;
117 + using TransformB = TransformB_;
118 + using ArchTag = typename DispatchPolicy::ArchTag;
119 +
120 + using CtaShape_MNK = decltype(shape_div(TileShape{}, ClusterShape{}));
121 + using MainloopPipeline = cutlass::PipelineTmaAsync<DispatchPolicy::Stages>;
122 + using PipelineState = cutlass::PipelineState<DispatchPolicy::Stages>;
123 + using PipelineParams = typename MainloopPipeline::Params;
124 +
125 + // Two threads per CTA are producers (1 for operand tile and 32 for scales)
126 + static constexpr int NumProducerThreadEvents = 33;
127 +
128 + static constexpr int ScaleGranularityM = ScaleGranularityM_ == 0 ? size<0>(TileShape{}) : ScaleGranularityM_;
129 + static constexpr int ScaleMsPerTile = size<0>(TileShape{}) / ScaleGranularityM;
130 +
131 + static_assert(cute::rank(SmemLayoutAtomA{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)");
132 + static_assert((size<0>(TileShape{}) % size<0>(SmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
133 + static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
134 +
135 + static_assert(cute::rank(SmemLayoutAtomB{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)");
136 + static_assert((size<1>(TileShape{}) % size<0>(SmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
137 + static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape.");
138 +
139 + static_assert((size<0>(TileShape{}) % ScaleGranularityM) == 0, "FP8 scaling granularity must evenly divide tile shape along M.");
140 +
141 + // Tile along modes in a way that maximizes the TMA box size.
142 + using SmemLayoutA = decltype(tile_to_shape(
143 + SmemLayoutAtomA{},
144 + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int<DispatchPolicy::Stages>{}),
145 + cute::conditional_t< ::cutlass::gemm::detail::is_major<0,StrideA>(), Step<_2,_1,_3>, Step<_1,_2,_3>>{}));
146 + using SmemLayoutB = decltype(tile_to_shape(
147 + SmemLayoutAtomB{},
148 + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int<DispatchPolicy::Stages>{}),
149 + cute::conditional_t< ::cutlass::gemm::detail::is_major<0,StrideB>(), Step<_2,_1,_3>, Step<_1,_2,_3>>{}));
150 +
151 + // Block scaling gmem-to-smem copy atom
152 + using SmemBlockScalingCopyAtomA = Copy_Atom<SM80_CP_ASYNC_CACHEALWAYS<ElementBlockScale>, ElementBlockScale>;
153 + using SmemBlockScalingCopyAtomB = Copy_Atom<SM80_CP_ASYNC_CACHEALWAYS<ElementBlockScale>, ElementBlockScale>;
154 +
155 + // Block scaling smem layout
156 + using SmemLayoutScaleA = Layout<Shape<Int<ScaleMsPerTile>, Int<DispatchPolicy::Stages>>>;
157 + using SmemLayoutScaleB = Layout<Shape<Int<DispatchPolicy::Stages>>, Stride<_1>>; // `ScaleNsPerTile` is always 1.
158 +
159 + static_assert(DispatchPolicy::Stages >= 2, "Specialization requires Stages set to value 1 or more.");
160 + static_assert(cute::is_base_of<cute::GMMA::DescriptorIterator, typename TiledMma::FrgTypeA>::value &&
161 + cute::is_base_of<cute::GMMA::DescriptorIterator, typename TiledMma::FrgTypeB>::value,
162 + "MMA atom must source both A and B operand from smem_desc for this mainloop.");
163 + static_assert(cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD> || cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD_MULTICAST>,
164 + "GmemTiledCopy - invalid SM90 TMA copy atom specified.");
165 + static_assert(cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD> || cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD_MULTICAST>,
166 + "GmemTiledCopy - invalid SM90 TMA copy atom specified.");
167 + static_assert(cute::is_same_v<ElementAccumulator, ElementBlockScale>,
168 + "ElementAccumulator and ElementBlockScale should be same datatype");
169 +
170 + struct SharedStorage
171 + {
172 + struct TensorStorage : cute::aligned_struct<128> {
173 + cute::array_aligned<typename TiledMma::ValTypeA, cute::cosize_v<SmemLayoutA>> smem_A; // mxk
174 + cute::array_aligned<typename TiledMma::ValTypeB, cute::cosize_v<SmemLayoutB>> smem_B; // nxk
175 + cute::array_aligned<ElementBlockScale, cute::cosize_v<SmemLayoutScaleA>> smem_scale_A; // ScaleMsPerTile x k
176 + cute::array_aligned<ElementBlockScale, cute::cosize_v<SmemLayoutScaleB>> smem_scale_B; // 1xk
177 + } tensors;
178 +
179 + using PipelineStorage = typename MainloopPipeline::SharedStorage;
180 + PipelineStorage pipeline;
181 + };
182 + using TensorStorage = typename SharedStorage::TensorStorage;
183 + using PipelineStorage = typename SharedStorage::PipelineStorage;
184 +
185 + // Host side kernel arguments
186 + struct Arguments {
187 + ElementA const* ptr_A;
188 + StrideA dA;
189 + ElementB const* ptr_B;
190 + StrideB dB;
191 + ElementBlockScale const* ptr_scale_A;
192 + ElementBlockScale const* ptr_scale_B;
193 + };
194 +
195 + // Device side kernel params
196 + struct Params {
197 + // Assumption: StrideA is congruent with Problem_MK
198 + using TMA_A = decltype(make_tma_copy_A_sm90(
199 + GmemTiledCopyA{},
200 + make_tensor(static_cast<ElementA const*>(nullptr), repeat_like(StrideA{}, int32_t(0)), StrideA{}),
201 + SmemLayoutA{}(_,_,0),
202 + TileShape{},
203 + ClusterShape{}));
204 + // Assumption: StrideB is congruent with Problem_NK
205 + using TMA_B = decltype(make_tma_copy_B_sm90(
206 + GmemTiledCopyB{},
207 + make_tensor(static_cast<ElementB const*>(nullptr), repeat_like(StrideB{}, int32_t(0)), StrideB{}),
208 + SmemLayoutB{}(_,_,0),
209 + TileShape{},
210 + ClusterShape{}));
211 + TMA_A tma_load_a;
212 + TMA_B tma_load_b;
213 + uint32_t tma_transaction_bytes = TmaTransactionBytes;
214 + uint32_t tma_transaction_bytes_mk = TmaTransactionBytesMK;
215 + uint32_t tma_transaction_bytes_nk = TmaTransactionBytesNK;
216 + // Block scaling factors for A and B
217 + ElementBlockScale const* ptr_scale_A;
218 + ElementBlockScale const* ptr_scale_B;
219 + };
220 +
221 + //
222 + // Methods
223 + //
224 +
225 + template <class ProblemShape>
226 + static constexpr Params
227 + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
228 + (void) workspace;
229 +
230 + // Optionally append 1s until problem shape is rank-4 (MNKL), in case it is only rank-3 (MNK)
231 + auto problem_shape_MNKL = append<4>(problem_shape, 1);
232 + auto [M,N,K,L] = problem_shape_MNKL;
233 +
234 + auto ptr_A = reinterpret_cast<ElementA const*>(args.ptr_A);
235 + auto ptr_B = reinterpret_cast<ElementB const*>(args.ptr_B);
236 +
237 + Tensor tensor_a = make_tensor(ptr_A, make_layout(make_shape(M,K,L), args.dA));
238 + Tensor tensor_b = make_tensor(ptr_B, make_layout(make_shape(N,K,L), args.dB));
239 + typename Params::TMA_A tma_load_a = make_tma_copy_A_sm90(
240 + GmemTiledCopyA{},
241 + tensor_a,
242 + SmemLayoutA{}(_,_,cute::Int<0>{}),
243 + TileShape{},
244 + ClusterShape{});
245 + typename Params::TMA_B tma_load_b = make_tma_copy_B_sm90(
246 + GmemTiledCopyB{},
247 + tensor_b,
248 + SmemLayoutB{}(_,_,cute::Int<0>{}),
249 + TileShape{},
250 + ClusterShape{});
251 + uint32_t transaction_bytes_mk = TmaTransactionBytesMK;
252 + uint32_t transaction_bytes_nk = TmaTransactionBytesNK;
253 + uint32_t transaction_bytes = transaction_bytes_mk + transaction_bytes_nk;
254 +
255 + return {
256 + tma_load_a,
257 + tma_load_b,
258 + transaction_bytes,
259 + transaction_bytes_mk,
260 + transaction_bytes_nk,
261 + args.ptr_scale_A,
262 + args.ptr_scale_B
263 + };
264 + }
265 +
266 + template<class ProblemShape>
267 + static bool
268 + can_implement(
269 + ProblemShape const& problem_shape,
270 + [[maybe_unused]] Arguments const& args) {
271 + constexpr int tma_alignment_bits = 128;
272 + auto problem_shape_MNKL = append<4>(problem_shape, 1);
273 + auto [M,N,K,L] = problem_shape_MNKL;
274 +
275 + bool implementable = true;
276 + constexpr int min_tma_aligned_elements_A = tma_alignment_bits / cutlass::sizeof_bits<ElementA>::value;
277 + implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_A>(cute::make_shape(M,K,L), StrideA{});
278 + constexpr int min_tma_aligned_elements_B = tma_alignment_bits / cutlass::sizeof_bits<ElementB>::value;
279 + implementable = implementable && cutlass::detail::check_alignment<min_tma_aligned_elements_B>(cute::make_shape(N,K,L), StrideB{});
280 +
281 + if (!implementable) {
282 + CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Problem Size doesn't meet the minimum alignment requirements for TMA.\n");
283 + }
284 + return implementable;
285 + }
286 +
287 + static constexpr int K_PIPE_MAX = DispatchPolicy::Stages;
288 + static constexpr int K_PIPE_MMAS = 1;
289 + static constexpr uint32_t TmaTransactionBytesMK =
290 + cutlass::bits_to_bytes(size<0>(SmemLayoutA{}) * size<1>(SmemLayoutA{}) * static_cast<uint32_t>(sizeof_bits<ElementA>::value));
291 + static constexpr uint32_t TmaTransactionBytesNK =
292 + cutlass::bits_to_bytes(size<0>(SmemLayoutB{}) * size<1>(SmemLayoutB{}) * static_cast<uint32_t>(sizeof_bits<ElementB>::value));
293 + static constexpr uint32_t TmaTransactionBytes = TmaTransactionBytesMK + TmaTransactionBytesNK;
294 +
295 + /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
296 + CUTLASS_DEVICE
297 + static void prefetch_tma_descriptors(Params const& mainloop_params)
298 + {
299 + cute::prefetch_tma_descriptor(mainloop_params.tma_load_a.get_tma_descriptor());
300 + cute::prefetch_tma_descriptor(mainloop_params.tma_load_b.get_tma_descriptor());
301 + }
302 +
303 + /// Set up the data needed by this collective for load and mma.
304 + /// Returns a tuple of tensors. The collective and the kernel layer have the contract
305 + /// Returned tuple must contain at least two elements, with the first two elements being:
306 + /// gA_mkl - The tma tensor, A after a local tile so it has shape (BLK_M,BLK_K,m,k,l)
307 + /// gB_nkl - The tma tensor, B after a local tile so it has shape (BLK_N,BLK_K,n,k,l)
308 + template <class ProblemShape_MNKL>
309 + CUTLASS_DEVICE auto
310 + load_init(ProblemShape_MNKL const& problem_shape_MNKL, Params const& mainloop_params) const {
311 + using X = Underscore;
312 + // Separate out problem shape for convenience
313 + auto [M,N,K,L] = problem_shape_MNKL;
314 +
315 + // TMA requires special handling of strides to deal with coord codomain mapping
316 + // Represent the full tensors -- get these from TMA
317 + Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(make_shape(M,K,L)); // (m,k,l)
318 + Tensor mB_nkl = mainloop_params.tma_load_b.get_tma_tensor(make_shape(N,K,L)); // (n,k,l)
319 +
320 + // Make tiled views, defer the slice
321 + Tensor gA_mkl = local_tile(mA_mkl, TileShape{}, make_coord(_,_,_), Step<_1, X,_1>{}); // (BLK_M,BLK_K,m,k,l)
322 + Tensor gB_nkl = local_tile(mB_nkl, TileShape{}, make_coord(_,_,_), Step< X,_1,_1>{}); // (BLK_N,BLK_K,n,k,l)
323 +
324 + constexpr auto scales_m = Int<ScaleMsPerTile>{};
325 + auto tM = get<2>(gA_mkl.shape());
326 + auto tN = get<2>(gB_nkl.shape());
327 + auto tK = get<3>(gA_mkl.shape());
328 +
329 + // Make the tiled views of scale tensors
330 + auto scaleA_shape = make_shape(M / ScaleGranularityM, tK, L); // (scale_m,k,l)
331 + auto scaleA_layout = make_ordered_layout(scaleA_shape, Step<_0, _1, _2>{});
332 + auto scaleB_shape = make_shape(tN, tK, L); // (n,k,l)
333 + auto scaleB_layout = make_ordered_layout(scaleB_shape, Step<_1, _0, _2>{});
334 +
335 + // Note that mScaleA_mkl and mScaleB_nkl are already blocked tiled in the `m` host and
336 + // gScaleA_mkl and gScaleB_nkl in `g` global memory are same as mScaleA_mkl and mScaleB_nkl.
337 + Tensor mScaleA_mkl = make_tensor(make_gmem_ptr(mainloop_params.ptr_scale_A), scaleA_layout); // (scale_m,k,l)
338 + Tensor mScaleB_nkl = make_tensor(make_gmem_ptr(mainloop_params.ptr_scale_B), scaleB_layout); // (n,k,l)
339 +
340 + return cute::make_tuple(gA_mkl, gB_nkl, mScaleA_mkl, mScaleB_nkl);
341 + }
342 +
343 + /// Perform a collective-scoped matrix multiply-accumulate
344 + /// Producer Perspective
345 + template <
346 + class TensorA, class TensorB,
347 + class TensorScaleA, class TensorScaleB,
348 + class KTileIterator, class BlockCoord
349 + >
350 + CUTLASS_DEVICE void
351 + load(
352 + Params const& mainloop_params,
353 + MainloopPipeline pipeline,
354 + PipelineState smem_pipe_write,
355 + cute::tuple<TensorA, TensorB, TensorScaleA, TensorScaleB> const& load_inputs,
356 + BlockCoord const& blk_coord,
357 + KTileIterator k_tile_iter, int k_tile_count,
358 + int thread_idx,
359 + uint32_t block_rank_in_cluster,
360 + TensorStorage& shared_tensors) {
361 + int lane_predicate = cute::elect_one_sync();
362 +
363 + // Blockscaling: Tma loads for load_input and CpAsync for load_scale
364 + Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.data()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE)
365 + Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.data()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE)
366 + Tensor sScaleA = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_A.data()), SmemLayoutScaleA{}); // (ScaleMsPerTile,k)
367 + Tensor sScaleB = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_B.data()), SmemLayoutScaleB{}); // (k)
368 +
369 + //
370 + // Prepare the TMA loads for A and B
371 + //
372 +
373 + constexpr uint32_t cluster_shape_x = get<0>(ClusterShape());
374 + uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x};
375 +
376 + Tensor gA_mkl = get<0>(load_inputs);
377 + Tensor gB_nkl = get<1>(load_inputs);
378 +
379 + auto block_tma_a = mainloop_params.tma_load_a.get_slice(cluster_local_block_id.y);
380 + auto block_tma_b = mainloop_params.tma_load_b.get_slice(cluster_local_block_id.x);
381 +
382 + // Partition the inputs based on the current block coordinates.
383 + auto [m_coord, n_coord, k_coord, l_coord] = blk_coord;
384 + Tensor gA = gA_mkl(_,_,m_coord,_,l_coord); // (BLK_M,BLK_K,k)
385 + Tensor gB = gB_nkl(_,_,n_coord,_,l_coord); // (BLK_N,BLK_K,k)
386 +
387 +
388 + // Block scaling: load_scale has scaling tensors in global memory which are not tiled
389 + Tensor mScaleA_mkl = get<2>(load_inputs);
390 + Tensor mScaleB_nkl = get<3>(load_inputs);
391 + auto scales_m = get<0>(mScaleA_mkl.shape());
392 +
393 + Tensor cScaleA_mkl = make_identity_tensor(mScaleA_mkl.shape());
394 +
395 + Tensor gScaleA = local_tile(
396 + mScaleA_mkl, make_tile(Int<ScaleMsPerTile>{}),
397 + make_coord(m_coord,_,l_coord)); // (ScaleMsPerTile,k,1)
398 + Tensor cScaleA = local_tile(
399 + cScaleA_mkl, make_tile(Int<ScaleMsPerTile>{}),
400 + make_coord(m_coord,_,l_coord));
401 + Tensor gScaleB = mScaleB_nkl(n_coord,_,l_coord); // (1,k,1)
402 +
403 + // TODO: test `scale_copy_a` with `ScaleMsPerTile` < 128
404 + TiledCopy scale_copy_a = make_tiled_copy(SmemBlockScalingCopyAtomA{},
405 + Layout<Shape<_32, _1>>{}, Layout<Shape<_4, _1>>{}); // (1,1,1)
406 + TiledCopy scale_copy_b = make_tiled_copy(SmemBlockScalingCopyAtomB{},
407 + Layout<Shape<_1>>{}, Layout<Shape<_1>>{}); // (1,1,1)
408 + ThrCopy thr_scale_copy_a = scale_copy_a.get_slice(threadIdx.x);
409 + ThrCopy thr_scale_copy_b = scale_copy_b.get_slice(threadIdx.x);
410 +
411 + Tensor tAgA_ScaleA = thr_scale_copy_a.partition_S(gScaleA);
412 + Tensor tAcA_ScaleA = thr_scale_copy_a.partition_S(cScaleA);
413 + Tensor tAsA_ScaleA = thr_scale_copy_a.partition_D(sScaleA);
414 +
415 + Tensor tBgB_ScaleB = thr_scale_copy_b.partition_S(gScaleB);
416 + Tensor tBsB_ScaleB = thr_scale_copy_b.partition_D(sScaleB);
417 +
418 + // Applies the mapping from block_tma_a
419 + Tensor tAgA = block_tma_a.partition_S(gA); // (TMA,TMA_M,TMA_K,k)
420 + Tensor tAsA = block_tma_a.partition_D(sA); // (TMA,TMA_M,TMA_K,PIPE)
421 +
422 + Tensor tBgB = block_tma_b.partition_S(gB); // (TMA,TMA_N,TMA_K,k)
423 + Tensor tBsB = block_tma_b.partition_D(sB); // (TMA,TMA_N,TMA_K,PIPE)
424 +
425 + uint16_t mcast_mask_a = 0;
426 + uint16_t mcast_mask_b = 0;
427 +
428 + // Issue TmaLoads for GEMM operands A/B and CpAsync for scale tensors
429 + // Maps the tile -> block, value
430 + if constexpr (cute::is_same_v<GmemTiledCopyA, SM90_TMA_LOAD_MULTICAST>) {
431 + auto block_layout = Layout<typename DispatchPolicy::ClusterShape>{}; // (m,n) -> block_id
432 + for (int n = 0; n < size<1>(block_layout); ++n) {
433 + mcast_mask_a |= (uint16_t(1) << block_layout(cluster_local_block_id.x,n,Int<0>{}));
434 + }
435 + }
436 +
437 + if constexpr (cute::is_same_v<GmemTiledCopyB, SM90_TMA_LOAD_MULTICAST>) {
438 + auto block_layout = Layout<typename DispatchPolicy::ClusterShape>{}; // (m,n) -> block_id
439 + for (int m = 0; m < size<0>(block_layout); ++m) {
440 + mcast_mask_b |= (uint16_t(1) << block_layout(m,cluster_local_block_id.y,Int<0>{}));
441 + }
442 + }
443 +
444 + // Allocate predicate tensors for a_scales (since we can't guarantee that
445 + // all scales are valid, since we could have a partial tiles along M)
446 + Tensor tApA_ScaleA = make_tensor<bool>(shape(tAsA_ScaleA(_,_,0)));
447 + #pragma unroll
448 + for (int i = 0; i < size(tApA_ScaleA); ++i) {
449 + tApA_ScaleA(i) = get<0>(tAcA_ScaleA(i)) < scales_m;
450 + }
451 +
452 + // Mainloop
453 + CUTLASS_PRAGMA_NO_UNROLL
454 + for ( ; k_tile_count > 0; --k_tile_count) {
455 + // LOCK smem_pipe_write for _writing_
456 + pipeline.producer_acquire(smem_pipe_write);
457 +
458 + //
459 + // Copy gmem to smem for *k_tile_iter
460 + //
461 + int write_stage = smem_pipe_write.index();
462 + using BarrierType = typename MainloopPipeline::ProducerBarrierType;
463 + BarrierType* tma_barrier = pipeline.producer_get_barrier(smem_pipe_write);
464 +
465 + // Copy operands A and B from global memory to shared memory
466 + if (lane_predicate) copy(mainloop_params.tma_load_a.with(*tma_barrier, mcast_mask_a), tAgA(_,_,_,*k_tile_iter), tAsA(_,_,_,write_stage));
467 + if (lane_predicate) copy(mainloop_params.tma_load_b.with(*tma_barrier, mcast_mask_b), tBgB(_,_,_,*k_tile_iter), tBsB(_,_,_,write_stage));
468 +
469 + // Copy scale tensors from global memory to shared memory
470 + copy_if(scale_copy_a, tApA_ScaleA, tAgA_ScaleA(_,_,*k_tile_iter), tAsA_ScaleA(_,_,write_stage));
471 + copy(scale_copy_b, tBgB_ScaleB(_,*k_tile_iter), tBsB_ScaleB(_,write_stage));
472 + pipeline.producer_commit(smem_pipe_write, cutlass::arch::cpasync_barrier_arrive_noinc);
473 +
474 + ++k_tile_iter;
475 +
476 + // Advance smem_pipe_write
477 + ++smem_pipe_write;
478 + }
479 + }
480 +
481 + /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster
482 + CUTLASS_DEVICE void
483 + load_tail(
484 + MainloopPipeline pipeline,
485 + PipelineState smem_pipe_write) {
486 + int lane_predicate = cute::elect_one_sync();
487 +
488 + // Issue the epilogue waits
489 + if (lane_predicate) {
490 + /* This helps avoid early exit of blocks in Cluster
491 + * Waits for all stages to either be released (all
492 + * Consumer UNLOCKs), or if the stage was never used
493 + * then would just be acquired since the phase was
494 + * still inverted from make_producer_start_state
495 + */
496 + pipeline.producer_tail(smem_pipe_write);
497 + }
498 + }
499 +
500 + /// Perform a collective-scoped matrix multiply-accumulate
501 + /// Consumer Perspective
502 + template <
503 + class FrgTensorC
504 + >
505 + CUTLASS_DEVICE void
506 + mma(MainloopPipeline pipeline,
507 + PipelineState smem_pipe_read,
508 + FrgTensorC& accum,
509 + int k_tile_count,
510 + int thread_idx,
511 + TensorStorage& shared_tensors,
512 + Params const& mainloop_params) {
513 +
514 +
515 + static_assert(is_rmem<FrgTensorC>::value, "C tensor must be rmem resident.");
516 + static_assert(cute::rank(SmemLayoutA{}) == 3, "Smem layout must be rank 3.");
517 + static_assert(cute::rank(SmemLayoutB{}) == 3, "Smem layout must be rank 3.");
518 + static_assert(cute::is_void_v<SmemCopyAtomA>,
519 + "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions.");
520 + static_assert(cute::is_void_v<SmemCopyAtomB>,
521 + "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions.");
522 +
523 + Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.data()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE)
524 + Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.data()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE)
525 +
526 + // Block scaling
527 + Tensor sScaleAViewAsC = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_A.data()),
528 + Layout<
529 + Shape<Shape<Int<ScaleGranularityM>, Int<ScaleMsPerTile>>, cute::tuple_element_t<1, TileShape>, Int<DispatchPolicy::Stages>>,
530 + Stride<Stride<_0, _1>, _0, Int<ScaleMsPerTile>>
531 + >{}); // ((ScaleGranularityM,ScaleMsPerTile),n,k)
532 + Tensor sScaleB = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_B.data()), SmemLayoutScaleB{}); // (k)
533 +
534 + //
535 + // Define C accumulators and A/B partitioning
536 + //
537 +
538 + // Layout of warp group to thread mapping
539 +
540 + static_assert(stride<0>(typename TiledMma::ALayout{}) == 0 and
541 + stride<0>(typename TiledMma::BLayout{}) == 0 and
542 + size<0>(typename TiledMma::ALayout{}) == NumThreadsPerWarpGroup and
543 + size<0>(typename TiledMma::BLayout{}) == NumThreadsPerWarpGroup,
544 + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup");
545 +
546 + constexpr int MmaWarpGroups = size(TiledMma{}) / NumThreadsPerWarpGroup;
547 + Layout warp_group_thread_layout = make_layout(Int<MmaWarpGroups>{},
548 + Int<NumThreadsPerWarpGroup>{});
549 +
550 + int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / NumThreadsPerWarpGroup, 0);
551 +
552 + TiledMma tiled_mma;
553 + auto thread_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx));
554 +
555 + Tensor tCsScaleAViewAsC = tiled_mma.get_slice(thread_idx).partition_C(sScaleAViewAsC); // (MMA,MMA_M,MMA_N,PIPE), `thread_mma` above is correct when partitioning A and B, but it is not correct when partitioning C.
556 +
557 + Tensor tCsA = thread_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,PIPE)
558 + Tensor tCsB = thread_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE)
559 +
560 + // Allocate "fragments/descriptors"
561 + Tensor tCrA = thread_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K,PIPE)
562 + Tensor tCrB = thread_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE)
563 +
564 + CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(accum)); // M
565 + CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<2>(accum)); // N
566 + CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCsB)); // K
567 + CUTE_STATIC_ASSERT_V(size<3>(tCsA) == size<3>(tCsB)); // PIPE
568 + CUTE_STATIC_ASSERT_V(Int<DispatchPolicy::Stages>{} == size<2>(sA)); // PIPE
569 + CUTE_STATIC_ASSERT_V(Int<DispatchPolicy::Stages>{} == size<2>(sB)); // PIPE
570 +
571 + //
572 + // PIPELINED MAIN LOOP
573 + //
574 + static_assert((0 <= K_PIPE_MMAS) && (K_PIPE_MMAS < K_PIPE_MAX),
575 + "ERROR : Incorrect number of MMAs in flight");
576 +
577 + // We release buffers to producer warps(dma load) with some mmas in flight
578 + PipelineState smem_pipe_release = smem_pipe_read;
579 +
580 + // Per block scale values for operand A and B
581 +
582 + using RegLayoutScaleAViewAsC = decltype(make_layout_like(tCsScaleAViewAsC(_, _, _, 0).layout())); // `make_layout_like` makes a compact layout.
583 + using RegLayoutScaleAEssential = decltype(filter_zeros(RegLayoutScaleAViewAsC{}.stride(), RegLayoutScaleAViewAsC{}.shape())); // an interface to traverse the underlying storage for the compact layout mentioned above
584 +
585 + Tensor tCrScaleAViewAsC = make_tensor<ElementBlockScale>(RegLayoutScaleAViewAsC{}); // (MMA,MMA_M,MMA_N)
586 + ElementBlockScale scale_b;
587 +
588 + // Prologue GMMAs
589 + int prologue_mma_count = min(K_PIPE_MMAS, k_tile_count);
590 +
591 + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
592 +
593 + GmmaFP8AccumulationWithScale accumulation(accum, size<2>(TileShape{}) / size<2>(typename TiledMma::AtomShape_MNK{}), size<2>(tCrA));
594 + warpgroup_fence_operand(accumulation());
595 + CUTLASS_PRAGMA_UNROLL
596 + for (int k_tile_prologue = prologue_mma_count; k_tile_prologue > 0; --k_tile_prologue)
597 + {
598 + // WAIT on smem_pipe_read until its data are available (phase bit flips from rdPhaseBit value)
599 + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
600 + pipeline.consumer_wait(smem_pipe_read, barrier_token);
601 +
602 + if (accumulation.prepare_if_needed()) {
603 + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
604 + }
605 +
606 + int read_stage = smem_pipe_read.index();
607 +
608 + // Load per block scale values from shared memory to registers.
609 + scale_b = sScaleB[read_stage];
610 + CUTLASS_PRAGMA_UNROLL
611 + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
612 + tCrScaleAViewAsC.data()[i] = tCsScaleAViewAsC(_, _, _, read_stage)(idx2crd(i, RegLayoutScaleAEssential{}));
613 + }
614 + if constexpr (ScaleMsPerTile == 1) {
615 + static_assert(size(RegLayoutScaleAEssential{}) == 1);
616 + tCrScaleAViewAsC.data()[0] = __shfl_sync(0xffffffff, tCrScaleAViewAsC.data()[0] * scale_b, 0); // `tCrScaleAViewAsC.data()[0]` are all same in a warp group when `ScaleMsPerTile == 1`.
617 + } else {
618 + CUTLASS_PRAGMA_UNROLL
619 + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
620 + tCrScaleAViewAsC.data()[i] = tCrScaleAViewAsC.data()[i] * scale_b;
621 + }
622 + }
623 +
624 + warpgroup_arrive();
625 + // Unroll the K mode manually to set scale D to 1
626 + CUTLASS_PRAGMA_UNROLL
627 + for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) {
628 + // (V,M,K) x (V,N,K) => (V,M,N)
629 + cute::gemm(tiled_mma, tCrA(_,_,k_block,read_stage), tCrB(_,_,k_block,read_stage), accumulation());
630 + tiled_mma.accumulate_ = GMMA::ScaleOut::One;
631 + }
632 + warpgroup_commit_batch();
633 +
634 + // Block scale the accumulators with reg tensor `tCrScaleAViewAsC`
635 + accumulation.scale_if_needed(tCrScaleAViewAsC);
636 +
637 + ++smem_pipe_read;
638 + }
639 +
640 + warpgroup_fence_operand(accumulation());
641 + // Mainloop GMMAs
642 + k_tile_count -= prologue_mma_count;
643 +
644 + CUTLASS_PRAGMA_NO_UNROLL
645 + for ( ; k_tile_count > 0; --k_tile_count)
646 + {
647 + // WAIT on smem_pipe_read until its data are available (phase bit flips from rdPhaseBit value)
648 + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
649 + pipeline.consumer_wait(smem_pipe_read, barrier_token);
650 +
651 + //
652 + // Compute on k_tile
653 + //
654 +
655 + int read_stage = smem_pipe_read.index();
656 +
657 + // Load per block scale values from shared memory to registers (at most twice per block along M and exactly once per block along N)
658 + scale_b = sScaleB[read_stage];
659 + CUTLASS_PRAGMA_UNROLL
660 + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
661 + tCrScaleAViewAsC.data()[i] = tCsScaleAViewAsC(_, _, _, read_stage)(idx2crd(i, RegLayoutScaleAEssential{}));
662 + }
663 + if constexpr (ScaleMsPerTile == 1) {
664 + static_assert(size(RegLayoutScaleAEssential{}) == 1);
665 + tCrScaleAViewAsC.data()[0] = __shfl_sync(0xffffffff, tCrScaleAViewAsC.data()[0] * scale_b, 0); // `tCrScaleAViewAsC.data()[0]` are all same in a warp group when `ScaleMsPerTile == 1`.
666 + } else {
667 + CUTLASS_PRAGMA_UNROLL
668 + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) {
669 + tCrScaleAViewAsC.data()[i] = tCrScaleAViewAsC.data()[i] * scale_b;
670 + }
671 + }
672 +
673 + if (accumulation.prepare_if_needed()) {
674 + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
675 + }
676 +
677 + warpgroup_fence_operand(accumulation());
678 + warpgroup_arrive();
679 + // Unroll the K mode manually to set scale D to 1
680 + CUTLASS_PRAGMA_UNROLL
681 + for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) {
682 + // (V,M,K) x (V,N,K) => (V,M,N)
683 + cute::gemm(tiled_mma, tCrA(_,_,k_block,read_stage), tCrB(_,_,k_block,read_stage), accumulation());
684 + tiled_mma.accumulate_ = GMMA::ScaleOut::One;
685 + }
686 + warpgroup_commit_batch();
687 +
688 + /// Wait on the GMMA barrier for K_PIPE_MMAS (or fewer) outstanding to ensure smem_pipe_write is consumed
689 + warpgroup_wait<K_PIPE_MMAS>();
690 + warpgroup_fence_operand(accumulation());
691 +
692 + // Block scale the accumulators with reg tensor `tCrScaleAViewAsC`
693 + accumulation.scale_if_needed(tCrScaleAViewAsC);
694 +
695 + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it
696 +
697 + // Advance smem_pipe_read and smem_pipe_release
698 + ++smem_pipe_read;
699 + ++smem_pipe_release;
700 + }
701 +
702 + accumulation.scale_residue_if_needed(tCrScaleAViewAsC);
703 +
704 + warpgroup_fence_operand(accumulation());
705 + }
706 +
707 + /// Perform a Consumer Epilogue to release all buffers
708 + CUTLASS_DEVICE void
709 + mma_tail(MainloopPipeline pipeline, PipelineState smem_pipe_release, int k_tile_count) {
710 + // Prologue GMMAs
711 + int prologue_mma_count = min(K_PIPE_MMAS, k_tile_count);
712 + k_tile_count -= prologue_mma_count;
713 +
714 + smem_pipe_release.advance(k_tile_count);
715 +
716 + // Wait on all GMMAs to complete
717 + warpgroup_wait<0>();
718 +
719 + for (int count = 0; count < prologue_mma_count; ++count) {
720 + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it
721 + ++smem_pipe_release;
722 + }
723 + }
724 +};
725 +
726 +/////////////////////////////////////////////////////////////////////////////////////////////////
727 +
728 +} // namespace cutlass::gemm::collective
729 +
730 +/////////////////////////////////////////////////////////////////////////////////////////////////
csrc/cutlass_extensions/gemm/dispatch_policy.hpp new
+39
@@ -0,0 +1,39 @@
1 +#pragma once
2 +
3 +#include "cutlass/gemm/dispatch_policy.hpp"
4 +
5 +namespace cutlass::gemm {
6 +
7 +//////////////////////////////////////////////////////////////////////////////
8 +
9 +// FP8 related policies (including Blocked Scaled Accumulation)
10 +// `ScaleGranularityM` specifies scaling granularity along M, while zero-value
11 +// `ScaleGranularityM` indicates that scaling granularity is
12 +// `size<0>(TileShape_MNK{})` along M.
13 +template <int ScaleGranularityM = 0>
14 +struct KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum
15 + : KernelTmaWarpSpecializedCooperative {};
16 +
17 +// n-buffer in smem (Hopper TMA), pipelined with Hopper GMMA and TMA, Warp
18 +// specialized dynamic schedule For FP8 kernels with Block Scaling
19 +template <int Stages_, class ClusterShape_ = Shape<_1, _1, _1>,
20 + class KernelSchedule = KernelTmaWarpSpecialized,
21 + int ScaleGranularityM =
22 + 0 // `ScaleGranularityM` specifies scaling granularity along M,
23 + // while zero-value `ScaleGranularityM` indicates that scaling
24 + // granularity is `size<0>(TileShape_MNK{})` along M.
25 + >
26 +struct MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8
27 + : MainloopSm90TmaGmmaWarpSpecialized<Stages_, ClusterShape_,
28 + KernelSchedule> {
29 + static_assert(
30 + cute::is_same_v<
31 + KernelSchedule,
32 + KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<
33 + ScaleGranularityM>>,
34 + "KernelSchedule must be one of the warp specialized policies");
35 +};
36 +
37 +//////////////////////////////////////////////////////////////////////////////
38 +
39 +} // namespace cutlass::gemm
\ No newline at end of file
csrc/cutlass_extensions/vllm_collective_builder.cuh
+1 -1
@@ -1,6 +1,6 @@
1 #pragma once
2
3 -#include "cutlass/gemm/collective/collective_builder.hpp"
3 +#include "cutlass_extensions/gemm/collective/collective_builder.hpp"
4
5 namespace cutlass::gemm::collective {
6 using namespace cute;
csrc/quantization/cutlass_w8a8/c3x/cutlass_gemm_caller.cuh new
+93
@@ -0,0 +1,93 @@
1 +#pragma once
2 +
3 +// clang-format will break include orders
4 +// clang-format off
5 +#include <torch/all.h>
6 +
7 +#include <ATen/cuda/CUDAContext.h>
8 +
9 +#include "cutlass/cutlass.h"
10 +
11 +#include "cute/tensor.hpp"
12 +#include "cute/atom/mma_atom.hpp"
13 +#include "cutlass/numeric_types.h"
14 +
15 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
16 +#include "cutlass/gemm/kernel/gemm_universal.hpp"
17 +#include "cutlass/epilogue/collective/collective_builder.hpp"
18 +#include "cutlass/gemm/collective/collective_builder.hpp"
19 +
20 +#include "core/math.hpp"
21 +#include "cutlass_extensions/common.hpp"
22 +// clang-format on
23 +
24 +namespace vllm::c3x {
25 +
26 +static inline cute::Shape<int, int, int, int> get_problem_shape(
27 + torch::Tensor const& a, torch::Tensor const& b) {
28 + int32_t m = a.size(0), n = b.size(1), k = a.size(1);
29 + return {m, n, k, 1};
30 +}
31 +
32 +template <typename GemmKernel>
33 +void cutlass_gemm_caller(torch::Device device,
34 + cute::Shape<int, int, int, int> prob_shape,
35 + typename GemmKernel::MainloopArguments mainloop_args,
36 + typename GemmKernel::EpilogueArguments epilogue_args) {
37 + typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
38 + prob_shape, mainloop_args, epilogue_args};
39 +
40 + // Launch the CUTLASS GEMM kernel.
41 + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
42 + GemmOp gemm_op;
43 + CUTLASS_CHECK(gemm_op.can_implement(args));
44 +
45 + size_t workspace_size = gemm_op.get_workspace_size(args);
46 + auto const workspace_options =
47 + torch::TensorOptions().dtype(torch::kUInt8).device(device);
48 + auto workspace = torch::empty(workspace_size, workspace_options);
49 +
50 + auto stream = at::cuda::getCurrentCUDAStream(device.index());
51 +
52 + cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
53 + CUTLASS_CHECK(status);
54 +}
55 +
56 +template <typename Gemm, typename... EpilogueArgs>
57 +void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
58 + torch::Tensor const& b,
59 + EpilogueArgs&&... epilogue_params) {
60 + using ElementAB = typename Gemm::ElementAB;
61 + using ElementD = typename Gemm::ElementD;
62 + using GemmKernel = typename Gemm::GemmKernel;
63 +
64 + int64_t lda = a.stride(0);
65 + int64_t ldb = b.stride(1);
66 + int64_t ldc = out.stride(0);
67 +
68 + using StrideA = cute::Stride<int64_t, cute::Int<1>, int64_t>;
69 + using StrideB = cute::Stride<int64_t, cute::Int<1>, int64_t>;
70 + using StrideC = typename Gemm::StrideC;
71 +
72 + StrideA a_stride{lda, cute::Int<1>{}, 0};
73 + StrideB b_stride{ldb, cute::Int<1>{}, 0};
74 + StrideC c_stride{ldc, cute::Int<1>{}, cute::Int<0>{}};
75 +
76 + typename GemmKernel::ProblemShape prob_shape = get_problem_shape(a, b);
77 +
78 + auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
79 + auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
80 + typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
81 + b_stride};
82 +
83 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
84 + typename GemmKernel::EpilogueArguments epilogue_args{
85 + Gemm::Epilogue::prepare_args(
86 + std::forward<EpilogueArgs>(epilogue_params)...),
87 + c_ptr, c_stride, c_ptr, c_stride};
88 +
89 + cutlass_gemm_caller<GemmKernel>(a.device(), prob_shape, mainloop_args,
90 + epilogue_args);
91 +}
92 +
93 +} // namespace vllm::c3x
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh renamed
-74
@@ -2,9 +2,6 @@
2
3 // clang-format will break include orders
4 // clang-format off
5 -#include <torch/all.h>
6 -
7 -#include <ATen/cuda/CUDAContext.h>
5
6 #include "cutlass/cutlass.h"
7
@@ -32,21 +29,6 @@ using namespace cute;
29
30 namespace vllm {
31
35 -// A wrapper for the GEMM kernel that is used to guard against compilation on
36 -// architectures that will never use the kernel. The purpose of this is to
37 -// reduce the size of the compiled binary.
38 -// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
39 -// into code that will be executed on the device where it is defined.
40 -template <typename Kernel>
41 -struct enable_sm90_or_later : Kernel {
42 - template <typename... Args>
43 - CUTLASS_DEVICE void operator()(Args&&... args) {
44 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
45 - Kernel::operator()(std::forward<Args>(args)...);
46 -#endif
47 - }
48 -};
49 -
32 template <typename ElementAB_, typename ElementD_,
33 template <typename, typename, typename> typename Epilogue_,
34 typename TileShape, typename ClusterShape, typename KernelSchedule,
@@ -101,60 +83,4 @@ struct cutlass_3x_gemm {
83 struct GemmKernel : public KernelType {};
84 };
85
104 -template <typename Gemm, typename... EpilogueArgs>
105 -void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
106 - torch::Tensor const& b,
107 - EpilogueArgs&&... epilogue_params) {
108 - using ElementAB = typename Gemm::ElementAB;
109 - using ElementD = typename Gemm::ElementD;
110 -
111 - int32_t m = a.size(0);
112 - int32_t n = b.size(1);
113 - int32_t k = a.size(1);
114 -
115 - int64_t lda = a.stride(0);
116 - int64_t ldb = b.stride(1);
117 - int64_t ldc = out.stride(0);
118 -
119 - using StrideA = Stride<int64_t, Int<1>, int64_t>;
120 - using StrideB = Stride<int64_t, Int<1>, int64_t>;
121 - using StrideC = typename Gemm::StrideC;
122 -
123 - StrideA a_stride{lda, Int<1>{}, 0};
124 - StrideB b_stride{ldb, Int<1>{}, 0};
125 - StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
126 -
127 - using GemmKernel = typename Gemm::GemmKernel;
128 - typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
129 -
130 - auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
131 - auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
132 - typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
133 - b_stride};
134 -
135 - auto c_ptr = static_cast<ElementD*>(out.data_ptr());
136 - typename GemmKernel::EpilogueArguments epilogue_args{
137 - Gemm::Epilogue::prepare_args(
138 - std::forward<EpilogueArgs>(epilogue_params)...),
139 - c_ptr, c_stride, c_ptr, c_stride};
140 -
141 - typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
142 - prob_shape, mainloop_args, epilogue_args};
143 -
144 - // Launch the CUTLASS GEMM kernel.
145 - using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
146 - GemmOp gemm_op;
147 - CUTLASS_CHECK(gemm_op.can_implement(args));
148 -
149 - size_t workspace_size = gemm_op.get_workspace_size(args);
150 - auto const workspace_options =
151 - torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
152 - auto workspace = torch::empty(workspace_size, workspace_options);
153 -
154 - auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
155 -
156 - cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
157 - CUTLASS_CHECK(status);
158 -}
159 -
86 } // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu new
+24
@@ -0,0 +1,24 @@
1 +#include "scaled_mm_kernels.hpp"
2 +#include "scaled_mm_sm90_int8_dispatch.cuh"
3 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
4 +
5 +namespace vllm {
6 +
7 +void cutlass_scaled_mm_azp_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
8 + torch::Tensor const& b,
9 + torch::Tensor const& a_scales,
10 + torch::Tensor const& b_scales,
11 + torch::Tensor const& azp_adj,
12 + std::optional<torch::Tensor> const& azp,
13 + std::optional<torch::Tensor> const& bias) {
14 + if (azp) {
15 + return cutlass_scaled_mm_sm90_int8_epilogue<
16 + c3x::ScaledEpilogueBiasAzpToken>(out, a, b, a_scales, b_scales, azp_adj,
17 + *azp, bias);
18 + } else {
19 + return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogueBiasAzp>(
20 + out, a, b, a_scales, b_scales, azp_adj, bias);
21 + }
22 +}
23 +
24 +} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8.cu new
+24
@@ -0,0 +1,24 @@
1 +
2 +#include "scaled_mm_kernels.hpp"
3 +#include "scaled_mm_blockwise_sm90_fp8_dispatch.cuh"
4 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
5 +
6 +namespace vllm {
7 +
8 +void cutlass_scaled_mm_blockwise_sm90_fp8(torch::Tensor& out,
9 + torch::Tensor const& a,
10 + torch::Tensor const& b,
11 + torch::Tensor const& a_scales,
12 + torch::Tensor const& b_scales) {
13 + if (out.dtype() == torch::kBFloat16) {
14 + cutlass_gemm_blockwise_sm90_fp8_dispatch<cutlass::bfloat16_t>(
15 + out, a, b, a_scales, b_scales);
16 +
17 + } else {
18 + TORCH_CHECK(out.dtype() == torch::kFloat16);
19 + cutlass_gemm_blockwise_sm90_fp8_dispatch<cutlass::half_t>(
20 + out, a, b, a_scales, b_scales);
21 + }
22 +}
23 +
24 +} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh new
+168
@@ -0,0 +1,168 @@
1 +#pragma once
2 +
3 +#include "cutlass/cutlass.h"
4 +#include "cutlass/numeric_types.h"
5 +
6 +#include "cute/tensor.hpp"
7 +#include "cutlass/tensor_ref.h"
8 +#include "cutlass/gemm/dispatch_policy.hpp"
9 +#include "cutlass/gemm/collective/collective_builder.hpp"
10 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
11 +#include "cutlass/gemm/kernel/gemm_universal.hpp"
12 +#include "cutlass/gemm/kernel/tile_scheduler_params.h"
13 +#include "cutlass/epilogue/dispatch_policy.hpp"
14 +#include "cutlass/epilogue/collective/collective_builder.hpp"
15 +
16 +#include "cutlass_extensions/gemm/dispatch_policy.hpp"
17 +#include "cutlass_extensions/gemm/collective/collective_builder.hpp"
18 +
19 +#include "cutlass_gemm_caller.cuh"
20 +
21 +namespace vllm {
22 +
23 +using namespace cute;
24 +
25 +template <typename OutType, int GroupSizeM_, int GroupSizeN_, int GroupSizeK_,
26 + int TileSizeM_ = 128, class ClusterShape = Shape<_1, _2, _1>>
27 +struct cutlass_3x_gemm_fp8_blockwise {
28 + using GroupSizeM = Int<GroupSizeM_>;
29 + using GroupSizeN = Int<GroupSizeN_>;
30 + using GroupSizeK = Int<GroupSizeK_>;
31 + using TileSizeM = Int<TileSizeM_>;
32 +
33 + static_assert(TileSizeM_ % GroupSizeM_ == 0,
34 + "TileSizeM must be a multiple of GroupSizeM");
35 +
36 + using ElementAB = cutlass::float_e4m3_t;
37 +
38 + using ElementA = ElementAB;
39 + using LayoutA = cutlass::layout::RowMajor;
40 + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
41 +
42 + using ElementB = ElementAB;
43 + using LayoutB = cutlass::layout::ColumnMajor;
44 + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
45 +
46 + using ElementD = OutType;
47 + using StrideD = Stride<int64_t, Int<1>, Int<0>>;
48 + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits<ElementD>::value;
49 +
50 + using ElementC = void;
51 + using StrideC = StrideD;
52 + static constexpr int AlignmentC = AlignmentD;
53 +
54 + using ElementAccumulator = float;
55 + using ElementBlockScale = float;
56 + using ElementCompute = float;
57 + using ArchTag = cutlass::arch::Sm90;
58 + using OperatorClass = cutlass::arch::OpClassTensorOp;
59 + using TileShape = Shape<TileSizeM, GroupSizeN, GroupSizeK>;
60 +
61 + using KernelSchedule = cutlass::gemm::
62 + KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum<
63 + GroupSizeM_>;
64 + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
65 + using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
66 +
67 + using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT<
68 + cutlass::epilogue::fusion::Sm90AccFetch>;
69 +
70 + using CollectiveEpilogue =
71 + typename cutlass::epilogue::collective::CollectiveBuilder<
72 + ArchTag, OperatorClass, TileShape, ClusterShape, EpilogueTileType,
73 + ElementAccumulator, ElementCompute, ElementC, StrideC, AlignmentC,
74 + ElementD, StrideD, AlignmentD, EpilogueSchedule,
75 + StoreEpilogueCompute>::CollectiveOp;
76 +
77 + using CollectiveMainloop =
78 + typename cutlass::gemm::collective::CollectiveBuilder<
79 + ArchTag, OperatorClass, ElementA, LayoutA, AlignmentA, ElementB,
80 + LayoutB, AlignmentB, ElementAccumulator, TileShape, ClusterShape,
81 + cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
82 + sizeof(typename CollectiveEpilogue::SharedStorage))>,
83 + KernelSchedule>::CollectiveOp;
84 +
85 + using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
86 + Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
87 + cutlass::gemm::PersistentScheduler>>;
88 +
89 + struct GemmKernel : public KernelType {};
90 +
91 + using StrideA = typename GemmKernel::StrideA;
92 + using StrideB = typename GemmKernel::StrideB;
93 +};
94 +
95 +template <typename Gemm>
96 +void cutlass_gemm_caller_blockwise(torch::Tensor& out, torch::Tensor const& a,
97 + torch::Tensor const& b,
98 + torch::Tensor const& a_scales,
99 + torch::Tensor const& b_scales) {
100 + using GemmKernel = typename Gemm::GemmKernel;
101 +
102 + using ElementAB = typename Gemm::ElementAB;
103 + using ElementD = typename Gemm::ElementD;
104 +
105 + auto prob_shape = c3x::get_problem_shape(a, b);
106 + int32_t m = get<0>(prob_shape), n = get<1>(prob_shape),
107 + k = get<2>(prob_shape);
108 +
109 + int64_t lda = a.stride(0);
110 + int64_t ldb = b.stride(1);
111 + int64_t ldc = out.stride(0);
112 +
113 + using StrideA = Stride<int64_t, Int<1>, int64_t>;
114 + using StrideB = Stride<int64_t, Int<1>, int64_t>;
115 + using StrideC = typename Gemm::StrideC;
116 +
117 + StrideA a_stride{lda, Int<1>{}, 0};
118 + StrideB b_stride{ldb, Int<1>{}, 0};
119 + StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
120 +
121 + auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
122 + auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
123 + auto a_scales_ptr = static_cast<float*>(a_scales.data_ptr());
124 + auto b_scales_ptr = static_cast<float*>(b_scales.data_ptr());
125 +
126 + // Check is the t is contiguous and is 1D or 2D with one of the dimensions
127 + // being 1 (i.e. a row or column vector)
128 + auto is_contiguous_vector = [](const torch::Tensor& t) {
129 + auto t_sizes = t.sizes();
130 + return t.is_contiguous() &&
131 + (t.dim() == 1 ||
132 + (t.dim() == 2 &&
133 + *std::min_element(t_sizes.begin(), t_sizes.end()) == 1));
134 + };
135 +
136 + // TODO(lucas): lets clean-up the kernel so that we pass in Strides so
137 + // we don't have to deal with enforcing implicit layouts
138 + TORCH_CHECK(a_scales.size(0) == m / Gemm::GroupSizeM::value);
139 + TORCH_CHECK(a_scales.size(1) == k / Gemm::GroupSizeK::value);
140 + TORCH_CHECK(a_scales.stride(0) == 1 || is_contiguous_vector(a_scales),
141 + "a_scales must be M major");
142 + TORCH_CHECK(b_scales.size(0) == k / Gemm::GroupSizeK::value);
143 + TORCH_CHECK(b_scales.size(1) == n / Gemm::GroupSizeN::value);
144 + TORCH_CHECK(b_scales.stride(0) == 1 || is_contiguous_vector(b_scales),
145 + "b_scales must be K major");
146 + typename GemmKernel::MainloopArguments mainloop_args{
147 + a_ptr, a_stride, b_ptr, b_stride, a_scales_ptr, b_scales_ptr};
148 +
149 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
150 + typename GemmKernel::EpilogueArguments epilogue_args{
151 + {}, c_ptr, c_stride, c_ptr, c_stride};
152 +
153 + c3x::cutlass_gemm_caller<GemmKernel>(a.device(), prob_shape, mainloop_args,
154 + epilogue_args);
155 +}
156 +
157 +template <typename OutType>
158 +void cutlass_gemm_blockwise_sm90_fp8_dispatch(torch::Tensor& out,
159 + torch::Tensor const& a,
160 + torch::Tensor const& b,
161 + torch::Tensor const& a_scales,
162 + torch::Tensor const& b_scales) {
163 + cutlass_gemm_caller_blockwise<
164 + cutlass_3x_gemm_fp8_blockwise<OutType, 1, 128, 128>>(out, a, b, a_scales,
165 + b_scales);
166 +}
167 +
168 +} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_kernels.hpp new
+33
@@ -0,0 +1,33 @@
1 +#pragma once
2 +
3 +#include <torch/all.h>
4 +
5 +namespace vllm {
6 +
7 +void cutlass_scaled_mm_sm90_fp8(torch::Tensor& out, torch::Tensor const& a,
8 + torch::Tensor const& b,
9 + torch::Tensor const& a_scales,
10 + torch::Tensor const& b_scales,
11 + std::optional<torch::Tensor> const& bias);
12 +
13 +void cutlass_scaled_mm_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
14 + torch::Tensor const& b,
15 + torch::Tensor const& a_scales,
16 + torch::Tensor const& b_scales,
17 + std::optional<torch::Tensor> const& bias);
18 +
19 +void cutlass_scaled_mm_azp_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
20 + torch::Tensor const& b,
21 + torch::Tensor const& a_scales,
22 + torch::Tensor const& b_scales,
23 + torch::Tensor const& azp_adj,
24 + std::optional<torch::Tensor> const& azp,
25 + std::optional<torch::Tensor> const& bias);
26 +
27 +void cutlass_scaled_mm_blockwise_sm90_fp8(torch::Tensor& out,
28 + torch::Tensor const& a,
29 + torch::Tensor const& b,
30 + torch::Tensor const& a_scales,
31 + torch::Tensor const& b_scales);
32 +
33 +} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu new
+24
@@ -0,0 +1,24 @@
1 +#include "scaled_mm_kernels.hpp"
2 +#include "scaled_mm_sm90_fp8_dispatch.cuh"
3 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
4 +
5 +namespace vllm {
6 +
7 +void cutlass_scaled_mm_sm90_fp8(torch::Tensor& out, torch::Tensor const& a,
8 + torch::Tensor const& b,
9 + torch::Tensor const& a_scales,
10 + torch::Tensor const& b_scales,
11 + std::optional<torch::Tensor> const& bias) {
12 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
13 + if (bias) {
14 + TORCH_CHECK(bias->dtype() == out.dtype(),
15 + "currently bias dtype must match output dtype ", out.dtype());
16 + return cutlass_scaled_mm_sm90_fp8_epilogue<c3x::ScaledEpilogueBias>(
17 + out, a, b, a_scales, b_scales, *bias);
18 + } else {
19 + return cutlass_scaled_mm_sm90_fp8_epilogue<c3x::ScaledEpilogue>(
20 + out, a, b, a_scales, b_scales);
21 + }
22 +}
23 +
24 +} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8_dispatch.cuh renamed
+25 -1
@@ -1,6 +1,7 @@
1 #pragma once
2
3 -#include "scaled_mm_c3x.cuh"
3 +#include "scaled_mm.cuh"
4 +#include "cutlass_gemm_caller.cuh"
5
6 /**
7 * This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm
@@ -9,6 +10,8 @@
10
11 namespace vllm {
12
13 +using c3x::cutlass_gemm_caller;
14 +
15 template <typename InType, typename OutType,
16 template <typename, typename, typename> typename Epilogue>
17 struct sm90_fp8_config_default {
@@ -93,4 +96,25 @@ inline void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out,
96 }
97 }
98
99 +template <template <typename, typename, typename> typename Epilogue,
100 + typename... EpilogueArgs>
101 +void cutlass_scaled_mm_sm90_fp8_epilogue(torch::Tensor& out,
102 + torch::Tensor const& a,
103 + torch::Tensor const& b,
104 + EpilogueArgs&&... epilogue_args) {
105 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
106 + TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
107 +
108 + if (out.dtype() == torch::kBFloat16) {
109 + return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
110 + cutlass::bfloat16_t, Epilogue>(
111 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
112 + } else {
113 + TORCH_CHECK(out.dtype() == torch::kFloat16);
114 + return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
115 + cutlass::half_t, Epilogue>(
116 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
117 + }
118 +}
119 +
120 } // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu new
+24
@@ -0,0 +1,24 @@
1 +#include "scaled_mm_kernels.hpp"
2 +#include "scaled_mm_sm90_int8_dispatch.cuh"
3 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
4 +
5 +namespace vllm {
6 +
7 +void cutlass_scaled_mm_sm90_int8(torch::Tensor& out, torch::Tensor const& a,
8 + torch::Tensor const& b,
9 + torch::Tensor const& a_scales,
10 + torch::Tensor const& b_scales,
11 + std::optional<torch::Tensor> const& bias) {
12 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
13 + if (bias) {
14 + TORCH_CHECK(bias->dtype() == out.dtype(),
15 + "currently bias dtype must match output dtype ", out.dtype());
16 + return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogueBias>(
17 + out, a, b, a_scales, b_scales, *bias);
18 + } else {
19 + return cutlass_scaled_mm_sm90_int8_epilogue<c3x::ScaledEpilogue>(
20 + out, a, b, a_scales, b_scales);
21 + }
22 +}
23 +
24 +} // namespace vllm
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8_dispatch.cuh renamed
+24 -1
@@ -1,6 +1,7 @@
1 #pragma once
2
3 -#include "scaled_mm_c3x.cuh"
3 +#include "scaled_mm.cuh"
4 +#include "cutlass_gemm_caller.cuh"
5
6 /**
7 * This file defines Gemm kernel configurations for SM90 (int8) based on the
@@ -9,6 +10,8 @@
10
11 namespace vllm {
12
13 +using c3x::cutlass_gemm_caller;
14 +
15 template <typename InType, typename OutType,
16 template <typename, typename, typename> typename Epilogue>
17 struct sm90_int8_config_default {
@@ -137,4 +140,24 @@ inline void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out,
140 }
141 }
142
143 +template <template <typename, typename, typename> typename Epilogue,
144 + typename... EpilogueArgs>
145 +void cutlass_scaled_mm_sm90_int8_epilogue(torch::Tensor& out,
146 + torch::Tensor const& a,
147 + torch::Tensor const& b,
148 + EpilogueArgs&&... epilogue_args) {
149 + TORCH_CHECK(a.dtype() == torch::kInt8);
150 + TORCH_CHECK(b.dtype() == torch::kInt8);
151 +
152 + if (out.dtype() == torch::kBFloat16) {
153 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
154 + Epilogue>(
155 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
156 + } else {
157 + TORCH_CHECK(out.dtype() == torch::kFloat16);
158 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
159 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
160 + }
161 +}
162 +
163 } // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+47 -57
@@ -1,52 +1,13 @@
1 #include <cudaTypedefs.h>
2 +#include "c3x/scaled_mm_kernels.hpp"
3
3 -#if defined CUDA_VERSION && CUDA_VERSION >= 12000
4 -
5 - #include "scaled_mm_c3x_sm90_fp8_dispatch.cuh"
6 - #include "scaled_mm_c3x_sm90_int8_dispatch.cuh"
7 -
8 - #include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
9 -using namespace vllm;
4 +#include "core/math.hpp"
5
6 /*
7 This file defines quantized GEMM operations using the CUTLASS 3.x API, for
8 NVIDIA GPUs with sm90a (Hopper) or later.
9 */
10
16 -template <template <typename, typename, typename> typename Epilogue,
17 - typename... EpilogueArgs>
18 -void cutlass_scaled_mm_sm90_epilogue(torch::Tensor& out, torch::Tensor const& a,
19 - torch::Tensor const& b,
20 - EpilogueArgs&&... epilogue_args) {
21 - if (a.dtype() == torch::kInt8) {
22 - TORCH_CHECK(b.dtype() == torch::kInt8);
23 -
24 - if (out.dtype() == torch::kBFloat16) {
25 - return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
26 - Epilogue>(
27 - out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
28 - } else {
29 - TORCH_CHECK(out.dtype() == torch::kFloat16);
30 - return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
31 - out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
32 - }
33 - } else {
34 - TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
35 - TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
36 -
37 - if (out.dtype() == torch::kBFloat16) {
38 - return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
39 - cutlass::bfloat16_t, Epilogue>(
40 - out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
41 - } else {
42 - TORCH_CHECK(out.dtype() == torch::kFloat16);
43 - return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
44 - cutlass::half_t, Epilogue>(
45 - out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
46 - }
47 - }
48 -}
49 -
11 void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
12 torch::Tensor const& b,
13 torch::Tensor const& a_scales,
@@ -54,14 +15,50 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
15 std::optional<torch::Tensor> const& bias) {
16 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
17 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
57 - if (bias) {
58 - TORCH_CHECK(bias->dtype() == c.dtype(),
59 - "currently bias dtype must match output dtype ", c.dtype());
60 - return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
61 - c, a, b, a_scales, b_scales, *bias);
18 +
19 + using GroupShape = std::array<int64_t, 2>;
20 +
21 + int M = a.size(0), N = b.size(1), K = a.size(1);
22 +
23 + GroupShape a_scale_group_shape = [&, &s = a_scales]() -> GroupShape {
24 + if (s.numel() == 1) return {M, K}; // tensor-wise
25 + if (s.dim() == 2)
26 + return {ceil_div(a.size(0), s.size(0)), ceil_div(a.size(1), s.size(1))};
27 + TORCH_CHECK(false, "Unsupported scale shape for scale_a");
28 + }();
29 +
30 + GroupShape b_scale_group_shape = [&, &s = b_scales]() -> GroupShape {
31 + if (s.numel() == 1) return {K, N}; // tensor-wise
32 + if (s.dim() == 2)
33 + return {ceil_div(b.size(0), s.size(0)), ceil_div(b.size(1), s.size(1))};
34 + TORCH_CHECK(false, "Unsupported scale shape for scale_b");
35 + }();
36 +
37 + if ((a_scale_group_shape == GroupShape{M, K} ||
38 + a_scale_group_shape == GroupShape{1, K}) &&
39 + (b_scale_group_shape == GroupShape{K, N} ||
40 + b_scale_group_shape == GroupShape{K, 1})) {
41 + // "standard per-tensor/per-token/per-channel" scaling
42 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
43 + if (a.dtype() == torch::kFloat8_e4m3fn) {
44 + vllm::cutlass_scaled_mm_sm90_fp8(c, a, b, a_scales, b_scales, bias);
45 + } else {
46 + TORCH_CHECK(a.dtype() == torch::kInt8);
47 + vllm::cutlass_scaled_mm_sm90_int8(c, a, b, a_scales, b_scales, bias);
48 + }
49 + } else if (a_scale_group_shape == GroupShape{1, 128} &&
50 + b_scale_group_shape == GroupShape{128, 128}) {
51 + // 1x128 per-token group scales for activations
52 + // 128x128 blockwise scales for weights
53 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn &&
54 + b.dtype() == torch::kFloat8_e4m3fn,
55 + "Currently only FP8 is supported for A group shape 1x128 and "
56 + "B group shape 128x128");
57 + TORCH_CHECK(!bias, "Bias not yet supported blockwise scaled_mm");
58 +
59 + vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
60 } else {
63 - return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogue>(
64 - c, a, b, a_scales, b_scales);
61 + TORCH_CHECK(false, "Unsupported scale group shapes for CUTLASS 3.x GEMM");
62 }
63 }
64
@@ -75,13 +72,6 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
72 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
73 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
74
78 - if (azp) {
79 - return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzpToken>(
80 - out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
81 - } else {
82 - return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzp>(
83 - out, a, b, a_scales, b_scales, azp_adj, bias);
84 - }
75 + vllm::cutlass_scaled_mm_azp_sm90_int8(out, a, b, a_scales, b_scales, azp_adj,
76 + azp, bias);
77 }
86 -
87 -#endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
-3
@@ -89,15 +89,12 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
89 TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
90 TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
91 b.size(1) == c.size(1));
92 - TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
93 - TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
92
93 // Check for strides and alignment
94 TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
95 TORCH_CHECK(b.stride(0) == 1); // Column-major
96 TORCH_CHECK(c.stride(0) % 16 == 0 &&
97 b.stride(1) % 16 == 0); // 16 Byte Alignment
100 - TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
98
99 if (bias) {
100 TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous() &&
csrc/quantization/machete/machete_mainloop.cuh
+4
@@ -272,6 +272,10 @@ struct MacheteCollectiveMma {
272 using PipelineState = cutlass::PipelineState<DispatchPolicy::Stages>;
273
274 using PipelineParams = typename MainloopPipeline::Params;
275 +
276 + // One threads per CTA are producers (1 for operand tile)
277 + static constexpr int NumProducerThreadEvents = 1;
278 +
279 using ScaleTileShape = decltype(make_shape(shape<0>(TileShape{}),
280 shape<1>(SmemLayoutAtomScale{})));
281
tests/kernels/test_cutlass.py
+132 -56
@@ -10,6 +10,7 @@ import torch
10 from tests.kernels.utils import opcheck
11 from vllm import _custom_ops as ops
12 from vllm.platforms import current_platform
13 +from vllm.utils import cdiv
14
15 from .utils import baseline_scaled_mm, to_fp8, to_int8
16
@@ -39,6 +40,11 @@ CUDA_DEVICES = [
40 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
41 ]
42
43 +# -1 means full extent in that dimension
44 +TENSORWISE_GROUP_SHAPE = (-1, -1)
45 +PER_TOKEN_GROUP_SHAPE = (1, -1)
46 +PER_OUT_CH_GROUP_SHAPE = (-1, 1)
47 +
48 capability = current_platform.get_device_capability()
49 capability = capability[0] * 10 + capability[1]
50
@@ -47,11 +53,22 @@ def rand_int8(shape: tuple, device: str = "cuda"):
53 return to_int8(torch.rand(shape, device=device) * 255 - 128)
54
55
56 +def group_scale_helper(shape, group_shape):
57 + return [shape[i] if s < 0 else s for i, s in enumerate(group_shape)]
58 +
59 +
60 +def scale_shape(shape, group_shape):
61 + assert len(shape) == len(group_shape)
62 + group_shape = group_scale_helper(shape, group_shape)
63 + return tuple(
64 + cdiv(shape[i], group_shape[i]) for i in range(len(group_shape)))
65 +
66 +
67 def cutlass_fp8_gemm_helper(m: int,
68 n: int,
69 k: int,
53 - per_token_act_quant: bool,
54 - per_out_channel_weight_quant: bool,
70 + a_scale_group_shape: tuple,
71 + b_scale_group_shape: tuple,
72 use_bias: bool,
73 out_dtype: Type[torch.dtype] = torch.bfloat16,
74 device: str = "cuda"):
@@ -60,13 +77,17 @@ def cutlass_fp8_gemm_helper(m: int,
77 a = to_fp8(torch.randn((m, k), device=device))
78 b = to_fp8(torch.randn((n, k), device=device).t())
79
63 - m_a_scales = m if per_token_act_quant else 1
64 - n_b_scales = n if per_out_channel_weight_quant else 1
80 + a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
81 + b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
82 +
83 + scale_a = (torch.randn(a_scales_shape, device=device, dtype=torch.float32))
84 + scale_b = (torch.randn(b_scales_shape, device=device, dtype=torch.float32))
85 +
86 + # make scales M-major for blockwise quant, doesn't affect 1D scales
87 + scale_a = scale_a.t().contiguous().t()
88 + # make scales K-major for blockwise quant, doesn't affect 1D scales
89 + scale_b = scale_b.t().contiguous().t()
90
66 - scale_a = (torch.randn((m_a_scales, 1), device=device,
67 - dtype=torch.float32))
68 - scale_b = (torch.randn((1, n_b_scales), device=device,
69 - dtype=torch.float32))
91 if use_bias:
92 bias = torch.rand((n, ), device=device, dtype=out_dtype) * 10
93 else:
@@ -84,8 +105,8 @@ def cutlass_fp8_gemm_helper(m: int,
105 def cutlass_int8_gemm_helper(m: int,
106 n: int,
107 k: int,
87 - per_token_act_quant: bool,
88 - per_out_channel_weight_quant: bool,
108 + a_scale_group_shape: tuple,
109 + b_scale_group_shape: tuple,
110 use_bias: bool,
111 out_dtype: Type[torch.dtype] = torch.bfloat16,
112 device: str = "cuda"):
@@ -94,13 +115,11 @@ def cutlass_int8_gemm_helper(m: int,
115 a = to_int8(torch.randn((m, k), device=device) * 5)
116 b = to_int8(torch.randn((n, k), device=device).t() * 5)
117
97 - m_a_scales = m if per_token_act_quant else 1
98 - n_b_scales = n if per_out_channel_weight_quant else 1
118 + a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
119 + b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
120
100 - scale_a = (torch.randn((m_a_scales, 1), device=device,
101 - dtype=torch.float32))
102 - scale_b = (torch.randn((1, n_b_scales), device=device,
103 - dtype=torch.float32))
121 + scale_a = (torch.randn(a_scales_shape, device=device, dtype=torch.float32))
122 + scale_b = (torch.randn(b_scales_shape, device=device, dtype=torch.float32))
123
124 if use_bias:
125 bias = torch.rand((n, ), device=device, dtype=out_dtype) * 10
@@ -117,82 +136,135 @@ def cutlass_int8_gemm_helper(m: int,
136
137
138 @pytest.mark.parametrize("m,n,k", MNK_FACTORS)
120 -@pytest.mark.parametrize("per_act_token", [True, False])
121 -@pytest.mark.parametrize("per_out_ch", [True, False])
139 +@pytest.mark.parametrize("a_scale_group_shape",
140 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
141 +@pytest.mark.parametrize("b_scale_group_shape",
142 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
143 @pytest.mark.parametrize("use_bias", [True, False])
144 @pytest.mark.skipif(not current_platform.has_device_capability(89),
145 reason="FP8 is not supported on this GPU type.")
125 -def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
126 - per_out_ch: bool, use_bias: bool):
127 - cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
146 +def test_cutlass_fp8_gemm(m: int, n: int, k: int, a_scale_group_shape,
147 + b_scale_group_shape, use_bias: bool):
148 + cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape,
149 + use_bias)
150
151
152 @pytest.mark.parametrize("m,n,k", MNK_FACTORS)
131 -@pytest.mark.parametrize("per_act_token", [True, False])
132 -@pytest.mark.parametrize("per_out_ch", [True, False])
153 +@pytest.mark.parametrize("a_scale_group_shape,b_scale_group_shape",
154 + [((1, 128), (128, 128))])
155 +@pytest.mark.parametrize("use_bias", [False])
156 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
157 + reason="FP8 blockwise is not supported on this GPU type.")
158 +def test_cutlass_fp8_blockwise_scale_gemm(m: int, n: int, k: int,
159 + a_scale_group_shape,
160 + b_scale_group_shape, use_bias: bool):
161 + if k % b_scale_group_shape[0] != 0 or n % b_scale_group_shape[1] != 0:
162 + return
163 + if m % a_scale_group_shape[0] != 0 or k % a_scale_group_shape[1] != 0:
164 + return
165 + cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape,
166 + use_bias)
167 +
168 +
169 +@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
170 +@pytest.mark.parametrize("a_scale_group_shape",
171 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
172 +@pytest.mark.parametrize("b_scale_group_shape",
173 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
174 @pytest.mark.parametrize("use_bias", [True, False])
134 -def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
135 - per_out_ch: bool, use_bias: bool):
136 - cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
175 +def test_cutlass_int8_gemm(m: int, n: int, k: int, a_scale_group_shape,
176 + b_scale_group_shape, use_bias: bool):
177 + cutlass_int8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape,
178 + use_bias)
179
180
139 -@pytest.mark.parametrize("per_act_token", [True, False])
140 -@pytest.mark.parametrize("per_out_ch", [True, False])
181 +@pytest.mark.parametrize("a_scale_group_shape",
182 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
183 +@pytest.mark.parametrize("b_scale_group_shape",
184 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
185 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
186 @pytest.mark.parametrize("use_bias", [True, False])
143 -def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
187 +def test_cutlass_int8_gemm_output_dtype(a_scale_group_shape,
188 + b_scale_group_shape,
189 out_dtype: Type[torch.dtype],
190 use_bias: bool):
191 cutlass_int8_gemm_helper(512,
192 512,
193 512,
149 - per_act_token,
150 - per_out_ch,
194 + a_scale_group_shape,
195 + b_scale_group_shape,
196 use_bias,
197 out_dtype=out_dtype)
198
199
155 -@pytest.mark.parametrize("per_act_token", [True, False])
156 -@pytest.mark.parametrize("per_out_ch", [True, False])
200 +@pytest.mark.parametrize("a_scale_group_shape",
201 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
202 +@pytest.mark.parametrize("b_scale_group_shape",
203 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
204 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
205 @pytest.mark.parametrize("use_bias", [True, False])
206 @pytest.mark.skipif(not current_platform.has_device_capability(89),
207 reason="FP8 is not supported on this GPU type.")
161 -def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
208 +def test_cutlass_fp8_gemm_output_dtype(a_scale_group_shape,
209 + b_scale_group_shape,
210 out_dtype: Type[torch.dtype],
211 use_bias: bool):
212 cutlass_fp8_gemm_helper(512,
213 512,
214 512,
167 - per_act_token,
168 - per_out_ch,
215 + a_scale_group_shape,
216 + b_scale_group_shape,
217 use_bias,
218 out_dtype=out_dtype)
219
220
173 -@pytest.mark.parametrize("per_act_token", [True, False])
174 -@pytest.mark.parametrize("per_out_ch", [True, False])
221 +@pytest.mark.parametrize("a_scale_group_shape,b_scale_group_shape",
222 + [((1, 128), (128, 128))])
223 +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
224 +@pytest.mark.parametrize("use_bias", [False])
225 +@pytest.mark.skipif(not current_platform.has_device_capability(90),
226 + reason="FP8 blockwise is not supported on this GPU type.")
227 +def test_cutlass_fp8_blockwise_scale_gemm_dtype(a_scale_group_shape,
228 + b_scale_group_shape,
229 + out_dtype: Type[torch.dtype],
230 + use_bias: bool):
231 + cutlass_fp8_gemm_helper(512,
232 + 512,
233 + 512,
234 + a_scale_group_shape,
235 + b_scale_group_shape,
236 + use_bias,
237 + out_dtype=out_dtype)
238 +
239 +
240 +@pytest.mark.parametrize("a_scale_group_shape",
241 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
242 +@pytest.mark.parametrize("b_scale_group_shape",
243 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
244 @pytest.mark.parametrize("use_bias", [True, False])
245 @pytest.mark.parametrize("device", CUDA_DEVICES)
246 @pytest.mark.skipif(not current_platform.has_device_capability(89),
247 reason="FP8 is not supported on this GPU type.")
179 -def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
248 +def test_cutlass_fp8_gemm_devices(a_scale_group_shape, b_scale_group_shape,
249 use_bias: bool, device: str):
181 - cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch, use_bias,
182 - torch.bfloat16, device)
250 + cutlass_fp8_gemm_helper(512, 512, 512, a_scale_group_shape,
251 + b_scale_group_shape, use_bias, torch.bfloat16,
252 + device)
253
254
185 -@pytest.mark.parametrize("per_act_token", [True, False])
186 -@pytest.mark.parametrize("per_out_ch", [True, False])
255 +@pytest.mark.parametrize("a_scale_group_shape",
256 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
257 +@pytest.mark.parametrize("b_scale_group_shape",
258 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
259 @pytest.mark.parametrize("use_bias", [True, False])
260 @pytest.mark.parametrize("device", CUDA_DEVICES)
189 -def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
261 +def test_cutlass_int8_gemm_devices(a_scale_group_shape, b_scale_group_shape,
262 use_bias: bool, device: str):
263 cutlass_int8_gemm_helper(512,
264 512,
265 512,
194 - per_act_token,
195 - per_out_ch,
266 + a_scale_group_shape,
267 + b_scale_group_shape,
268 use_bias,
269 out_dtype=torch.bfloat16,
270 device=device)
@@ -203,28 +275,32 @@ def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
275 # of a large power of two. In any case, the kernel will have a naive fallback
276 # when N and K are not divisible by 16. But M is the number of tokens and the
277 # kernel must handle any M thrown at it.
206 -@pytest.mark.parametrize("per_act_token", [True, False])
207 -@pytest.mark.parametrize("per_out_ch", [True, False])
278 +@pytest.mark.parametrize("a_scale_group_shape",
279 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
280 +@pytest.mark.parametrize("b_scale_group_shape",
281 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
282 @pytest.mark.parametrize("use_bias", [True, False])
283 @pytest.mark.skipif(not current_platform.has_device_capability(89),
284 reason="FP8 is not supported on this GPU type.")
211 -def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
285 +def test_cutlass_fp8_gemm_m_sweep(a_scale_group_shape, b_scale_group_shape,
286 use_bias: bool):
287 for nk in range(32, 128, 32):
288 for m in range(1, 128):
215 - cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
216 - use_bias)
289 + cutlass_fp8_gemm_helper(m, nk, nk, a_scale_group_shape,
290 + b_scale_group_shape, use_bias)
291
292
219 -@pytest.mark.parametrize("per_act_token", [True, False])
220 -@pytest.mark.parametrize("per_out_ch", [True, False])
293 +@pytest.mark.parametrize("a_scale_group_shape",
294 + [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
295 +@pytest.mark.parametrize("b_scale_group_shape",
296 + [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE])
297 @pytest.mark.parametrize("use_bias", [True, False])
222 -def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
298 +def test_cutlass_int8_gemm_m_sweep(a_scale_group_shape, b_scale_group_shape,
299 use_bias: bool):
300 for nk in range(32, 128, 32):
301 for m in range(1, 128):
226 - cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
227 - use_bias)
302 + cutlass_int8_gemm_helper(m, nk, nk, a_scale_group_shape,
303 + b_scale_group_shape, use_bias)
304
305
306 @pytest.mark.parametrize("m", [32, 64, 128])
tests/kernels/utils.py
+30 -2
@@ -1119,8 +1119,36 @@ def baseline_scaled_mm(a: torch.Tensor,
1119 scale_b: torch.Tensor,
1120 out_dtype: Type[torch.dtype],
1121 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
1122 - output = (scale_a * (scale_b * (torch.mm(
1123 - a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
1122 +
1123 + # We treat N-dimensional group scaling as extended numpy-style broadcasting
1124 + # in numpy simply stretches dimensions with an extent of 1 to match the
1125 + # the target shape by repeating the data along that dimension (broadcasting)
1126 + # , we extend these semantics to say if the extent of a dimension in the
1127 + # source shape is not 1 and does not match the target shape we repeat each
1128 + # element along that dimension src_shape[dim] // target_shape[dim] times
1129 + # example if we have:
1130 + # a = [[1, 2], and target_shape = (2, 4)
1131 + # [3, 4]]
1132 + # then we would expand a to:
1133 + # a = [[1, 1, 2, 2],
1134 + # [3, 3, 4, 4]]
1135 + # NOTE this function this function does not explicitly broadcast dimensions
1136 + # with an extent of 1, since this can be done implicitly by pytorch
1137 + def group_broadcast(t, shape):
1138 + for i, s in enumerate(shape):
1139 + if t.shape[i] != s and t.shape[i] != 1:
1140 + assert s % t.shape[i] == 0
1141 + t = t.unsqueeze(i + 1)\
1142 + .expand(*t.shape[:i+1], s // t.shape[i], *t.shape[i+1:])\
1143 + .flatten(i, i + 1)
1144 + return t
1145 +
1146 + scale_a = group_broadcast(scale_a, a.shape)
1147 + scale_b = group_broadcast(scale_b, b.shape)
1148 +
1149 + output = torch.mm((scale_a * a.to(dtype=torch.float32)),
1150 + (scale_b * b.to(dtype=torch.float32))).to(out_dtype)
1151 +
1152 if bias is not None:
1153 output = output + bias
1154
vllm/_custom_ops.py
+22
@@ -441,6 +441,28 @@ def cutlass_scaled_mm(a: torch.Tensor,
441 scale_b: torch.Tensor,
442 out_dtype: torch.dtype,
443 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
444 + """
445 + `cutlass_scaled_mm` implements a fused version of
446 + `output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)`
447 + where scale_a * a and scale_b * b are implemented using numpy-style
448 + broadcasting.
449 +
450 + In order to support blockwise scaling like found in DeepSeek V3 we also
451 + support extended "group" broadcast rules. We extend the numpy-style
452 + broadcasting rules with the following rule:
453 + "if the extent of a dimension in the source shape is between 1 and
454 + corresponding extent in the target shape we repeat each element along
455 + that dimension src_shape[dim] // target_shape[dim] times consecutively"
456 + example if we have:
457 + a = [[1, 2], and target_shape = (2, 4)
458 + [3, 4]]
459 + then we would expand a to:
460 + a = [[1, 1, 2, 2],
461 + [3, 3, 4, 4]]
462 + currently we only support the case:
463 + scale_a.shape * [1, 128] == a.shape
464 + scale_b.shape * [128, 128] == b.shape
465 + """
466 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
467 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
468 assert bias is None or bias.shape[0] == b.shape[