@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96d999fbe8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Kernel] Initial Machete W4A8 support + Refactors (#9855)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Lucas Wilkinson committed Nov 18, 2024 at 14:59 UTC 96d999fbe8d610fa4c5b7cad6bb0d0158d1d5b8b
28 files changed +2616 -1694
benchmarks/kernels/benchmark_machete.py
+385 -134
@@ -2,8 +2,10 @@ import argparse
2 import copy
3 import itertools
4 import math
5 +import os
6 import pickle as pkl
7 import time
8 +from dataclasses import dataclass
9 from itertools import product
10 from typing import Callable, Iterable, List, Optional, Tuple
11
@@ -15,11 +17,12 @@ from weight_shapes import WEIGHT_SHAPES
17
18 from vllm import _custom_ops as ops
19 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, marlin_permute_scales)
20 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, marlin_permute_scales,
21 + marlin_zero_points)
22 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
23 MarlinWorkspace)
24 from vllm.model_executor.layers.quantization.utils.quant_utils import (
22 - gptq_pack, pack_rows, quantize_weights)
25 + pack_rows, quantize_weights)
26 from vllm.scalar_type import ScalarType, scalar_types
27 from vllm.utils import FlexibleArgumentParser
28
@@ -27,149 +30,349 @@ DEFAULT_MODELS = ["meta-llama/Llama-3-8b", "meta-llama/Llama-2-70b-hf"]
30 DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512, 1024]
31 DEFAULT_TP_SIZES = [1]
32
33 +NVTX_PROFILE = os.environ.get("NVTX_PROFILE", False)
34 +
35 +if NVTX_PROFILE:
36 + import nvtx
37 +
38 +
39 +def terse_type_name(dt):
40 + return {
41 + torch.bfloat16: "bf16",
42 + torch.float16: "fp16",
43 + torch.int8: "int8",
44 + torch.float8_e4m3fn: "fp8",
45 + torch.bfloat16: "bf16",
46 + torch.float: "float",
47 + torch.int: "int",
48 + }[dt]
49 +
50 +
51 +@dataclass
52 +class BenchmarkTensors:
53 + w_ref: torch.Tensor
54 + a: torch.Tensor
55 +
56 + w_q: torch.Tensor
57 + group_size: Optional[int]
58 + wtype: ScalarType
59 + w_g_s: torch.Tensor
60 + w_g_zp: Optional[torch.Tensor]
61 + w_ch_s: Optional[torch.Tensor]
62 + w_tok_s: Optional[torch.Tensor]
63 +
64 +
65 +@dataclass
66 +class TypeConfig:
67 + act_type: torch.dtype
68 + weight_type: ScalarType
69 + output_type: Optional[torch.dtype]
70 + group_scale_type: Optional[torch.dtype]
71 + group_zero_type: Optional[torch.dtype]
72 + channel_scale_type: Optional[torch.dtype]
73 + token_scale_type: Optional[torch.dtype]
74 +
75 +
76 +def rand_data(shape, dtype=torch.float16, scale=1):
77 + if dtype.is_floating_point:
78 + return (scale * torch.rand(shape, device="cuda") - 0.3).to(dtype)
79 + else:
80 + return torch.randint(-15, 15, shape, dtype=dtype, device="cuda")
81 +
82 +
83 +def quantize_and_pack(atype: torch.dtype,
84 + w: torch.Tensor,
85 + wtype: ScalarType,
86 + stype: Optional[torch.dtype],
87 + group_size: Optional[int],
88 + zero_points: bool = False):
89 + assert wtype.is_integer(), "TODO: support floating point weights"
90 +
91 + w_ref, w_q, w_s, w_zp = quantize_weights(
92 + w,
93 + wtype,
94 + group_size=group_size,
95 + zero_points=zero_points,
96 + # to match how the kernel applies zps
97 + ref_zero_points_after_scales=True)
98
31 -def machete_pack_weights(w_q: torch.tensor, wtype: ScalarType) -> torch.tensor:
99 w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
33 - w_q = w_q.t().contiguous().t() # make col major
34 - return ops.machete_prepack_B(w_q, wtype)
100 + return w_ref, w_q, w_s, w_zp
101
102
37 -def make_bench_tensors(
38 - atype: torch.dtype, wtype: ScalarType, group_size: int, m: int, n: int,
39 - k: int
40 -) -> Tuple[torch.tensor, List[Tuple[torch.tensor, torch.tensor, torch.tensor,
41 - torch.tensor]]]:
42 - assert wtype.is_integer(), "TODO: support floating point weights"
103 +def create_bench_tensors(shape: Tuple[int, int, int], types: TypeConfig,
104 + group_size: Optional[int]) -> List[BenchmarkTensors]:
105 + m, n, k = shape
106
107 # we want to make sure that weights don't fit into L2 cache between runs so
108 # we construct enough weights to exceed L2 cache, which is 50mb on a H100
109 # so we target total weight size > 2*50mb
47 - num_weights = math.ceil(2 * 50 * 1024**2 * 8 / (k * n * wtype.size_bits))
48 -
49 - a = torch.randn((m, k), device="cuda", dtype=atype) * 5
50 - weights = [
51 - torch.randn((k, n), device="cuda", dtype=atype)
52 - for _ in range(num_weights)
53 - ]
54 - quanitized_weights = [
55 - quantize_weights(w, wtype, group_size) for w in weights
56 - ]
57 -
58 - return a, quanitized_weights
110 + num_weights = math.ceil(2 * 50 * 1024**2 * 8 /
111 + (k * n * types.weight_type.size_bits))
112 +
113 + a = rand_data((m, k), types.act_type, scale=5)
114 +
115 + benchmark_tensors: List[BenchmarkTensors] = []
116 + for _ in range(num_weights):
117 + w = rand_data((k, n), types.act_type, scale=5)
118 +
119 + if types.group_scale_type is not None:
120 + w = w.to(types.group_scale_type)
121 + if w.dtype.itemsize == 1:
122 + w = w.to(torch.float16)
123 +
124 + w_ref, w_q_packed, w_s, w_zp = quantize_and_pack(
125 + a.dtype, w, types.weight_type, types.group_scale_type, group_size,
126 + types.group_zero_type is not None)
127 +
128 + if not a.dtype.is_floating_point:
129 + aiinfo = torch.iinfo(a.dtype)
130 + w_ref = w_ref.round().clamp(aiinfo.min, aiinfo.max)
131 +
132 + w_ref = w_ref.to(torch.float32)
133 +
134 + w_ch_s = None if types.channel_scale_type is None else\
135 + rand_data((n,), types.channel_scale_type)
136 + w_tok_s = None if types.token_scale_type is None else\
137 + rand_data((m,), types.token_scale_type)
138 +
139 + benchmark_tensors.append(
140 + BenchmarkTensors(w_ref=w_ref,
141 + a=a,
142 + w_q=w_q_packed,
143 + wtype=types.weight_type,
144 + w_g_s=w_s,
145 + w_g_zp=w_zp,
146 + group_size=group_size,
147 + w_ch_s=w_ch_s,
148 + w_tok_s=w_tok_s))
149 +
150 + return benchmark_tensors
151 +
152 +
153 +def torch_matmul_f16_create_bench_fn(bt: BenchmarkTensors) -> Callable:
154 + a = bt.a
155 + w = bt.w_ref.to(bt.a.dtype) # use float reference tensor
156 + if a.dtype not in [torch.float16, torch.bfloat16]:
157 + a = a.to(torch.float16)
158 + w = w.to(torch.float16)
159 + return lambda: torch.matmul(a, w)
160 +
161 +
162 +def cutlass_scaled_mm_create_bench_fn(bt: BenchmarkTensors) -> Callable:
163 + if bt.w_ch_s is not None and bt.w_tok_s is not None:
164 + scale_a = bt.w_tok_s.to(torch.float32)
165 + scale_b = bt.w_ch_s.to(torch.float32)
166 + else:
167 + scale_a = torch.tensor(1.0, dtype=torch.float32, device=bt.a.device)
168 + scale_b = torch.tensor(1.0, dtype=torch.float32, device=bt.a.device)
169 + w_col_major = bt.w_ref.to(bt.a.dtype).t().contiguous().t()
170 + return lambda: ops.cutlass_scaled_mm(
171 + bt.a, w_col_major, scale_a, scale_b, out_dtype=torch.float16)
172 +
173 +
174 +def marlin_create_bench_fn(bt: BenchmarkTensors) -> Callable:
175 + device = bt.a.device
176 +
177 + workspace = MarlinWorkspace(bt.w_ref.shape[1], GPTQ_MARLIN_MIN_THREAD_N,
178 + GPTQ_MARLIN_MAX_PARALLEL)
179 +
180 + if bt.w_g_zp is None:
181 + w_zp = torch.empty(0, dtype=torch.int, device=device)
182 + else:
183 + w_zp = marlin_zero_points(bt.w_g_zp, bt.w_ref.shape[0],
184 + bt.w_ref.shape[1], bt.wtype.size_bits)
185 +
186 + if bt.group_size is None:
187 + w_s = torch.tensor([], device="cuda", dtype=torch.half)
188 + else:
189 + w_s = marlin_permute_scales(bt.w_g_s, bt.w_ref.shape[0],
190 + bt.w_ref.shape[1], bt.group_size)
191 +
192 + sort_indices = torch.empty(0, dtype=torch.int, device=device)
193 + g_idx = torch.empty(0, dtype=torch.int, device=device)
194 + w_q = ops.gptq_marlin_repack(bt.w_q, sort_indices, bt.w_ref.shape[0],
195 + bt.w_ref.shape[1], bt.wtype.size_bits)
196 +
197 + if bt.a.dtype.is_floating_point:
198 + assert bt.w_ch_s is None
199 + assert bt.w_tok_s is None
200 + assert bt.group_size is not None
201 +
202 + fn = lambda: ops.gptq_marlin_gemm(a=bt.a,
203 + b_q_weight=w_q,
204 + b_scales=w_s,
205 + b_zeros=w_zp,
206 + g_idx=g_idx,
207 + perm=sort_indices,
208 + workspace=workspace.scratch,
209 + b_q_type=bt.wtype,
210 + size_m=bt.a.shape[0],
211 + size_n=bt.w_ref.shape[1],
212 + size_k=bt.w_ref.shape[0],
213 + is_k_full=True)
214 + else:
215 + assert bt.a.dtype == torch.int8
216 + assert bt.wtype == scalar_types.uint4b8
217 +
218 + if bt.w_ch_s is not None:
219 + s_ch = bt.w_ch_s.to(torch.float32)
220 + else:
221 + s_ch = torch.ones(bt.w_ref.shape[1],
222 + dtype=torch.float32,
223 + device=device)
224 +
225 + if bt.w_tok_s is not None:
226 + s_tok = bt.w_tok_s.to(torch.float32)
227 + else:
228 + s_tok = torch.ones(bt.a.shape[0],
229 + dtype=torch.float32,
230 + device=device)
231 +
232 + fn = lambda: ops.marlin_qqq_gemm(a=bt.a,
233 + b_q_weight=w_q,
234 + s_group=w_s,
235 + s_tok=s_tok,
236 + s_ch=s_ch,
237 + workspace=workspace.scratch,
238 + size_m=bt.a.shape[0],
239 + size_n=bt.w_ref.shape[1],
240 + size_k=bt.w_ref.shape[0])
241 +
242 + return fn
243 +
244 +
245 +def machete_create_bench_fn(bt: BenchmarkTensors,
246 + out_type=torch.dtype,
247 + schedule=None) -> Callable:
248 + w_q = bt.w_q.t().contiguous().t() # make col major
249 + w_q = ops.machete_prepack_B(w_q, bt.a.dtype, bt.wtype,
250 + None if bt.w_g_s is None else bt.w_g_s.dtype)
251 +
252 + w_g_zp = bt.w_g_zp
253 + if w_g_zp is not None:
254 + w_g_zp = -1 * bt.w_g_s * (w_g_zp.to(bt.w_g_s.dtype))
255 +
256 + return lambda: ops.machete_mm(
257 + a=bt.a,
258 + b_q=bt.w_q,
259 + b_type=bt.wtype,
260 + b_group_scales=bt.w_g_s,
261 + b_group_zeros=w_g_zp,
262 + b_group_size=bt.group_size,
263 + b_channel_scales=bt.w_ch_s,
264 + a_token_scales=bt.w_tok_s,
265 + out_type=out_type,
266 + schedule=schedule,
267 + )
268
269
270 # impl
271
63 -
272 # bench
65 -def bench_fn(label: str, sub_label: str, description: str,
66 - fn: Callable) -> TMeasurement:
273
68 - min_run_time = 1
69 - return TBenchmark.Timer(
70 - stmt="fn()",
274 +
275 +def bench_fns(label: str, sub_label: str, description: str,
276 + fns: List[Callable]):
277 +
278 + min_run_time = 1 if not NVTX_PROFILE else 0.1
279 + res = TBenchmark.Timer(
280 + stmt="""
281 + for fn in fns:
282 + fn()
283 + """,
284 globals={
72 - "fn": fn
285 + "fns": fns
286 },
287 label=label,
288 sub_label=sub_label,
289 description=description,
290 ).blocked_autorange(min_run_time=min_run_time)
291
292 + if NVTX_PROFILE:
293 + with nvtx.annotate("mm-bench"), nvtx.annotate(
294 + f"{label}|{sub_label}|{description}"):
295 + fns[0]()
296
80 -def loop_over_weights(
81 - a: torch.tensor, weights: List[Tuple[torch.tensor, torch.tensor,
82 - torch.tensor, torch.tensor]],
83 - fn: Callable[[torch.tensor, torch.tensor, torch.tensor, torch.tensor],
84 - None]):
85 - for w_ref, w_q, w_s, _ in weights:
86 - fn(a, w_ref, w_q, w_s)
297 + return res
298
299
300 _SWEEP_SCHEDULES_RESULTS: Optional[pd.DataFrame] = None
301 _SWEEP_SCHEDULES_RESULTS_CSV: Optional[str] = None
302
303
93 -def bench(atype: torch.dtype,
94 - wtype: ScalarType,
304 +def bench(types: TypeConfig,
305 group_size: int,
306 m: int,
307 k: int,
308 n: int,
309 label: str,
310 sub_label: str,
101 - benchmark_marlinv1: bool = True,
102 - sweep_schedules: bool = True) -> Iterable[TMeasurement]:
103 - global _SWEEP_SCHEDULES_RESULTS
104 -
105 - a, weights = make_bench_tensors(atype, wtype, group_size, m, n, k)
106 - sub_label += f", L={len(weights)}"
107 -
108 - weights_machete = [(w_ref, machete_pack_weights(w_q, wtype), w_s, w_zp)
109 - for w_ref, w_q, w_s, w_zp in weights]
311 + sweep_schedules: bool = True) -> List[TMeasurement]:
312 + benchmark_tensors = create_bench_tensors((m, n, k), types, group_size)
313 + sub_label += f", L={len(benchmark_tensors)}"
314 +
315 + name_type_string = f"W{types.weight_type}"+\
316 + f"-A{terse_type_name(types.act_type)}"
317 + if types.group_scale_type is not None:
318 + name_type_string += f"-GS{terse_type_name(types.group_scale_type)}"
319 + if types.group_zero_type is not None:
320 + name_type_string += f"-GZ{terse_type_name(types.group_zero_type)}"
321 + if group_size is not None:
322 + name_type_string += f"-G{group_size}"
323 + if types.channel_scale_type is not None:
324 + name_type_string += f"-CS{terse_type_name(types.channel_scale_type)}"
325 + if types.token_scale_type is not None:
326 + name_type_string += f"-TS{terse_type_name(types.token_scale_type)}"
327
328 timers = []
329 # pytorch impl
330 timers.append(
114 - bench_fn(
115 - label, sub_label, "torch.matmul", lambda: loop_over_weights(
116 - a,
117 - weights,
118 - lambda a, w_ref, w_q, w_s: torch.matmul(a, w_ref),
119 - )))
331 + bench_fns(
332 + label, sub_label, "torch.matmul (fp16)",
333 + [torch_matmul_f16_create_bench_fn(bt)
334 + for bt in benchmark_tensors]))
335
121 - if benchmark_marlinv1:
122 - w_ref = weights[0][0]
123 -
124 - w_zp_empty = torch.empty(0, dtype=torch.int, device=w_ref.device)
125 - sort_indices = torch.empty(0, dtype=torch.int, device=w_ref.device)
126 - g_idx = torch.empty(0, dtype=torch.int, device=w_ref.device)
127 -
128 - def marlinv1_pack_weights(w_q: torch.tensor) -> torch.tensor:
129 - w_q_gptq = gptq_pack(w_q, wtype.size_bits, *w_ref.shape)
130 - return ops.gptq_marlin_repack(w_q_gptq, sort_indices, *w_ref.shape,
131 - wtype.size_bits)
132 -
133 - def marlinv1_permute_scales(w_s: torch.tensor) -> torch.tensor:
134 - return marlin_permute_scales(w_s, *w_ref.shape, group_size)
135 -
136 - weights_marlinv1 = [(w_ref, marlinv1_pack_weights(w_q),
137 - marlinv1_permute_scales(w_s), w_zp)
138 - for w_ref, w_q, w_s, w_zp in weights]
139 -
140 - workspace = MarlinWorkspace(w_ref.shape[1], GPTQ_MARLIN_MIN_THREAD_N,
141 - GPTQ_MARLIN_MAX_PARALLEL)
142 -
143 - # marlinv1
336 + if types.act_type == torch.int8 or types.act_type == torch.float8_e4m3fn:
337 + timers.append(
338 + bench_fns(
339 + label, sub_label,
340 + f"cutlass_scaled_mm ({terse_type_name(types.act_type)})", [
341 + cutlass_scaled_mm_create_bench_fn(bt)
342 + for bt in benchmark_tensors
343 + ]))
344 +
345 + if types.act_type != torch.float8_e4m3fn:
346 timers.append(
145 - bench_fn(
146 - label, sub_label, "marlin_orig", lambda: loop_over_weights(
147 - a, weights_marlinv1, lambda a, w_ref, w_q, w_s: ops.
148 - gptq_marlin_gemm(a,
149 - w_q,
150 - w_s,
151 - w_zp_empty,
152 - g_idx,
153 - sort_indices,
154 - workspace.scratch,
155 - wtype,
156 - size_m=a.shape[0],
157 - size_n=w_ref.shape[1],
158 - size_k=w_ref.shape[0],
159 - is_k_full=True))))
347 + bench_fns(label, sub_label, f"marlin ({name_type_string})",
348 + [marlin_create_bench_fn(bt)
349 + for bt in benchmark_tensors]))
350
351 # machete
352 timers.append(
163 - bench_fn(
164 - label, sub_label, "machete_heuristic", lambda: loop_over_weights(
165 - a, weights_machete, lambda a, _, w_q, w_s: ops.machete_gemm(
166 - a, w_q, wtype, b_scales=w_s, b_group_size=group_size))))
353 + bench_fns(label, sub_label, f"machete ({name_type_string})", [
354 + machete_create_bench_fn(bt, out_type=types.output_type)
355 + for bt in benchmark_tensors
356 + ]))
357
358 if sweep_schedules:
359 + global _SWEEP_SCHEDULES_RESULTS
360 +
361 print("Finding best schedule for machete")
362 best = None
363 best_schedule = None
172 - schedules = ops.machete_supported_schedules(wtype)
364 + schedules = ops.machete_supported_schedules(
365 + a_type=types.act_type,
366 + b_type=types.weight_type,
367 + group_scales_type=types.group_scale_type,
368 + group_zeros_type=types.group_zero_type,
369 + token_scales_type=types.token_scale_type,
370 + channel_scales_type=types.channel_scale_type,
371 + out_type=types.output_type)
372 +
373 + if schedules is None or len(schedules) == 0:
374 + raise ValueError("No schedules found to sweep")
375 +
376 for schedule in reversed(schedules):
377 schedule_M = int(schedule.split("_")[0].split("x")[1])
378
@@ -177,16 +380,11 @@ def bench(atype: torch.dtype,
380 if schedule_M >= 2 * max(m, 16) or schedule_M < m // 4:
381 continue
382
180 - def run(a, _, w_q, w_s, schedule=schedule):
181 - ops.machete_gemm(a,
182 - w_q,
183 - wtype,
184 - w_s,
185 - b_group_size=group_size,
186 - schedule=schedule)
187 -
188 - res = bench_fn(label, sub_label, "machete_best",
189 - lambda: loop_over_weights(a, weights_machete, run))
383 + res = bench_fns(label, sub_label, "machete_best", [
384 + machete_create_bench_fn(
385 + bt, out_type=types.output_type, schedule=schedule)
386 + for bt in benchmark_tensors
387 + ])
388
389 results_row = {
390 "M": m,
@@ -213,25 +411,33 @@ def bench(atype: torch.dtype,
411
412
413 # runner
216 -def print_timers(timers: Iterable[TMeasurement]):
414 +def print_timers(timers: List[TMeasurement]):
415 compare = TBenchmark.Compare(timers)
416 compare.print()
417
418
221 -def run(dtype: torch.dtype, sweep_schedules: bool,
222 - MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
419 +def run(args, MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
420 + types = TypeConfig(
421 + act_type=args.act_type,
422 + weight_type=scalar_types.uint4b8 if args.group_zero_type is None \
423 + else scalar_types.uint4,
424 + output_type=args.out_type,
425 + group_scale_type=args.group_scale_type,
426 + group_zero_type=args.group_zero_type,
427 + channel_scale_type=args.channel_scale_type,
428 + token_scale_type=args.token_scale_type,
429 + )
430
224 - results = []
431 + results: List[TMeasurement] = []
432 for m, k, n in MKNs:
226 - timers = bench(dtype,
227 - scalar_types.uint4b8,
228 - 128,
433 + timers = bench(types,
434 + args.group_size,
435 m,
436 k,
437 n,
232 - f"{dtype}-gemm",
438 + f"{args.act_type}-gemm",
439 f"MKN=({m}x{k}x{n})",
234 - sweep_schedules=sweep_schedules)
440 + sweep_schedules=args.sweep_schedules)
441 print_timers(timers)
442 results.extend(timers)
443
@@ -240,7 +446,7 @@ def run(dtype: torch.dtype, sweep_schedules: bool,
446
447 # output makers
448 def make_output(
243 - data: Iterable[TMeasurement],
449 + data: List[TMeasurement],
450 MKNs: Iterable[Tuple[int, int, int]],
451 base_description: str,
452 timestamp=None,
@@ -262,7 +468,6 @@ def run_square_bench(args):
468 dim_sizes = list(
469 range(args.dim_start, args.dim_end + 1, args.dim_increment))
470 MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
265 -
471 data = run(args.dtype, args.sweep_schedules, MKNs)
472
473 make_output(data, MKNs, f"square_bench-{args.dtype}")
@@ -306,33 +511,49 @@ def run_model_bench(args):
511 for k, n in KNs:
512 MKNs.append((m, k, n))
513
309 - data = run(args.dtype, args.sweep_schedules, MKNs)
514 + data = run(args, MKNs)
515 model_bench_data.append(data)
516
517 + type_string = f"{args.act_type}"
518 +
519 # Print all results
520 for data, model_tp in zip(model_bench_data, models_tps):
521 model, tp_size = model_tp
315 - print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
522 + print(f"== Results {type_string} {model}-TP{tp_size} ====")
523 print_timers(data)
524
318 - timestamp = int(time.time())
525 + timestr = time.strftime("%Y%m%d-%H%M%S")
526
320 - all_data = []
527 + all_results = []
528 for d in model_bench_data:
322 - all_data.extend(d)
529 + all_results.extend(d)
530 +
531 # pickle all data
324 - with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
325 - pkl.dump(all_data, f)
532 + with open(f"model_bench-{type_string}-{timestr}.pkl", "wb") as f:
533 + args_dict = vars(args)
534 + args_dict.pop("func")
535 + pkl.dump({
536 + "args": args_dict,
537 + "results": all_results,
538 + }, f)
539
540
541 if __name__ == "__main__":
542
543 def to_torch_dtype(dt):
331 - if dt == "bfloat16":
332 - return torch.bfloat16
333 - if dt == "float16":
334 - return torch.float16
335 - raise ValueError("unsupported dtype")
544 + return {
545 + "bfloat16": torch.bfloat16,
546 + "float16": torch.float16,
547 + "int8": torch.int8,
548 + "float8_e4m3fn": torch.float8_e4m3fn,
549 + "int": torch.int,
550 + "float": torch.float,
551 + }[dt]
552 +
553 + class ToTorchDtype(argparse.Action):
554 +
555 + def __call__(self, parser, namespace, values, option_string=None):
556 + setattr(namespace, self.dest, to_torch_dtype(values))
557
558 parser = FlexibleArgumentParser(
559 description="""
@@ -352,12 +573,42 @@ Benchmark Machete GEMM.
573 """, # noqa: E501
574 formatter_class=argparse.RawTextHelpFormatter,
575 )
355 -
576 parser.add_argument(
357 - "--dtype",
358 - type=to_torch_dtype,
577 + "--act-type",
578 + action=ToTorchDtype,
579 required=True,
360 - help="Available options are ['bfloat16', 'float16']",
580 + choices=['bfloat16', 'float16', 'int8', 'float8_e4m3fn'],
581 + )
582 + parser.add_argument(
583 + "--group-scale-type",
584 + action=ToTorchDtype,
585 + choices=['bfloat16', 'float16'],
586 + )
587 + parser.add_argument(
588 + "--group-zero-type",
589 + type=to_torch_dtype,
590 + choices=['bfloat16', 'float16'],
591 + )
592 + parser.add_argument(
593 + "--channel-scale-type",
594 + action=ToTorchDtype,
595 + choices=['float'],
596 + )
597 + parser.add_argument(
598 + "--token-scale-type",
599 + action=ToTorchDtype,
600 + choices=['float'],
601 + )
602 + parser.add_argument(
603 + "--out-type",
604 + action=ToTorchDtype,
605 + choices=['bfloat16', 'float16'],
606 + )
607 + parser.add_argument(
608 + "--group-size",
609 + type=int,
610 + help="Available options are ['None', '-1', '128'], default=128",
611 + default=128,
612 )
613 parser.add_argument(
614 "--sweep-schedules",
benchmarks/kernels/graph_machete_bench.py
+3 -2
@@ -20,10 +20,11 @@ if __name__ == "__main__":
20 args = parser.parse_args()
21
22 with open(args.filename, 'rb') as f:
23 - data: List[TMeasurement] = pickle.load(f)
23 + data = pickle.load(f)
24 + raw_results: List[TMeasurement] = data["results"]
25
26 results = defaultdict(lambda: list())
26 - for v in data:
27 + for v in raw_results:
28 result = re.search(r"MKN=\(\d+x(\d+x\d+)\)", v.task_spec.sub_label)
29 if result is not None:
30 KN = result.group(1)
benchmarks/kernels/weight_shapes.py
+6
@@ -40,4 +40,10 @@ WEIGHT_SHAPES = {
40 ([8192, 57344], 1),
41 ([28672, 8192], 0),
42 ],
43 + "meta-llama/Llama-3.1-405b-hf": [
44 + ([16384, 18432], 1),
45 + ([16384, 16384], 0),
46 + ([16384, 106496], 1),
47 + ([53248, 16384], 0),
48 + ],
49 }
csrc/cutlass_extensions/cute_utils.cuh
+2 -2
@@ -20,9 +20,9 @@ CUTE_HOST_DEVICE static constexpr auto permute_layout(Layout l) {
20 // is the layout f(x) = x
21 template <typename Layout>
22 CUTE_HOST_DEVICE static constexpr bool is_identity_layout() {
23 - if constexpr (std::is_same_v<Layout, void>)
23 + if constexpr (std::is_same_v<Layout, void>) {
24 return true;
25 - else {
25 + } else {
26 constexpr auto coalesced_layout = coalesce(Layout{});
27 if constexpr (rank(coalesced_layout) == 1 &&
28 stride<0>(coalesced_layout) == 1) {
csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c2x.hpp renamed
+1
@@ -52,6 +52,7 @@
52 // clang-format off
53
54 #include "cutlass/epilogue/threadblock/fusion/visitor_2x.hpp"
55 +#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
56 #include "cute/tensor.hpp"
57
58 namespace cutlass::epilogue::threadblock {
csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp renamed
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp new
+317
@@ -0,0 +1,317 @@
1 +#include "cutlass_extensions/epilogue/broadcast_load_epilogue_c2x.hpp"
2 +
3 +/*
4 + This file defines custom epilogues for fusing channel scales, token scales,
5 + bias, and activation zero-points onto a GEMM operation using the
6 + CUTLASS 2.x API, for sm80 (Ampere) NVIDIA GPUs.
7 +
8 + Epilogues must contain a public type named EVTCompute of type Sm80EVT,
9 + as well as a static prepare_args function that constructs an
10 + EVTCompute::Arguments struct.
11 +*/
12 +
13 +namespace vllm::c2x {
14 +
15 +using namespace cute;
16 +
17 +/*
18 + * This class provides the common load descriptors for the
19 + * ScaledEpilogue[...] classes
20 + */
21 +template <typename ElementD, typename OutputTileThreadMap>
22 +struct ScaledEpilogueBase {
23 + protected:
24 + using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
25 +
26 + template <typename T>
27 + using ColOrScalarLoad =
28 + cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
29 + OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
30 +
31 + template <typename T>
32 + using RowOrScalarLoad =
33 + cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
34 + OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
35 +
36 + template <typename T>
37 + using ColLoad = cutlass::epilogue::threadblock::VisitorColBroadcast<
38 + OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
39 +
40 + template <typename T>
41 + using RowLoad = cutlass::epilogue::threadblock::VisitorRowBroadcast<
42 + OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
43 +
44 + template <typename T>
45 + using RowOrZeroLoad =
46 + cutlass::epilogue::threadblock::VisitorRowOrZeroBroadcast<
47 + OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
48 +
49 + // This utility function constructs the arguments for the load descriptors
50 + // from a tensor. It can handle both row and column, as well as row/column or
51 + // scalar cases.
52 + template <typename Descriptor, typename T>
53 + static auto args_from_tensor(torch::Tensor const& tensor) {
54 + using Arguments = typename Descriptor::Arguments;
55 + auto* data_ptr = static_cast<T*>(tensor.data_ptr());
56 + if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
57 + std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
58 + return Arguments{data_ptr, tensor.numel() != 1};
59 + } else {
60 + // it would technically work but no use case as data_ptr is never nullptr
61 + static_assert(!std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
62 + return Arguments{data_ptr};
63 + }
64 + }
65 +
66 + // This overload handles the case where there might not be a tensor, in which
67 + // case a nullptr is passed and a constant (0) is used.
68 + template <typename Descriptor, typename T>
69 + static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
70 + static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
71 + using Arguments = typename Descriptor::Arguments;
72 + auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
73 + return Arguments{data_ptr};
74 + }
75 +};
76 +
77 +/*
78 + This epilogue function defines a quantized GEMM operation similar to
79 + torch._scaled_mm.
80 +
81 + A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
82 + per-row. B can be quantized per-tensor or per-column.
83 + Any combination of per-tensor and per-row or column is supported.
84 + A and B must have symmetric quantization (zero point == 0).
85 +
86 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
87 + scales are applied elementwise with numpy-style broadcasting.
88 +
89 + ScaleA and ScaleB define the epilogue functions that apply the scales for
90 + the A and B operands respectively. These scales may be either per-tensor or
91 + per row or column.
92 +*/
93 +template <typename ElementD, typename OutputTileThreadMap>
94 +struct ScaledEpilogue
95 + : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
96 + private:
97 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
98 + using Accum = typename SUPER::Accum;
99 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
100 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
101 +
102 + using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
103 + cutlass::multiplies, float, float,
104 + cutlass::FloatRoundStyle::round_to_nearest>;
105 +
106 + using EVTCompute0 =
107 + cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
108 +
109 + using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
110 + cutlass::multiplies, ElementD, float,
111 + cutlass::FloatRoundStyle::round_to_nearest>;
112 +
113 + public:
114 + using EVTCompute =
115 + cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
116 + using ArgumentType = typename EVTCompute::Arguments;
117 +
118 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
119 + torch::Tensor const& b_scales) {
120 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
121 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
122 +
123 + typename EVTCompute0::Arguments evt0_args{b_args};
124 + return ArgumentType{a_args, evt0_args};
125 + }
126 +};
127 +
128 +/*
129 + * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
130 + * This bias can also be used in the per-tensor azp case, where the activation
131 + * zero point (azp) is used to compute an azp correction term,
132 + * which is folded into the bias.
133 + *
134 + * The bias tensor must be per-output channel.
135 + * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
136 + */
137 +template <typename ElementD, typename OutputTileThreadMap>
138 +struct ScaledEpilogueBias
139 + : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
140 + protected:
141 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
142 + using Accum = typename SUPER::Accum;
143 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
144 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
145 + using Bias = typename SUPER::template RowLoad<ElementD>;
146 + using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
147 + cutlass::multiplies, float, float,
148 + cutlass::FloatRoundStyle::round_to_nearest>;
149 +
150 + using EVTCompute0 =
151 + cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
152 +
153 + using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
154 + cutlass::multiply_add, ElementD, float,
155 + cutlass::FloatRoundStyle::round_to_nearest>;
156 +
157 + public:
158 + using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
159 + EVTCompute0, Bias>;
160 + using ArgumentType = typename EVTCompute::Arguments;
161 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
162 + torch::Tensor const& b_scales,
163 + torch::Tensor const& bias) {
164 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
165 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
166 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
167 +
168 + typename EVTCompute0::Arguments evt0_args{b_args};
169 + return ArgumentType{a_args, evt0_args, bias_args};
170 + }
171 +};
172 +
173 +/*
174 + * This epilogue directly supports per-tensor azp in int32 form.
175 + * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
176 + * term, which should already be multiplied with the scalar azp.
177 + * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
178 + *
179 + * This epilogue also supports bias, which remains per-channel.
180 + */
181 +template <typename ElementD, typename OutputTileThreadMap>
182 +struct ScaledEpilogueBiasAzp
183 + : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
184 + private:
185 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
186 + using Accum = typename SUPER::Accum;
187 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
188 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
189 + using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
190 +
191 + // This is the full AZP term, azp * J @ B, shape (1,n)
192 + using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
193 +
194 + // Compute float(accum - azp_adj), both operands are int32_t
195 + using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
196 + cutlass::minus, float, int32_t,
197 + cutlass::FloatRoundStyle::round_to_nearest>;
198 +
199 + using EVTComputeAzp =
200 + cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Accum, AzpWithAdj>;
201 +
202 + using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
203 + cutlass::multiplies, float, float,
204 + cutlass::FloatRoundStyle::round_to_nearest>;
205 +
206 + using EVTComputeScaleB =
207 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
208 + EVTComputeAzp>;
209 +
210 + using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
211 + cutlass::multiply_add, ElementD, float,
212 + cutlass::FloatRoundStyle::round_to_nearest>;
213 +
214 + public:
215 + using EVTCompute =
216 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
217 + EVTComputeScaleB, Bias>;
218 +
219 + using ArgumentType = typename EVTCompute::Arguments;
220 +
221 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
222 + torch::Tensor const& b_scales,
223 + torch::Tensor const& azp_adj,
224 + c10::optional<torch::Tensor> const& bias) {
225 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
226 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
227 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
228 + auto azp_adj_args =
229 + SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
230 +
231 + typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
232 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
233 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
234 + }
235 +};
236 +
237 +/*
238 + * This epilogue supports per-token azp by computing and applying
239 + * the correction term using a rank-1 update. If the term were materialized,
240 + * it would require O(m*n) space, and this way it only requires O(m+n) space.
241 + * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
242 + * point for each row of A.
243 + * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
244 + *
245 + * This epilogue also supports bias, which remains per-channel.
246 + */
247 +template <typename ElementD, typename OutputTileThreadMap>
248 +struct ScaledEpilogueBiasAzpToken
249 + : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
250 + private:
251 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
252 + using Accum = typename SUPER::Accum;
253 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
254 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
255 + using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
256 +
257 + // Per-token azp term, shape (m,1)
258 + using Azp = typename SUPER::template ColLoad<int32_t>;
259 +
260 + // This is the AZP adjustment term, J @ B, shape (1,n)
261 + using AzpAdj = typename SUPER::template RowLoad<int32_t>;
262 +
263 + // Compute azp * azp_adj
264 + using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
265 + cutlass::multiplies, int32_t, int32_t,
266 + cutlass::FloatRoundStyle::round_to_nearest>;
267 +
268 + using EVTComputeAzp =
269 + cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Azp, AzpAdj>;
270 +
271 + // Compute float(accum - azp*azp_adj), all operands are int32_t
272 + using ComputeAcc = cutlass::epilogue::threadblock::VisitorCompute<
273 + cutlass::minus, float, int32_t,
274 + cutlass::FloatRoundStyle::round_to_nearest>;
275 +
276 + using EVTComputeAcc =
277 + cutlass::epilogue::threadblock::Sm80EVT<ComputeAcc, Accum, EVTComputeAzp>;
278 +
279 + using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
280 + cutlass::multiplies, float, float,
281 + cutlass::FloatRoundStyle::round_to_nearest>;
282 +
283 + using EVTComputeScaleB =
284 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
285 + EVTComputeAcc>;
286 +
287 + using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
288 + cutlass::multiply_add, ElementD, float,
289 + cutlass::FloatRoundStyle::round_to_nearest>;
290 +
291 + public:
292 + using EVTCompute =
293 + cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
294 + EVTComputeScaleB, Bias>;
295 +
296 + using ArgumentType = typename EVTCompute::Arguments;
297 +
298 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
299 + torch::Tensor const& b_scales,
300 + torch::Tensor const& azp_adj,
301 + torch::Tensor const& azp,
302 + c10::optional<torch::Tensor> const& bias) {
303 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
304 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
305 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
306 + auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
307 + auto azp_adj_args =
308 + SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
309 +
310 + typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
311 + typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
312 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
313 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
314 + }
315 +};
316 +
317 +}; // namespace vllm::c2x
\ No newline at end of file
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp new
+315
@@ -0,0 +1,315 @@
1 +#include "cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp"
2 +
3 +/*
4 + This file defines custom epilogues for fusing channel scales, token scales,
5 + bias, and activation zero-points onto a GEMM operation using the
6 + CUTLASS 3.x API, for NVIDIA GPUs with sm90a (Hopper) or later.
7 +
8 + Epilogues must contain a public type named EVTCompute of type Sm90EVT,
9 + as well as a static prepare_args function that constructs an
10 + EVTCompute::Arguments struct.
11 +*/
12 +
13 +namespace vllm::c3x {
14 +
15 +using namespace cute;
16 +
17 +/*
18 + * This class provides the common load descriptors for the
19 + * ScaledEpilogue[...] classes
20 + */
21 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
22 +struct ScaledEpilogueBase {
23 + protected:
24 + using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
25 +
26 + template <typename T>
27 + using ColOrScalarLoad = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
28 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
29 + Stride<Int<1>, Int<0>, Int<0>>>;
30 +
31 + template <typename T>
32 + using RowOrScalarLoad = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
33 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
34 + Stride<Int<0>, Int<1>, Int<0>>>;
35 +
36 + // Don't want to support nullptr by default
37 + template <typename T, bool EnableNullPtr = false>
38 + using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
39 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
40 + Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
41 +
42 + // Don't want to support nullptr by default
43 + template <typename T, bool EnableNullPtr = false>
44 + using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
45 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
46 + Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
47 +
48 + // This utility function constructs the arguments for the load descriptors
49 + // from a tensor. It can handle both row and column, as well as row/column or
50 + // scalar cases.
51 + template <typename Descriptor, typename T>
52 + static auto args_from_tensor(torch::Tensor const& tensor) {
53 + using Arguments = typename Descriptor::Arguments;
54 + auto* data_ptr = static_cast<T*>(tensor.data_ptr());
55 + if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
56 + std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
57 + return Arguments{data_ptr, tensor.numel() != 1};
58 + } else {
59 + static_assert(!std::is_same_v<Descriptor, ColLoad<T, true>> &&
60 + !std::is_same_v<Descriptor, RowLoad<T, true>>);
61 + return Arguments{data_ptr};
62 + }
63 + }
64 +
65 + // This overload handles the case where there might not be a tensor, in which
66 + // case a nullptr is passed and a constant (0) is used.
67 + template <typename Descriptor, typename T>
68 + static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
69 + using Arguments = typename Descriptor::Arguments;
70 + auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
71 + static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
72 + std::is_same_v<Descriptor, RowLoad<T, true>>);
73 + return Arguments{data_ptr};
74 + }
75 +};
76 +
77 +/*
78 + This epilogue function defines a quantized GEMM operation similar to
79 + torch.scaled_mm_.
80 +
81 + A and B may be both either int8 or fp8_e4m3. A can be
82 + quantized per-tensor or per-row. B can be quantized per-tensor or per-column.
83 + Any combination of per-tensor and per-row or column is supported.
84 + A and B must have symmetric quantization (zero point == 0).
85 +
86 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
87 + scales are applied elementwise with numpy-style broadcasting.
88 +
89 + ScaleA and ScaleB define the epilogue functions that apply the scales for
90 + the A and B operands respectively. These scales may be either per-tensor or
91 + per row or column.
92 +*/
93 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
94 +struct ScaledEpilogue
95 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
96 + private:
97 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
98 + using Accum = typename SUPER::Accum;
99 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
100 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
101 +
102 + using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
103 + cutlass::multiplies, float, float,
104 + cutlass::FloatRoundStyle::round_to_nearest>;
105 +
106 + using EVTCompute0 =
107 + cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
108 +
109 + using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
110 + cutlass::multiplies, ElementD, float,
111 + cutlass::FloatRoundStyle::round_to_nearest>;
112 +
113 + public:
114 + using EVTCompute =
115 + cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
116 + using ArgumentType = typename EVTCompute::Arguments;
117 +
118 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
119 + torch::Tensor const& b_scales) {
120 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
121 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
122 +
123 + typename EVTCompute0::Arguments evt0_args{b_args};
124 + return ArgumentType{a_args, evt0_args};
125 + }
126 +};
127 +
128 +/*
129 + * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
130 + * This bias can also be used in the per-tensor azp case, where the activation
131 + * zero point (azp) is used to compute an azp correction term,
132 + * which is folded into the bias.
133 + *
134 + * The bias tensor must be per-output channel.
135 + * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
136 + */
137 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
138 +struct ScaledEpilogueBias
139 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
140 + private:
141 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
142 + using Accum = typename SUPER::Accum;
143 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
144 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
145 + using Bias = typename SUPER::template RowLoad<ElementD>;
146 +
147 + using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
148 + cutlass::multiplies, float, float,
149 + cutlass::FloatRoundStyle::round_to_nearest>;
150 +
151 + using EVTCompute0 =
152 + cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
153 +
154 + using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
155 + cutlass::multiply_add, ElementD, float,
156 + cutlass::FloatRoundStyle::round_to_nearest>;
157 +
158 + public:
159 + using EVTCompute =
160 + cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
161 +
162 + using ArgumentType = typename EVTCompute::Arguments;
163 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
164 + torch::Tensor const& b_scales,
165 + torch::Tensor const& bias) {
166 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
167 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
168 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
169 +
170 + typename EVTCompute0::Arguments evt0_args{b_args};
171 + return ArgumentType{a_args, evt0_args, bias_args};
172 + }
173 +};
174 +
175 +/*
176 + * This epilogue directly supports per-tensor azp in int32 form.
177 + * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
178 + * term, which should already be multiplied with the scalar azp.
179 + * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
180 + *
181 + * This epilogue also supports bias, which remains per-channel.
182 + */
183 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
184 +struct ScaledEpilogueBiasAzp
185 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
186 + private:
187 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
188 + using Accum = typename SUPER::Accum;
189 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
190 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
191 + using Bias = typename SUPER::template RowLoad<ElementD, true>;
192 +
193 + // This is the full AZP term, azp * J @ B, shape (1,n)
194 + using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
195 +
196 + // Compute float(accum - azp_adj), both operands are int32_t
197 + using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
198 + cutlass::minus, float, int32_t,
199 + cutlass::FloatRoundStyle::round_to_nearest>;
200 +
201 + using EVTComputeAzp =
202 + cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Accum, AzpWithAdj>;
203 +
204 + using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
205 + cutlass::multiplies, float, float,
206 + cutlass::FloatRoundStyle::round_to_nearest>;
207 +
208 + using EVTComputeScaleB =
209 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAzp>;
210 +
211 + using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
212 + cutlass::multiply_add, ElementD, float,
213 + cutlass::FloatRoundStyle::round_to_nearest>;
214 +
215 + public:
216 + using EVTCompute =
217 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
218 + EVTComputeScaleB, Bias>;
219 + using ArgumentType = typename EVTCompute::Arguments;
220 +
221 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
222 + torch::Tensor const& b_scales,
223 + torch::Tensor const& azp_adj,
224 + c10::optional<torch::Tensor> const& bias) {
225 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
226 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
227 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
228 + auto azp_adj_args =
229 + SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
230 +
231 + typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
232 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
233 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
234 + }
235 +};
236 +
237 +/*
238 + * This epilogue supports per-token azp by computing and applying
239 + * the correction term using a rank-1 update. If the term were materialized,
240 + * it would require O(m*n) space, and this way it only requires O(m+n) space.
241 + * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
242 + * point for each row of A.
243 + * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
244 + *
245 + * This epilogue also supports bias, which remains per-channel.
246 + */
247 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
248 +struct ScaledEpilogueBiasAzpToken
249 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
250 + private:
251 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
252 + using Accum = typename SUPER::Accum;
253 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
254 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
255 + using Bias = typename SUPER::template RowLoad<ElementD, true>;
256 +
257 + // Per-token azp term, shape (m,1)
258 + using Azp = typename SUPER::template ColLoad<int32_t>;
259 +
260 + // This is the AZP adjustment term, J @ B, shape (1,n)
261 + using AzpAdj = typename SUPER::template RowLoad<int32_t>;
262 +
263 + // Compute azp * azp_adj
264 + using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
265 + cutlass::multiplies, int32_t, int32_t,
266 + cutlass::FloatRoundStyle::round_to_nearest>;
267 +
268 + using EVTComputeAzp =
269 + cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Azp, AzpAdj>;
270 +
271 + // Compute float(accum - azp*azp_adj), all operands are int32_t
272 + using ComputeAcc = cutlass::epilogue::fusion::Sm90Compute<
273 + cutlass::minus, float, int32_t,
274 + cutlass::FloatRoundStyle::round_to_nearest>;
275 +
276 + using EVTComputeAcc =
277 + cutlass::epilogue::fusion::Sm90EVT<ComputeAcc, Accum, EVTComputeAzp>;
278 +
279 + using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
280 + cutlass::multiplies, float, float,
281 + cutlass::FloatRoundStyle::round_to_nearest>;
282 +
283 + using EVTComputeScaleB =
284 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAcc>;
285 +
286 + using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
287 + cutlass::multiply_add, ElementD, float,
288 + cutlass::FloatRoundStyle::round_to_nearest>;
289 +
290 + public:
291 + using EVTCompute =
292 + cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
293 + EVTComputeScaleB, Bias>;
294 + using ArgumentType = typename EVTCompute::Arguments;
295 +
296 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
297 + torch::Tensor const& b_scales,
298 + torch::Tensor const& azp_adj,
299 + torch::Tensor const& azp,
300 + c10::optional<torch::Tensor> const& bias) {
301 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
302 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
303 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
304 + auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
305 + auto azp_adj_args =
306 + SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
307 +
308 + typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
309 + typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
310 + typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
311 + return ArgumentType{a_args, evt_scale_b_args, bias_args};
312 + }
313 +};
314 +
315 +}; // namespace vllm::c3x
\ No newline at end of file
csrc/cutlass_extensions/vllm_cutlass_library_extension.py
+29
@@ -35,6 +35,35 @@ VLLMDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
35 }
36 }
37
38 +VLLMDataTypeSize: Dict[Union[VLLMDataType, DataType], int] = {
39 + **DataTypeSize, # type: ignore
40 + **{
41 + VLLMDataType.u4b8: 4,
42 + VLLMDataType.u8b128: 8,
43 + }
44 +}
45 +
46 +VLLMDataTypeVLLMScalarTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
47 + VLLMDataType.u4b8: "vllm::kU4B8",
48 + VLLMDataType.u8b128: "vllm::kU8B128",
49 + DataType.u4: "vllm::kU4",
50 + DataType.u8: "vllm::kU8",
51 + DataType.s4: "vllm::kS4",
52 + DataType.s8: "vllm::kS8",
53 + DataType.f16: "vllm::kFloat16",
54 + DataType.bf16: "vllm::kBfloat16",
55 +}
56 +
57 +VLLMDataTypeTorchDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
58 + DataType.u8: "at::ScalarType::Byte",
59 + DataType.s8: "at::ScalarType::Char",
60 + DataType.e4m3: "at::ScalarType::Float8_e4m3fn",
61 + DataType.s32: "at::ScalarType::Int",
62 + DataType.f16: "at::ScalarType::Half",
63 + DataType.bf16: "at::ScalarType::BFloat16",
64 + DataType.f32: "at::ScalarType::Float",
65 +}
66 +
67 VLLMKernelScheduleTag: Dict[Union[
68 MixedInputKernelScheduleType, KernelScheduleType], str] = {
69 **KernelScheduleTag, # type: ignore
csrc/cutlass_extensions/vllm_numeric_conversion.cuh
+218 -21
@@ -3,6 +3,7 @@
3 #include "cutlass/numeric_conversion.h"
4 #include "cutlass_extensions/vllm_custom_types.cuh"
5 #include "cutlass_extensions/cute_utils.cuh"
6 +#include "cutlass_extensions/vllm_type_utils.cuh"
7
8 // this file extends:
9 // https://github.com/NVIDIA/cutlass/blob/cutlass-3.5.0/include/cutlass/numeric_conversion.h
@@ -28,8 +29,19 @@ struct InterleavedNumericArrayConverter {
29
30 CUTLASS_DEVICE
31 static result_type convert(source_type const& source) {
31 - CUTE_INVALID_CONTROL_PATH(
32 - "InterleavedNumericArrayConverter not implemented\n");
32 + if (cute::elect_one_sync()) {
33 + if constexpr (std::is_same_v<IlvBlkLayout, void>) {
34 + printf(
35 + "Convert %s <= %s (N = %d, IlvBlkLayout = void), not implemented\n",
36 + nameof_v<T>, nameof_v<S>, N);
37 + } else {
38 + printf(
39 + "Convert %s <= %s (N = %d, size(IlvBlkLayout{}) = %d), not "
40 + "implemented\n",
41 + nameof_v<T>, nameof_v<S>, N, size(IlvBlkLayout{}));
42 + }
43 + __brkpt();
44 + }
45 return {};
46 }
47
@@ -56,11 +68,6 @@ struct InterleavedNumericArrayConverter<
68 result_type operator()(source_type const& s) const { return convert(s); }
69 };
70
59 -// TODO (LucasWilkinson): Implement
60 -// for Array<cutlass::float8_e4m3fn, N> <= Array<vllm_uint4b8_t, N>
61 -
62 -// ....
63 -
71 template <typename RegConvert32bit, typename T, typename S, int N>
72 struct ArrayConverterPacked32Bit {
73 using result_type = Array<T, N>;
@@ -86,14 +93,16 @@ struct ArrayConverterPacked32Bit {
93 using ScalarConverter = NumericConverter<T, S>;
94
95 template <typename PackedSrc>
89 - CUTLASS_DEVICE static uint32_t to_reg(PackedSrc const& source) {
96 + CUTLASS_DEVICE static auto to_regs(PackedSrc const& src) {
97 if constexpr (sizeof(PackedSrc) == 1) {
91 - return static_cast<uint32_t>(reinterpret_cast<const uint8_t&>(source));
98 + return Array<uint32_t, 1>{reinterpret_cast<uint8_t const&>(src)};
99 } else if constexpr (sizeof(PackedSrc) == 2) {
93 - return static_cast<uint32_t>(reinterpret_cast<const uint16_t&>(source));
100 + return Array<uint32_t, 1>{reinterpret_cast<uint16_t const&>(src)};
101 + } else if constexpr (sizeof(PackedSrc) == 4) {
102 + return Array<uint32_t, 1>{reinterpret_cast<uint32_t const&>(src)};
103 } else {
95 - static_assert(sizeof(PackedSrc) == 4);
96 - return reinterpret_cast<const uint32_t&>(source);
104 + static_assert(sizeof(PackedSrc) == 8);
105 + return reinterpret_cast<Array<uint32_t, 2> const&>(src);
106 }
107 }
108
@@ -110,7 +119,7 @@ struct ArrayConverterPacked32Bit {
119 static_assert(std::is_same_v<typename PackedSrcType::Element, S>);
120 static_assert(std::is_same_v<typename PackedResultType::Element, T>);
121
113 - return RegConvert32bit::template convert<PackedResultType>(to_reg(source));
122 + return RegConvert32bit::template convert<PackedResultType>(to_regs(source));
123 }
124
125 friend class detail::VectorizedConverter;
@@ -140,6 +149,131 @@ struct ArrayConverterPacked32Bit {
149 }
150 };
151
152 +// Convert 8 4bit values packed into a 32bit register to 8 8bit values packed
153 +// into 2 32bit register.
154 +template <uint8_t LUT0, uint8_t LUT1, uint8_t LUT2, uint8_t LUT3, //
155 + uint8_t LUT4, uint8_t LUT5, uint8_t LUT6, uint8_t LUT7, //
156 + uint8_t LUT8, uint8_t LUT9, uint8_t LUT10, uint8_t LUT11, //
157 + uint8_t LUT12, uint8_t LUT13, uint8_t LUT14, uint8_t LUT15>
158 +CUTLASS_DEVICE cutlass::AlignedArray<uint32_t, 2> lut_4bit_to_8bit_convert(
159 + uint32_t src) {
160 + cutlass::AlignedArray<uint32_t, 2> r;
161 + // Determines if the value is in the top half of the LUT if set or
162 + // (i.e. LUT[8:15]) in the bottom half (i.e. LUT[0:7]) if not set. Then move
163 + // into bit position 0x4 of each nibble so when or'd with final_prmt_base it
164 + // selects the correct candidate. When elements in final_prmt_base
165 + // are >= 0x4, the high candidate is selected (i.e. LUT[8:15]), when elements
166 + // are < 0x4, the low candidate is selected (i.e. LUT[0:7])
167 + uint32_t high_bit = (src & 0x88888888) >> 1;
168 +
169 + // `high_bit` is OR'd with 0x31203120 to find the correct value in the LUT
170 + // (selects correct high or low candidate)
171 + const uint32_t final_prmt_base = 0x32103210;
172 +
173 + // Ignore the high bit when indexing into LUT, for each 4bit value
174 + // we index into both the high and low candidates then use
175 + // high_bit | final_prmt_base to select the correct candidate
176 + uint32_t lut_idx = (src & 0x77777777);
177 +
178 + auto pack = [](uint8_t a, uint8_t b, uint8_t c, uint8_t d) {
179 + return uint32_t(a) | (uint32_t(b) << 8) | (uint32_t(c) << 16) |
180 + (uint32_t(d) << 24);
181 + };
182 +
183 + static constexpr uint32_t LOW_0 = pack(LUT0, LUT1, LUT2, LUT3);
184 + static constexpr uint32_t LOW_1 = pack(LUT4, LUT5, LUT6, LUT7);
185 + static constexpr uint32_t HIGH_0 = pack(LUT8, LUT9, LUT10, LUT11);
186 + static constexpr uint32_t HIGH_1 = pack(LUT12, LUT13, LUT14, LUT15);
187 +
188 + CUTLASS_PRAGMA_UNROLL
189 + for (int ii = 0; ii < 2; ++ii, lut_idx >>= 16, high_bit >>= 16) {
190 + uint32_t final_prmt_idx = final_prmt_base | high_bit;
191 +
192 + // This uses a look up table to convert packed int4s to packed int8s,
193 + // using the int4 value as the index to prmt. It first select both the
194 + // high and low candidates, then uses the high bit (i.e. `high_bit`) to
195 + // select the correct candidate.
196 + asm volatile(
197 + "{\n"
198 + " .reg .b32 low, high;\n"
199 + " prmt.b32 low, %1, %2, %5;\n"
200 + " prmt.b32 high, %3, %4, %5;\n"
201 + " prmt.b32 %0, low, high, %6;\n"
202 + "}\n"
203 + : "=r"(r[ii])
204 + : "n"(LOW_0), "n"(LOW_1), "n"(HIGH_0), "n"(HIGH_1), "r"(lut_idx),
205 + "r"(final_prmt_idx));
206 + }
207 +
208 + return r;
209 +};
210 +
211 +// for Array<int8_t, N> <= Array<vllm_uint4b8_t, N>
212 +template <FloatRoundStyle Round, int N>
213 +struct NumericArrayConverter<int8_t, vllm_uint4b8_t, N, Round> {
214 + using result_type = Array<int8_t, N>;
215 + using source_type = Array<vllm_uint4b8_t, N>;
216 +
217 + static FloatRoundStyle const round_style = Round;
218 +
219 + private:
220 + struct RegConvert {
221 + template <typename PackedResultType>
222 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
223 + // [-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7] as int8s
224 + auto r = lut_4bit_to_8bit_convert<0xF8, 0xF9, 0xFA, 0xFB, //
225 + 0xFC, 0xFD, 0xFE, 0xFF, //
226 + 0x00, 0x01, 0x02, 0x03, //
227 + 0x04, 0x05, 0x06, 0x07>(src_[0]);
228 + return reinterpret_cast<PackedResultType&>(r);
229 + };
230 + };
231 +
232 + public:
233 + CUTLASS_DEVICE
234 + static result_type convert(source_type const& source) {
235 + return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
236 + typename source_type::Element,
237 + N>::convert(source);
238 + }
239 +
240 + CUTLASS_DEVICE
241 + result_type operator()(source_type const& s) const { return convert(s); }
242 +};
243 +
244 +// for Array<cutlass::float_e4m3_t, N> <= Array<vllm_uint4b8_t, N>
245 +template <FloatRoundStyle Round, int N>
246 +struct NumericArrayConverter<cutlass::float_e4m3_t, vllm_uint4b8_t, N, Round> {
247 + using result_type = Array<cutlass::float_e4m3_t, N>;
248 + using source_type = Array<vllm_uint4b8_t, N>;
249 +
250 + static FloatRoundStyle const round_style = Round;
251 +
252 + private:
253 + struct RegConvert {
254 + template <typename PackedResultType>
255 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
256 + // [-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7] as fp8s
257 + auto r = lut_4bit_to_8bit_convert<0xD0, 0xCE, 0xCC, 0xCA, //
258 + 0xC8, 0xC4, 0xC0, 0xB8, //
259 + 0x00, 0x38, 0x40, 0x44, //
260 + 0x48, 0x4A, 0x4C, 0x4E>(src_[0]);
261 + return reinterpret_cast<PackedResultType&>(r);
262 + };
263 + };
264 +
265 + public:
266 + CUTLASS_DEVICE
267 + static result_type convert(source_type const& source) {
268 + return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
269 + typename source_type::Element,
270 + N>::convert(source);
271 + }
272 +
273 + CUTLASS_DEVICE
274 + result_type operator()(source_type const& s) const { return convert(s); }
275 +};
276 +
277 // for Array<cutlass::half_t, N> <= Array<vllm_uint4b8_t, N>
278 template <FloatRoundStyle Round, int N>
279 struct NumericArrayConverter<cutlass::half_t, vllm_uint4b8_t, N, Round> {
@@ -148,7 +282,8 @@ struct NumericArrayConverter<cutlass::half_t, vllm_uint4b8_t, N, Round> {
282
283 struct RegConvert {
284 template <typename PackedResultType>
151 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
285 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
286 + uint32_t src = src_[0];
287 using RegArray =
288 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
289 sizeof(PackedResultType)>;
@@ -249,7 +384,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
384 private:
385 struct RegConvert {
386 template <typename PackedResultType>
252 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
387 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
388 + uint32_t src = src_[0];
389 using RegArray =
390 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
391 sizeof(PackedResultType)>;
@@ -338,7 +474,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
474 private:
475 struct RegConvert {
476 template <typename PackedResultType>
341 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
477 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
478 + uint32_t src = src_[0];
479 using RegArray =
480 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
481 sizeof(PackedResultType)>;
@@ -417,7 +554,8 @@ struct NumericArrayConverter<cutlass::half_t, vllm_uint8b128_t, N, Round> {
554
555 struct RegConvert {
556 template <typename PackedResultType>
420 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
557 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
558 + uint32_t src = src_[0];
559 // Hold output FP16s in reg. We need 1 reg for every 2 elements
560 using RegArray =
561 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
@@ -469,7 +607,8 @@ struct NumericArrayConverter<float, vllm_uint8b128_t, N, Round> {
607 private:
608 struct RegConvert {
609 template <typename PackedResultType>
472 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
610 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
611 + uint32_t src = src_[0];
612 PackedResultType r;
613
614 // __byte_perm simulates the add.u32 0x4B000000 to every u8 element of
@@ -513,7 +652,8 @@ struct NumericArrayConverter<cutlass::bfloat16_t, vllm_uint4b8_t, N, Round> {
652 private:
653 struct RegConvert {
654 template <typename PackedResultType>
516 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src_reg) {
655 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
656 + uint32_t src_reg = src_[0];
657 // Hold output BF16s in reg. We need 1 reg for every 2 elements
658 using RegArray =
659 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
@@ -603,7 +743,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
743 private:
744 struct RegConvert {
745 template <typename PackedResultType>
606 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
746 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
747 + uint32_t src = src_[0];
748 using RegArray =
749 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
750 sizeof(PackedResultType)>;
@@ -671,7 +812,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
812 private:
813 struct RegConvert {
814 template <typename PackedResultType>
674 - CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
815 + CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
816 + uint32_t src = src_[0];
817 using RegArray =
818 cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
819 sizeof(PackedResultType)>;
@@ -788,6 +930,61 @@ struct NumericArrayConverter<cutlass::bfloat16_t, vllm_uint8b128_t, N, Round> {
930
931 #endif
932
933 +// for Array<int8_t, N> <= Array<cutlass::half_t, N>
934 +// FastFP16toINT8 from https://arxiv.org/pdf/2406.09904
935 +template <FloatRoundStyle Round, int N>
936 +struct NumericArrayConverter<int8_t, cutlass::half_t, N, Round> {
937 + using result_type = Array<int8_t, N>;
938 + using source_type = Array<cutlass::half_t, N>;
939 +
940 + struct RegConvert {
941 + // FastFP16toINT8 from https://arxiv.org/pdf/2406.09904
942 + template <typename PackedResultType, int src_regs>
943 + CUTLASS_DEVICE static PackedResultType convert(
944 + Array<uint32_t, src_regs> src) {
945 + // Hold output int8s in reg. We need 1 reg for every 4 elements
946 + using RegArray = cutlass::AlignedArray<
947 + uint32_t, std::max(PackedResultType::kElements / 4, size_t(1))>;
948 + RegArray r;
949 +
950 + static constexpr uint32_t MAGIC_BIAS_ = 0x64806480;
951 + auto MAGIC_BIAS = *reinterpret_cast<const half2*>(&MAGIC_BIAS_);
952 +
953 + *reinterpret_cast<half2*>(&src[0]) =
954 + __hadd2(*reinterpret_cast<half2*>(&src[0]), MAGIC_BIAS);
955 +
956 + if constexpr (src_regs > 1) {
957 + *reinterpret_cast<half2*>(&src[1]) =
958 + __hadd2(*reinterpret_cast<half2*>(&src[1]), MAGIC_BIAS);
959 + }
960 +
961 + static_assert(PackedResultType::kElements <= 4);
962 + uint32_t uint8s;
963 + static constexpr uint32_t MASK_0246 = 0x6420;
964 + static constexpr uint32_t UINT8s_TO_INT8s_MASK = 0x80808080;
965 + asm volatile("prmt.b32 %0,%1,%2,%3;\n"
966 + : "=r"(uint8s)
967 + : "r"(src[0]), "r"((src_regs > 1) ? src[1] : src[0]),
968 + "n"(MASK_0246));
969 +
970 + uint32_t int8s = (uint8s ^ UINT8s_TO_INT8s_MASK);
971 +
972 + return reinterpret_cast<PackedResultType&>(int8s);
973 + };
974 + };
975 +
976 + public:
977 + CUTLASS_DEVICE
978 + static result_type convert(source_type const& source) {
979 + return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
980 + typename source_type::Element,
981 + N>::convert(source);
982 + }
983 +
984 + CUTLASS_DEVICE
985 + result_type operator()(source_type const& s) const { return convert(s); }
986 +};
987 +
988 /////////////////////////////////////////////////////////////////////////////////////////////////
989
990 } // namespace cutlass
csrc/cutlass_extensions/vllm_type_utils.cuh new
+42
@@ -0,0 +1,42 @@
1 +#include "cutlass/bfloat16.h"
2 +#include "cutlass/half.h"
3 +#include "cuda_bf16.h"
4 +
5 +#include "cutlass_extensions/vllm_custom_types.cuh"
6 +
7 +namespace cutlass {
8 +
9 +template <typename T>
10 +struct nameof {
11 + static constexpr char const* value = "unknown";
12 +};
13 +
14 +template <typename T>
15 +inline constexpr auto nameof_v = nameof<T>::value;
16 +
17 +#define NAMEOF_TYPE(T) \
18 + template <> \
19 + struct nameof<T> { \
20 + static constexpr char const* value = #T; \
21 + };
22 +
23 +NAMEOF_TYPE(float_e4m3_t)
24 +NAMEOF_TYPE(float_e5m2_t)
25 +NAMEOF_TYPE(half_t)
26 +NAMEOF_TYPE(nv_bfloat16)
27 +NAMEOF_TYPE(bfloat16_t)
28 +NAMEOF_TYPE(float)
29 +
30 +NAMEOF_TYPE(int4b_t)
31 +NAMEOF_TYPE(int8_t)
32 +NAMEOF_TYPE(int32_t)
33 +NAMEOF_TYPE(int64_t)
34 +
35 +NAMEOF_TYPE(vllm_uint4b8_t)
36 +NAMEOF_TYPE(uint4b_t)
37 +NAMEOF_TYPE(uint8_t)
38 +NAMEOF_TYPE(vllm_uint8b128_t)
39 +NAMEOF_TYPE(uint32_t)
40 +NAMEOF_TYPE(uint64_t)
41 +
42 +}; // namespace cutlass
\ No newline at end of file
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+27 -26
@@ -8,6 +8,10 @@
8 #include "scaled_mm_c2x_sm89_fp8_dispatch.cuh"
9 #include "scaled_mm_c2x_sm89_int8_dispatch.cuh"
10
11 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp"
12 +
13 +using namespace vllm;
14 +
15 /*
16 This file defines quantized GEMM operations using the CUTLASS 2.x API, for
17 NVIDIA GPUs with SM versions prior to sm90 (Hopper).
@@ -22,12 +26,11 @@ void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
26 TORCH_CHECK(b.dtype() == torch::kInt8);
27
28 if (out.dtype() == torch::kBFloat16) {
25 - return vllm::cutlass_gemm_sm75_dispatch<int8_t, cutlass::bfloat16_t,
26 - Epilogue>(
29 + return cutlass_gemm_sm75_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
30 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
31 } else {
32 TORCH_CHECK(out.dtype() == torch::kFloat16);
30 - return vllm::cutlass_gemm_sm75_dispatch<int8_t, cutlass::half_t, Epilogue>(
33 + return cutlass_gemm_sm75_dispatch<int8_t, cutlass::half_t, Epilogue>(
34 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
35 }
36 }
@@ -42,10 +45,10 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
45 if (bias) {
46 TORCH_CHECK(bias->dtype() == out.dtype(),
47 "currently bias dtype must match output dtype ", out.dtype());
45 - return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBias>(
48 + return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogueBias>(
49 out, a, b, a_scales, b_scales, *bias);
50 } else {
48 - return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogue>(
51 + return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogue>(
52 out, a, b, a_scales, b_scales);
53 }
54 }
@@ -61,10 +64,10 @@ void cutlass_scaled_mm_azp_sm75(torch::Tensor& out, torch::Tensor const& a,
64 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
65
66 if (azp) {
64 - return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
67 + return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogueBiasAzpToken>(
68 out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
69 } else {
67 - return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzp>(
70 + return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogueBiasAzp>(
71 out, a, b, a_scales, b_scales, azp_adj, bias);
72 }
73 }
@@ -78,12 +81,11 @@ void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
81 TORCH_CHECK(b.dtype() == torch::kInt8);
82
83 if (out.dtype() == torch::kBFloat16) {
81 - return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
82 - Epilogue>(
84 + return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
85 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
86 } else {
87 TORCH_CHECK(out.dtype() == torch::kFloat16);
86 - return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
88 + return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
89 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
90 }
91 }
@@ -98,10 +100,10 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
100 if (bias) {
101 TORCH_CHECK(bias->dtype() == out.dtype(),
102 "currently bias dtype must match output dtype ", out.dtype());
101 - return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBias>(
103 + return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogueBias>(
104 out, a, b, a_scales, b_scales, *bias);
105 } else {
104 - return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogue>(
106 + return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogue>(
107 out, a, b, a_scales, b_scales);
108 }
109 }
@@ -117,10 +119,10 @@ void cutlass_scaled_mm_azp_sm80(torch::Tensor& out, torch::Tensor const& a,
119 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
120
121 if (azp) {
120 - return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
122 + return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogueBiasAzpToken>(
123 out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
124 } else {
123 - return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzp>(
125 + return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogueBiasAzp>(
126 out, a, b, a_scales, b_scales, azp_adj, bias);
127 }
128 }
@@ -134,13 +136,12 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
136 TORCH_CHECK(b.dtype() == torch::kInt8);
137
138 if (out.dtype() == torch::kBFloat16) {
137 - return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::bfloat16_t,
138 - Epilogue>(
139 + return cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::bfloat16_t,
140 + Epilogue>(
141 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
142 } else {
143 assert(out.dtype() == torch::kFloat16);
142 - return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::half_t,
143 - Epilogue>(
144 + return cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
145 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
146 }
147 } else {
@@ -148,13 +149,13 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
149 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
150
151 if (out.dtype() == torch::kBFloat16) {
151 - return vllm::cutlass_gemm_sm89_fp8_dispatch<
152 - cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue>(
152 + return cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
153 + cutlass::bfloat16_t, Epilogue>(
154 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
155 } else {
156 TORCH_CHECK(out.dtype() == torch::kFloat16);
156 - return vllm::cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
157 - cutlass::half_t, Epilogue>(
157 + return cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
158 + cutlass::half_t, Epilogue>(
159 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
160 }
161 }
@@ -170,10 +171,10 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
171 if (bias) {
172 TORCH_CHECK(bias->dtype() == out.dtype(),
173 "currently bias dtype must match output dtype ", out.dtype());
173 - return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBias>(
174 + return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogueBias>(
175 out, a, b, a_scales, b_scales, *bias);
176 } else {
176 - return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogue>(
177 + return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogue>(
178 out, a, b, a_scales, b_scales);
179 }
180 }
@@ -189,10 +190,10 @@ void cutlass_scaled_mm_azp_sm89(torch::Tensor& out, torch::Tensor const& a,
190 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
191
192 if (azp) {
192 - return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
193 + return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogueBiasAzpToken>(
194 out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
195 } else {
195 - return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzp>(
196 + return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogueBiasAzp>(
197 out, a, b, a_scales, b_scales, azp_adj, bias);
198 }
199 }
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
-302
@@ -21,7 +21,6 @@
21 #include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
22 #include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
23
24 -#include "broadcast_load_epilogue_c2x.hpp"
24 #include "common.hpp"
25 // clang-format on
26
@@ -71,307 +70,6 @@ struct enable_sm89_to_sm90 : Kernel {
70 #endif
71 }
72 };
74 -
75 -/*
76 - * This class provides the common load descriptors for the
77 - * ScaledEpilogue[...] classes
78 - */
79 -template <typename ElementD, typename OutputTileThreadMap>
80 -struct ScaledEpilogueBase {
81 - protected:
82 - using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
83 -
84 - template <typename T>
85 - using ColOrScalarLoad =
86 - cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
87 - OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
88 -
89 - template <typename T>
90 - using RowOrScalarLoad =
91 - cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92 - OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
93 -
94 - template <typename T>
95 - using ColLoad = cutlass::epilogue::threadblock::VisitorColBroadcast<
96 - OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
97 -
98 - template <typename T>
99 - using RowLoad = cutlass::epilogue::threadblock::VisitorRowBroadcast<
100 - OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
101 -
102 - template <typename T>
103 - using RowOrZeroLoad =
104 - cutlass::epilogue::threadblock::VisitorRowOrZeroBroadcast<
105 - OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
106 -
107 - // This utility function constructs the arguments for the load descriptors
108 - // from a tensor. It can handle both row and column, as well as row/column or
109 - // scalar cases.
110 - template <typename Descriptor, typename T>
111 - static auto args_from_tensor(torch::Tensor const& tensor) {
112 - using Arguments = typename Descriptor::Arguments;
113 - auto* data_ptr = static_cast<T*>(tensor.data_ptr());
114 - if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
115 - std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
116 - return Arguments{data_ptr, tensor.numel() != 1};
117 - } else {
118 - // it would technically work but no use case as data_ptr is never nullptr
119 - static_assert(!std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
120 - return Arguments{data_ptr};
121 - }
122 - }
123 -
124 - // This overload handles the case where there might not be a tensor, in which
125 - // case a nullptr is passed and a constant (0) is used.
126 - template <typename Descriptor, typename T>
127 - static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
128 - static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
129 - using Arguments = typename Descriptor::Arguments;
130 - auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
131 - return Arguments{data_ptr};
132 - }
133 -};
134 -
135 -/*
136 - This epilogue function defines a quantized GEMM operation similar to
137 - torch._scaled_mm.
138 -
139 - A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
140 - per-row. B can be quantized per-tensor or per-column.
141 - Any combination of per-tensor and per-row or column is supported.
142 - A and B must have symmetric quantization (zero point == 0).
143 -
144 - So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
145 - scales are applied elementwise with numpy-style broadcasting.
146 -
147 - ScaleA and ScaleB define the epilogue functions that apply the scales for
148 - the A and B operands respectively. These scales may be either per-tensor or
149 - per row or column.
150 -*/
151 -template <typename ElementD, typename OutputTileThreadMap>
152 -struct ScaledEpilogue
153 - : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
154 - private:
155 - using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156 - using Accum = typename SUPER::Accum;
157 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
158 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
159 -
160 - using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161 - cutlass::multiplies, float, float,
162 - cutlass::FloatRoundStyle::round_to_nearest>;
163 -
164 - using EVTCompute0 =
165 - cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
166 -
167 - using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
168 - cutlass::multiplies, ElementD, float,
169 - cutlass::FloatRoundStyle::round_to_nearest>;
170 -
171 - public:
172 - using EVTCompute =
173 - cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
174 - using ArgumentType = typename EVTCompute::Arguments;
175 -
176 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
177 - torch::Tensor const& b_scales) {
178 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
179 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
180 -
181 - typename EVTCompute0::Arguments evt0_args{b_args};
182 - return ArgumentType{a_args, evt0_args};
183 - }
184 -};
185 -
186 -/*
187 - * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
188 - * This bias can also be used in the per-tensor azp case, where the activation
189 - * zero point (azp) is used to compute an azp correction term,
190 - * which is folded into the bias.
191 - *
192 - * The bias tensor must be per-output channel.
193 - * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
194 - */
195 -template <typename ElementD, typename OutputTileThreadMap>
196 -struct ScaledEpilogueBias
197 - : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
198 - protected:
199 - using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
200 - using Accum = typename SUPER::Accum;
201 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
202 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
203 - using Bias = typename SUPER::template RowLoad<ElementD>;
204 - using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
205 - cutlass::multiplies, float, float,
206 - cutlass::FloatRoundStyle::round_to_nearest>;
207 -
208 - using EVTCompute0 =
209 - cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
210 -
211 - using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
212 - cutlass::multiply_add, ElementD, float,
213 - cutlass::FloatRoundStyle::round_to_nearest>;
214 -
215 - public:
216 - using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
217 - EVTCompute0, Bias>;
218 - using ArgumentType = typename EVTCompute::Arguments;
219 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
220 - torch::Tensor const& b_scales,
221 - torch::Tensor const& bias) {
222 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
223 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
224 - auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
225 -
226 - typename EVTCompute0::Arguments evt0_args{b_args};
227 - return ArgumentType{a_args, evt0_args, bias_args};
228 - }
229 -};
230 -
231 -/*
232 - * This epilogue directly supports per-tensor azp in int32 form.
233 - * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
234 - * term, which should already be multiplied with the scalar azp.
235 - * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
236 - *
237 - * This epilogue also supports bias, which remains per-channel.
238 - */
239 -template <typename ElementD, typename OutputTileThreadMap>
240 -struct ScaledEpilogueBiasAzp
241 - : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
242 - private:
243 - using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
244 - using Accum = typename SUPER::Accum;
245 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
246 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
247 - using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
248 -
249 - // This is the full AZP term, azp * J @ B, shape (1,n)
250 - using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
251 -
252 - // Compute float(accum - azp_adj), both operands are int32_t
253 - using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
254 - cutlass::minus, float, int32_t,
255 - cutlass::FloatRoundStyle::round_to_nearest>;
256 -
257 - using EVTComputeAzp =
258 - cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Accum, AzpWithAdj>;
259 -
260 - using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
261 - cutlass::multiplies, float, float,
262 - cutlass::FloatRoundStyle::round_to_nearest>;
263 -
264 - using EVTComputeScaleB =
265 - cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
266 - EVTComputeAzp>;
267 -
268 - using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
269 - cutlass::multiply_add, ElementD, float,
270 - cutlass::FloatRoundStyle::round_to_nearest>;
271 -
272 - public:
273 - using EVTCompute =
274 - cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
275 - EVTComputeScaleB, Bias>;
276 -
277 - using ArgumentType = typename EVTCompute::Arguments;
278 -
279 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
280 - torch::Tensor const& b_scales,
281 - torch::Tensor const& azp_adj,
282 - c10::optional<torch::Tensor> const& bias) {
283 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
284 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
285 - auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
286 - auto azp_adj_args =
287 - SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
288 -
289 - typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
290 - typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
291 - return ArgumentType{a_args, evt_scale_b_args, bias_args};
292 - }
293 -};
294 -
295 -/*
296 - * This epilogue supports per-token azp by computing and applying
297 - * the correction term using a rank-1 update. If the term were materialized,
298 - * it would require O(m*n) space, and this way it only requires O(m+n) space.
299 - * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
300 - * point for each row of A.
301 - * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
302 - *
303 - * This epilogue also supports bias, which remains per-channel.
304 - */
305 -template <typename ElementD, typename OutputTileThreadMap>
306 -struct ScaledEpilogueBiasAzpToken
307 - : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
308 - private:
309 - using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
310 - using Accum = typename SUPER::Accum;
311 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
312 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
313 - using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
314 -
315 - // Per-token azp term, shape (m,1)
316 - using Azp = typename SUPER::template ColLoad<int32_t>;
317 -
318 - // This is the AZP adjustment term, J @ B, shape (1,n)
319 - using AzpAdj = typename SUPER::template RowLoad<int32_t>;
320 -
321 - // Compute azp * azp_adj
322 - using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
323 - cutlass::multiplies, int32_t, int32_t,
324 - cutlass::FloatRoundStyle::round_to_nearest>;
325 -
326 - using EVTComputeAzp =
327 - cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Azp, AzpAdj>;
328 -
329 - // Compute float(accum - azp*azp_adj), all operands are int32_t
330 - using ComputeAcc = cutlass::epilogue::threadblock::VisitorCompute<
331 - cutlass::minus, float, int32_t,
332 - cutlass::FloatRoundStyle::round_to_nearest>;
333 -
334 - using EVTComputeAcc =
335 - cutlass::epilogue::threadblock::Sm80EVT<ComputeAcc, Accum, EVTComputeAzp>;
336 -
337 - using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
338 - cutlass::multiplies, float, float,
339 - cutlass::FloatRoundStyle::round_to_nearest>;
340 -
341 - using EVTComputeScaleB =
342 - cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
343 - EVTComputeAcc>;
344 -
345 - using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
346 - cutlass::multiply_add, ElementD, float,
347 - cutlass::FloatRoundStyle::round_to_nearest>;
348 -
349 - public:
350 - using EVTCompute =
351 - cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
352 - EVTComputeScaleB, Bias>;
353 -
354 - using ArgumentType = typename EVTCompute::Arguments;
355 -
356 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
357 - torch::Tensor const& b_scales,
358 - torch::Tensor const& azp_adj,
359 - torch::Tensor const& azp,
360 - c10::optional<torch::Tensor> const& bias) {
361 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
362 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
363 - auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
364 - auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
365 - auto azp_adj_args =
366 - SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
367 -
368 - typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
369 - typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
370 - typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
371 - return ArgumentType{a_args, evt_scale_b_args, bias_args};
372 - }
373 -};
374 -
73 template <typename Arch, template <typename> typename ArchGuard,
74 typename ElementAB_, typename ElementD_,
75 template <typename, typename> typename Epilogue_, typename TileShape,
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+7 -305
@@ -23,11 +23,12 @@
23 #include "cutlass/epilogue/collective/collective_builder.hpp"
24 #include "cutlass/gemm/collective/collective_builder.hpp"
25
26 -#include "broadcast_load_epilogue_c3x.hpp"
26 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
27 #include "common.hpp"
28 // clang-format on
29
30 using namespace cute;
31 +using namespace vllm;
32
33 /*
34 This file defines quantized GEMM operations using the CUTLASS 3.x API, for
@@ -56,305 +57,6 @@ struct enable_sm90_or_later : Kernel {
57 #endif
58 }
59 };
59 -
60 -/*
61 - * This class provides the common load descriptors for the
62 - * ScaledEpilogue[...] classes
63 - */
64 -template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
65 -struct ScaledEpilogueBase {
66 - protected:
67 - using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
68 -
69 - template <typename T>
70 - using ColOrScalarLoad = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
71 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
72 - Stride<Int<1>, Int<0>, Int<0>>>;
73 -
74 - template <typename T>
75 - using RowOrScalarLoad = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
76 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
77 - Stride<Int<0>, Int<1>, Int<0>>>;
78 -
79 - // Don't want to support nullptr by default
80 - template <typename T, bool EnableNullPtr = false>
81 - using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
82 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
83 - Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
84 -
85 - // Don't want to support nullptr by default
86 - template <typename T, bool EnableNullPtr = false>
87 - using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
88 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
89 - Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
90 -
91 - // This utility function constructs the arguments for the load descriptors
92 - // from a tensor. It can handle both row and column, as well as row/column or
93 - // scalar cases.
94 - template <typename Descriptor, typename T>
95 - static auto args_from_tensor(torch::Tensor const& tensor) {
96 - using Arguments = typename Descriptor::Arguments;
97 - auto* data_ptr = static_cast<T*>(tensor.data_ptr());
98 - if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
99 - std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
100 - return Arguments{data_ptr, tensor.numel() != 1};
101 - } else {
102 - static_assert(!std::is_same_v<Descriptor, ColLoad<T, true>> &&
103 - !std::is_same_v<Descriptor, RowLoad<T, true>>);
104 - return Arguments{data_ptr};
105 - }
106 - }
107 -
108 - // This overload handles the case where there might not be a tensor, in which
109 - // case a nullptr is passed and a constant (0) is used.
110 - template <typename Descriptor, typename T>
111 - static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
112 - using Arguments = typename Descriptor::Arguments;
113 - auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
114 - static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
115 - std::is_same_v<Descriptor, RowLoad<T, true>>);
116 - return Arguments{data_ptr};
117 - }
118 -};
119 -
120 -/*
121 - This epilogue function defines a quantized GEMM operation similar to
122 - torch.scaled_mm_.
123 -
124 - A and B may be both either int8 or fp8_e4m3. A can be
125 - quantized per-tensor or per-row. B can be quantized per-tensor or per-column.
126 - Any combination of per-tensor and per-row or column is supported.
127 - A and B must have symmetric quantization (zero point == 0).
128 -
129 - So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
130 - scales are applied elementwise with numpy-style broadcasting.
131 -
132 - ScaleA and ScaleB define the epilogue functions that apply the scales for
133 - the A and B operands respectively. These scales may be either per-tensor or
134 - per row or column.
135 -*/
136 -template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
137 -struct ScaledEpilogue
138 - : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
139 - private:
140 - using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
141 - using Accum = typename SUPER::Accum;
142 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
143 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
144 -
145 - using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
146 - cutlass::multiplies, float, float,
147 - cutlass::FloatRoundStyle::round_to_nearest>;
148 -
149 - using EVTCompute0 =
150 - cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
151 -
152 - using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
153 - cutlass::multiplies, ElementD, float,
154 - cutlass::FloatRoundStyle::round_to_nearest>;
155 -
156 - public:
157 - using EVTCompute =
158 - cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
159 - using ArgumentType = typename EVTCompute::Arguments;
160 -
161 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
162 - torch::Tensor const& b_scales) {
163 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
164 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
165 -
166 - typename EVTCompute0::Arguments evt0_args{b_args};
167 - return ArgumentType{a_args, evt0_args};
168 - }
169 -};
170 -
171 -/*
172 - * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
173 - * This bias can also be used in the per-tensor azp case, where the activation
174 - * zero point (azp) is used to compute an azp correction term,
175 - * which is folded into the bias.
176 - *
177 - * The bias tensor must be per-output channel.
178 - * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
179 - */
180 -template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
181 -struct ScaledEpilogueBias
182 - : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
183 - private:
184 - using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
185 - using Accum = typename SUPER::Accum;
186 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
187 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
188 - using Bias = typename SUPER::template RowLoad<ElementD>;
189 -
190 - using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
191 - cutlass::multiplies, float, float,
192 - cutlass::FloatRoundStyle::round_to_nearest>;
193 -
194 - using EVTCompute0 =
195 - cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
196 -
197 - using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
198 - cutlass::multiply_add, ElementD, float,
199 - cutlass::FloatRoundStyle::round_to_nearest>;
200 -
201 - public:
202 - using EVTCompute =
203 - cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
204 -
205 - using ArgumentType = typename EVTCompute::Arguments;
206 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
207 - torch::Tensor const& b_scales,
208 - torch::Tensor const& bias) {
209 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
210 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
211 - auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
212 -
213 - typename EVTCompute0::Arguments evt0_args{b_args};
214 - return ArgumentType{a_args, evt0_args, bias_args};
215 - }
216 -};
217 -
218 -/*
219 - * This epilogue directly supports per-tensor azp in int32 form.
220 - * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
221 - * term, which should already be multiplied with the scalar azp.
222 - * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
223 - *
224 - * This epilogue also supports bias, which remains per-channel.
225 - */
226 -template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
227 -struct ScaledEpilogueBiasAzp
228 - : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
229 - private:
230 - using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
231 - using Accum = typename SUPER::Accum;
232 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
233 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
234 - using Bias = typename SUPER::template RowLoad<ElementD, true>;
235 -
236 - // This is the full AZP term, azp * J @ B, shape (1,n)
237 - using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
238 -
239 - // Compute float(accum - azp_adj), both operands are int32_t
240 - using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
241 - cutlass::minus, float, int32_t,
242 - cutlass::FloatRoundStyle::round_to_nearest>;
243 -
244 - using EVTComputeAzp =
245 - cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Accum, AzpWithAdj>;
246 -
247 - using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
248 - cutlass::multiplies, float, float,
249 - cutlass::FloatRoundStyle::round_to_nearest>;
250 -
251 - using EVTComputeScaleB =
252 - cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAzp>;
253 -
254 - using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
255 - cutlass::multiply_add, ElementD, float,
256 - cutlass::FloatRoundStyle::round_to_nearest>;
257 -
258 - public:
259 - using EVTCompute =
260 - cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
261 - EVTComputeScaleB, Bias>;
262 - using ArgumentType = typename EVTCompute::Arguments;
263 -
264 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
265 - torch::Tensor const& b_scales,
266 - torch::Tensor const& azp_adj,
267 - c10::optional<torch::Tensor> const& bias) {
268 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
269 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
270 - auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
271 - auto azp_adj_args =
272 - SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
273 -
274 - typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
275 - typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
276 - return ArgumentType{a_args, evt_scale_b_args, bias_args};
277 - }
278 -};
279 -
280 -/*
281 - * This epilogue supports per-token azp by computing and applying
282 - * the correction term using a rank-1 update. If the term were materialized,
283 - * it would require O(m*n) space, and this way it only requires O(m+n) space.
284 - * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
285 - * point for each row of A.
286 - * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
287 - *
288 - * This epilogue also supports bias, which remains per-channel.
289 - */
290 -template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
291 -struct ScaledEpilogueBiasAzpToken
292 - : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
293 - private:
294 - using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
295 - using Accum = typename SUPER::Accum;
296 - using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
297 - using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
298 - using Bias = typename SUPER::template RowLoad<ElementD, true>;
299 -
300 - // Per-token azp term, shape (m,1)
301 - using Azp = typename SUPER::template ColLoad<int32_t>;
302 -
303 - // This is the AZP adjustment term, J @ B, shape (1,n)
304 - using AzpAdj = typename SUPER::template RowLoad<int32_t>;
305 -
306 - // Compute azp * azp_adj
307 - using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
308 - cutlass::multiplies, int32_t, int32_t,
309 - cutlass::FloatRoundStyle::round_to_nearest>;
310 -
311 - using EVTComputeAzp =
312 - cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Azp, AzpAdj>;
313 -
314 - // Compute float(accum - azp*azp_adj), all operands are int32_t
315 - using ComputeAcc = cutlass::epilogue::fusion::Sm90Compute<
316 - cutlass::minus, float, int32_t,
317 - cutlass::FloatRoundStyle::round_to_nearest>;
318 -
319 - using EVTComputeAcc =
320 - cutlass::epilogue::fusion::Sm90EVT<ComputeAcc, Accum, EVTComputeAzp>;
321 -
322 - using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
323 - cutlass::multiplies, float, float,
324 - cutlass::FloatRoundStyle::round_to_nearest>;
325 -
326 - using EVTComputeScaleB =
327 - cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAcc>;
328 -
329 - using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
330 - cutlass::multiply_add, ElementD, float,
331 - cutlass::FloatRoundStyle::round_to_nearest>;
332 -
333 - public:
334 - using EVTCompute =
335 - cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
336 - EVTComputeScaleB, Bias>;
337 - using ArgumentType = typename EVTCompute::Arguments;
338 -
339 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
340 - torch::Tensor const& b_scales,
341 - torch::Tensor const& azp_adj,
342 - torch::Tensor const& azp,
343 - c10::optional<torch::Tensor> const& bias) {
344 - auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
345 - auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
346 - auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
347 - auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
348 - auto azp_adj_args =
349 - SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
350 -
351 - typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
352 - typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
353 - typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
354 - return ArgumentType{a_args, evt_scale_b_args, bias_args};
355 - }
356 -};
357 -
60 template <typename ElementAB_, typename ElementD_,
61 template <typename, typename, typename> typename Epilogue_,
62 typename TileShape, typename ClusterShape, typename KernelSchedule,
@@ -721,11 +423,11 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
423 if (bias) {
424 TORCH_CHECK(bias->dtype() == c.dtype(),
425 "currently bias dtype must match output dtype ", c.dtype());
724 - return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBias>(
426 + return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
427 c, a, b, a_scales, b_scales, *bias);
428 } else {
727 - return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogue>(c, a, b, a_scales,
728 - b_scales);
429 + return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogue>(
430 + c, a, b, a_scales, b_scales);
431 }
432 }
433
@@ -740,10 +442,10 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
442 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
443
444 if (azp) {
743 - return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzpToken>(
445 + return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzpToken>(
446 out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
447 } else {
746 - return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzp>(
448 + return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzp>(
449 out, a, b, a_scales, b_scales, azp_adj, bias);
450 }
451 }
csrc/quantization/machete/generate.py
+429 -303
@@ -3,8 +3,10 @@ import math
3 import os
4 import shutil
5 from collections.abc import Iterable
6 -from dataclasses import dataclass
7 -from typing import List, Optional, Tuple, Union
6 +from copy import deepcopy
7 +from dataclasses import dataclass, fields
8 +from functools import reduce
9 +from typing import Dict, List, Optional, Tuple, Union
10
11 import jinja2
12 # yapf conflicts with isort for this block
@@ -14,7 +16,10 @@ from vllm_cutlass_library_extension import (DataType, EpilogueScheduleTag,
16 MixedInputKernelScheduleType,
17 TileSchedulerTag,
18 TileSchedulerType, VLLMDataType,
17 - VLLMDataTypeNames, VLLMDataTypeTag,
19 + VLLMDataTypeNames,
20 + VLLMDataTypeSize, VLLMDataTypeTag,
21 + VLLMDataTypeTorchDataTypeTag,
22 + VLLMDataTypeVLLMScalarTypeTag,
23 VLLMKernelScheduleTag)
24
25 # yapf: enable
@@ -27,49 +32,125 @@ DISPATCH_TEMPLATE = """
32 #include "../machete_mm_launcher.cuh"
33
34 namespace machete {
30 -using GemmDispatcher_ = GemmDispatcher<
31 - {{DataTypeTag[type_config.element_a]}}, // ElementA
32 - {{DataTypeTag[type_config.element_b]}}, // ElementB
33 - {{DataTypeTag[type_config.element_d]}}, // ElementD
34 - {{DataTypeTag[type_config.accumulator]}}, // Accumulator
35 - {{DataTypeTag[type_config.element_b_scale]}}, // Scales
36 - {{DataTypeTag[type_config.element_b_zeropoint]}}>; // Zeropoints
37 -
38 -{% for s in schedules %}extern torch::Tensor
39 -impl_{{type_name}}_sch_{{ gen_sch_name(s) }}(PyTorchArguments args);
40 -{% endfor %}
41 -template <>
42 -torch::Tensor GemmDispatcher_::dispatch(PyTorchArguments args) {
35 +
36 +{% for impl_config in impl_configs %}
37 +{% set type_sig = gen_type_sig(impl_config.types) -%}
38 +{% for s in impl_config.schedules %}
39 +extern torch::Tensor impl_{{type_sig}}_sch_{{gen_sch_sig(s)}}(MMArgs);
40 +{%- endfor %}
41 +
42 +torch::Tensor mm_dispatch_{{type_sig}}(MMArgs args) {
43 [[maybe_unused]] auto M = args.A.size(0);
44 [[maybe_unused]] auto N = args.B.size(1);
45 [[maybe_unused]] auto K = args.A.size(1);
46
47 - if (!args.schedule) {
48 - {%- for cond, s in heuristic %}
47 + if (!args.maybe_schedule) {
48 + {%- for cond, s in impl_config.heuristic %}
49 {%if cond is not none%}if ({{cond}})
50 {%- else %}else
51 {%- endif %}
52 - return impl_{{ type_name }}_sch_{{ gen_sch_name(s) }}(args);{% endfor %}
52 + return impl_{{type_sig}}_sch_{{ gen_sch_sig(s) }}(args);{% endfor %}
53 }
54
55 - {% for s in schedules %}
56 - if (*args.schedule == "{{ gen_sch_name(s) }}") {
57 - return impl_{{ type_name }}_sch_{{ gen_sch_name(s) }}(args);
58 - }
59 - {% endfor %}
55 + {%- for s in impl_config.schedules %}
56 + if (*args.maybe_schedule == "{{ gen_sch_sig(s) }}")
57 + return impl_{{type_sig}}_sch_{{ gen_sch_sig(s) }}(args);
58 + {%- endfor %}
59 TORCH_CHECK_NOT_IMPLEMENTED(false, "machete_gemm(..) is not implemented for "
61 - "schedule = ", *args.schedule);
60 + "schedule = ", *args.maybe_schedule);
61 }
62 +{%- endfor %}
63 +
64
64 -template <>
65 -std::vector<std::string> GemmDispatcher_::supported_schedules() {
66 - return {
67 - {% for s in schedules -%}
68 - "{{ gen_sch_name(s) }}"{{ ",
69 - " if not loop.last }}{%- endfor %}
70 - };
65 +static inline std::optional<at::ScalarType> maybe_scalartype(
66 + c10::optional<at::Tensor> const& t) {
67 + if (!t) {
68 + return std::nullopt;
69 + } else {
70 + return t->scalar_type();
71 + };
72 +}
73 +
74 +torch::Tensor mm_dispatch(MMArgs args) {
75 + auto out_type = args.maybe_out_type.value_or(args.A.scalar_type());
76 + auto a_type = args.A.scalar_type();
77 + auto maybe_g_scales_type = maybe_scalartype(args.maybe_group_scales);
78 + auto maybe_g_zeros_type = maybe_scalartype(args.maybe_group_zeros);
79 + auto maybe_ch_scales_type = maybe_scalartype(args.maybe_channel_scales);
80 + auto maybe_tok_scales_type = maybe_scalartype(args.maybe_token_scales);
81 +
82 + {% for impl_config in impl_configs %}
83 + {% set t = impl_config.types -%}
84 + {% set type_sig = gen_type_sig(t) -%}
85 + if (args.b_type == {{VLLMScalarTypeTag[t.b]}}
86 + && a_type == {{TorchTypeTag[t.a]}}
87 + && out_type == {{TorchTypeTag[t.out]}}
88 + && {%if t.b_group_scale != void -%}
89 + maybe_g_scales_type == {{TorchTypeTag[t.b_group_scale]}}
90 + {%- else %}!maybe_g_scales_type{%endif%}
91 + && {%if t.b_group_zeropoint != void -%}
92 + maybe_g_zeros_type == {{TorchTypeTag[t.b_group_zeropoint]}}
93 + {%- else %}!maybe_g_zeros_type{%endif%}
94 + && {%if t.b_channel_scale != void -%}
95 + maybe_ch_scales_type == {{TorchTypeTag[t.b_channel_scale]}}
96 + {%- else %}!maybe_ch_scales_type{%endif%}
97 + && {%if t.a_token_scale != void -%}
98 + maybe_tok_scales_type == {{TorchTypeTag[t.a_token_scale]}}
99 + {%- else %}!maybe_tok_scales_type{%endif%}
100 + ) {
101 + return mm_dispatch_{{type_sig}}(args);
102 + }
103 + {%- endfor %}
104 +
105 + TORCH_CHECK_NOT_IMPLEMENTED(
106 + false, "machete_mm(..) is not implemented for "
107 + "a_type=", args.A.scalar_type(),
108 + ", b_type=", args.b_type.str(),
109 + ", out_type=", out_type,
110 + ", with_group_scale_type=", maybe_g_scales_type
111 + ? toString(*maybe_g_scales_type) : "None",
112 + ", with_group_zeropoint_type=", maybe_g_zeros_type
113 + ? toString(*maybe_g_zeros_type) : "None",
114 + ", with_channel_scale_type=", maybe_ch_scales_type
115 + ? toString(*maybe_ch_scales_type) : "None",
116 + ", with_token_scale_type=", maybe_tok_scales_type
117 + ? toString(*maybe_tok_scales_type) : "None",
118 + "; implemented types are: \\n",
119 + {%- for impl_config in impl_configs %}
120 + {% set t = impl_config.types -%}
121 + "\\t{{gen_type_option_name(t)}}\\n",
122 + {%- endfor %}
123 + "");
124 }
125
126 +std::vector<std::string> supported_schedules_dispatch(
127 + SupportedSchedulesArgs args) {
128 + auto out_type = args.maybe_out_type.value_or(args.a_type);
129 +
130 + {% for impl_config in impl_configs %}
131 + {% set t = impl_config.types -%}
132 + {% set schs = impl_config.schedules -%}
133 + if (args.b_type == {{VLLMScalarTypeTag[t.b]}}
134 + && args.a_type == {{TorchTypeTag[t.a]}}
135 + && out_type == {{TorchTypeTag[t.out]}}
136 + && {%if t.b_group_scale != void -%}
137 + args.maybe_group_scales_type == {{TorchTypeTag[t.b_group_scale]}}
138 + {%- else %}!args.maybe_group_scales_type{%endif%}
139 + && {%if t.b_group_zeropoint != void-%}
140 + args.maybe_group_zeros_type == {{TorchTypeTag[t.b_group_zeropoint]}}
141 + {%- else %}!args.maybe_group_zeros_type{%endif%}
142 + ) {
143 + return {
144 + {%- for s in impl_config.schedules %}
145 + "{{gen_sch_sig(s)}}"{% if not loop.last %},{% endif %}
146 + {%- endfor %}
147 + };
148 + }
149 + {%- endfor %}
150 +
151 + return {};
152 +};
153 +
154 }; // namespace machete
155 """
156
@@ -77,20 +158,10 @@ IMPL_TEMPLATE = """
158 #include "../machete_mm_launcher.cuh"
159
160 namespace machete {
80 -template <typename Config, bool with_C, bool with_scales, bool with_zeropoints>
81 -using Kernel = MacheteKernelTemplate<
82 - {{DataTypeTag[type_config.element_a]}}, // ElementA
83 - {{DataTypeTag[type_config.element_b]}}, // ElementB
84 - {{DataTypeTag[type_config.element_d]}}, // ElementD
85 - {{DataTypeTag[type_config.accumulator]}}, // Accumulator
86 - {{DataTypeTag[type_config.element_b_scale]}}, // Scales
87 - {{DataTypeTag[type_config.element_b_zeropoint]}}, // Zeropoints
88 - cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
89 - Config, with_C, with_scales, with_zeropoints>;
90 -
91 -{% for sch in schedules %}
92 -{% set schedule_name = gen_sch_name(sch) -%}
93 -struct sch_{{schedule_name}} {
161 +
162 +{% for sch in unique_schedules(impl_configs) %}
163 +{% set sch_sig = gen_sch_sig(sch) -%}
164 +struct sch_{{sch_sig}} {
165 using TileShapeNM = Shape<{{
166 to_cute_constant(sch.tile_shape_mn)|join(', ')}}>;
167 using ClusterShape = Shape<{{
@@ -101,27 +172,34 @@ struct sch_{{schedule_name}} {
172 using TileScheduler = {{TileSchedulerTag[sch.tile_scheduler]}};
173 using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
174 };
104 -
175 +{% endfor %}
176 +
177 +{% for impl_config in impl_configs %}
178 +{% set t = impl_config.types -%}
179 +{% set schs = impl_config.schedules -%}
180 +{% set type_sig = gen_type_sig(t) -%}
181 +
182 +template<typename Sch>
183 +using Kernel_{{type_sig}} = MacheteKernelTemplate<
184 + {{DataTypeTag[t.a]}}, // ElementA
185 + {{DataTypeTag[t.b]}}, // ElementB
186 + {{DataTypeTag[t.out]}}, // ElementD
187 + {{DataTypeTag[t.accumulator]}}, // Accumulator
188 + {{DataTypeTag[t.b_group_scale]}}, // GroupScaleT
189 + {{DataTypeTag[t.b_group_zeropoint]}}, // GroupZeroT
190 + {{DataTypeTag[t.b_channel_scale]}}, // ChannelScaleT
191 + {{DataTypeTag[t.a_token_scale]}}, // TokenScaleT
192 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
193 + Sch>;
194 +
195 +{% for sch in schs %}
196 +{% set sch_sig = gen_sch_sig(sch) -%}
197 torch::Tensor
106 -impl_{{type_name}}_sch_{{schedule_name}}(PyTorchArguments args) {
107 - bool with_C = args.C.has_value(), with_scales = args.scales.has_value(),
108 - with_zeropoints = args.zeros.has_value();
109 -
110 - {% for s in specializations %}
111 - if (with_C == {{s.with_C|lower}}
112 - && with_zeropoints == {{s.with_zeropoints|lower}}
113 - && with_scales == {{s.with_scales|lower}}) {
114 - return run_impl<Kernel<sch_{{schedule_name}}, {{s.with_C|lower}},
115 - {{s.with_scales|lower}}, {{s.with_zeropoints|lower}}>>(args);
116 - }{% endfor %}
117 -
118 - TORCH_CHECK_NOT_IMPLEMENTED(
119 - false, "for the sake of compile times and binary size machete_mm(..) is "
120 - " not implemented for with_C=", with_C, ", with_scales=", with_scales,
121 - ", with_zeropoints=", with_zeropoints,
122 - " (for {{type_name}}_sch_{{schedule_name}})");
198 +impl_{{type_sig}}_sch_{{sch_sig}}(MMArgs args) {
199 + return run_impl<Kernel_{{type_sig}}<sch_{{sch_sig}}>>(args);
200 }
124 -{% endfor %}
201 +{%- endfor %}
202 +{%- endfor %}
203
204 }; // namespace machete
205 """
@@ -130,26 +208,34 @@ PREPACK_TEMPLATE = """
208 #include "../machete_prepack_launcher.cuh"
209
210 namespace machete {
133 -using PrepackBDispatcher_ = PrepackBDispatcher<
134 - {{DataTypeTag[type_config.element_a]}}, // ElementA
135 - {{DataTypeTag[type_config.element_b]}}, // ElementB
136 - {{DataTypeTag[type_config.element_d]}}, // ElementD
137 - {{DataTypeTag[type_config.accumulator]}}, // Accumulator
138 - {{DataTypeTag[type_config.element_b_scale]}}, // Scales
139 - {{DataTypeTag[type_config.element_b_zeropoint]}}>; // Zeropoints
140 -
141 -using PrepackedLayoutB = PrepackedLayoutBTemplate<
142 - {{DataTypeTag[type_config.element_a]}}, // ElementA
143 - {{DataTypeTag[type_config.element_b]}}, // ElementB
144 - {{DataTypeTag[type_config.element_d]}}, // ElementD
145 - {{DataTypeTag[type_config.accumulator]}}, // Accumulator
146 - cutlass::layout::ColumnMajor,
147 - cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>;
148 -
149 -template <>
150 -torch::Tensor PrepackBDispatcher_::dispatch(torch::Tensor B) {
151 - return prepack_impl<PrepackedLayoutB>(B);
211 +
212 +torch::Tensor prepack_B_dispatch(PrepackBArgs args) {
213 + auto convert_type = args.maybe_group_scales_type.value_or(args.a_type);
214 + {%- for t in types %}
215 + {% set b_type = unsigned_type_with_bitwidth(t.b_num_bits) %}
216 + if (args.a_type == {{TorchTypeTag[t.a]}}
217 + && args.b_type.size_bits() == {{t.b_num_bits}}
218 + && convert_type == {{TorchTypeTag[t.convert]}}) {
219 + return prepack_impl<
220 + PrepackedLayoutBTemplate<
221 + {{DataTypeTag[t.a]}}, // ElementA
222 + {{DataTypeTag[b_type]}}, // ElementB
223 + {{DataTypeTag[t.convert]}}, // ElementConvert
224 + {{DataTypeTag[t.accumulator]}}, // Accumulator
225 + cutlass::layout::ColumnMajor,
226 + cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>
227 + >(args.B);
228 + }
229 + {%- endfor %}
230 +
231 + TORCH_CHECK_NOT_IMPLEMENTED(false,
232 + "prepack_B_dispatch(..) is not implemented for "
233 + "atype = ", args.a_type,
234 + ", b_type = ", args.b_type.str(),
235 + ", with_group_scales_type= ", args.maybe_group_scales_type ?
236 + toString(*args.maybe_group_scales_type) : "None");
237 }
238 +
239 }; // namespace machete
240 """
241
@@ -166,32 +252,34 @@ class ScheduleConfig:
252 tile_scheduler: TileSchedulerType
253
254
169 -@dataclass
255 +@dataclass(frozen=True)
256 class TypeConfig:
171 - element_a: DataType
172 - element_b: Union[DataType, VLLMDataType]
173 - element_b_scale: DataType
174 - element_b_zeropoint: DataType
175 - element_d: DataType
257 + a: DataType
258 + b: Union[DataType, VLLMDataType]
259 + b_group_scale: DataType
260 + b_group_zeropoint: DataType
261 + b_channel_scale: DataType
262 + a_token_scale: DataType
263 + out: DataType
264 accumulator: DataType
265
266
179 -@dataclass
180 -class Specialization:
181 - with_C: bool
182 - with_zeropoints: bool
183 - with_scales: bool
267 +@dataclass(frozen=True)
268 +class PrepackTypeConfig:
269 + a: DataType
270 + b_num_bits: int
271 + convert: DataType
272 + accumulator: DataType
273
274
275 @dataclass
276 class ImplConfig:
188 - type_config: TypeConfig
189 - schedule_configs: List[ScheduleConfig]
190 - specializations: List[Specialization]
277 + types: TypeConfig
278 + schedules: List[ScheduleConfig]
279 heuristic: List[Tuple[Optional[str], ScheduleConfig]]
280
281
194 -def generate_schedule_name(schedule_config: ScheduleConfig) -> str:
282 +def generate_sch_sig(schedule_config: ScheduleConfig) -> str:
283 tile_shape = (
284 f"{schedule_config.tile_shape_mn[0]}x{schedule_config.tile_shape_mn[1]}"
285 )
@@ -209,40 +297,34 @@ def generate_schedule_name(schedule_config: ScheduleConfig) -> str:
297 f"_{epilogue_schedule}_{tile_scheduler}")
298
299
212 -# mostly unique shorter schedule_name
213 -def generate_terse_schedule_name(schedule_config: ScheduleConfig) -> str:
300 +# mostly unique shorter sch_sig
301 +def generate_terse_sch_sig(schedule_config: ScheduleConfig) -> str:
302 kernel_terse_names_replace = {
303 "KernelTmaWarpSpecializedCooperativeMixedInput_": "TmaMI_",
304 "TmaWarpSpecializedCooperative_": "TmaCoop_",
305 "StreamKScheduler": "streamK",
306 }
307
220 - schedule_name = generate_schedule_name(schedule_config)
308 + sch_sig = generate_sch_sig(schedule_config)
309 for orig, terse in kernel_terse_names_replace.items():
222 - schedule_name = schedule_name.replace(orig, terse)
223 - return schedule_name
310 + sch_sig = sch_sig.replace(orig, terse)
311 + return sch_sig
312
313
314 # unique type_name
227 -def generate_type_signature(kernel_type_config: TypeConfig):
228 - element_a = VLLMDataTypeNames[kernel_type_config.element_a]
229 - element_b = VLLMDataTypeNames[kernel_type_config.element_b]
230 - element_d = VLLMDataTypeNames[kernel_type_config.element_d]
231 - accumulator = VLLMDataTypeNames[kernel_type_config.accumulator]
232 - element_scale = VLLMDataTypeNames[kernel_type_config.element_b_scale]
233 - element_zeropoint = VLLMDataTypeNames[
234 - kernel_type_config.element_b_zeropoint]
235 -
236 - return (f"{element_a}{element_b}{element_d}"
237 - f"{accumulator}{element_scale}{element_zeropoint}")
238 -
315 +def generate_type_signature(kernel_types: TypeConfig):
316 + return str("".join([
317 + VLLMDataTypeNames[getattr(kernel_types, field.name)]
318 + for field in fields(TypeConfig)
319 + ]))
320
240 -# non-unique shorter type_name
241 -def generate_terse_type_signature(kernel_type_config: TypeConfig):
242 - element_a = VLLMDataTypeNames[kernel_type_config.element_a]
243 - element_b = VLLMDataTypeNames[kernel_type_config.element_b]
321
245 - return f"{element_a}{element_b}"
322 +def generate_type_option_name(kernel_types: TypeConfig):
323 + return ", ".join([
324 + f"{field.name.replace('b_', 'with_')+'_type'}=" +
325 + VLLMDataTypeNames[getattr(kernel_types, field.name)]
326 + for field in fields(TypeConfig)
327 + ])
328
329
330 def is_power_of_two(n):
@@ -263,13 +345,36 @@ def to_cute_constant(value: List[int]):
345 return _to_cute_constant(value)
346
347
348 +def unique_schedules(impl_configs: List[ImplConfig]):
349 + return list(
350 + set(sch for impl_config in impl_configs
351 + for sch in impl_config.schedules))
352 +
353 +
354 +def unsigned_type_with_bitwidth(num_bits):
355 + return {
356 + 4: DataType.u4,
357 + 8: DataType.u8,
358 + 16: DataType.u16,
359 + 32: DataType.u32,
360 + 64: DataType.u64,
361 + }[num_bits]
362 +
363 +
364 template_globals = {
365 + "void": DataType.void,
366 "DataTypeTag": VLLMDataTypeTag,
367 + "VLLMScalarTypeTag": VLLMDataTypeVLLMScalarTypeTag,
368 + "TorchTypeTag": VLLMDataTypeTorchDataTypeTag,
369 "KernelScheduleTag": VLLMKernelScheduleTag,
370 "EpilogueScheduleTag": EpilogueScheduleTag,
371 "TileSchedulerTag": TileSchedulerTag,
372 "to_cute_constant": to_cute_constant,
272 - "gen_sch_name": generate_terse_schedule_name,
373 + "gen_sch_sig": generate_terse_sch_sig,
374 + "gen_type_sig": generate_type_signature,
375 + "unique_schedules": unique_schedules,
376 + "unsigned_type_with_bitwidth": unsigned_type_with_bitwidth,
377 + "gen_type_option_name": generate_type_option_name
378 }
379
380
@@ -284,42 +389,82 @@ mm_impl_template = create_template(IMPL_TEMPLATE)
389 prepack_dispatch_template = create_template(PREPACK_TEMPLATE)
390
391
287 -def create_sources(impl_config: ImplConfig, num_impl_files=1):
392 +def create_sources(impl_configs: List[ImplConfig], num_impl_files=8):
393 sources = []
394
290 - type_name = generate_type_signature(impl_config.type_config)
291 - terse_type_name = generate_terse_type_signature(impl_config.type_config)
292 -
395 sources.append((
294 - f"machete_mm_{terse_type_name}",
295 - mm_dispatch_template.render(type_name=type_name,
296 - type_config=impl_config.type_config,
297 - schedules=impl_config.schedule_configs,
298 - heuristic=impl_config.heuristic),
396 + "machete_mm_dispatch",
397 + mm_dispatch_template.render(impl_configs=impl_configs),
398 ))
399
400 + prepack_types = []
401 + for impl_config in impl_configs:
402 + convert_type = impl_config.types.a \
403 + if impl_config.types.b_group_scale == DataType.void \
404 + else impl_config.types.b_group_scale
405 + prepack_types.append(
406 + PrepackTypeConfig(
407 + a=impl_config.types.a,
408 + b_num_bits=VLLMDataTypeSize[impl_config.types.b],
409 + convert=convert_type,
410 + accumulator=impl_config.types.accumulator,
411 + ))
412 +
413 + def prepacked_type_key(prepack_type: PrepackTypeConfig):
414 + # For now we we can just use the first accumulator type seen since
415 + # the tensor core shapes/layouts don't vary based on accumulator
416 + # type so we can generate less code this way
417 + return (prepack_type.a, prepack_type.b_num_bits, prepack_type.convert)
418 +
419 + unique_prepack_types = []
420 + prepack_types_seen = set()
421 + for prepack_type in prepack_types:
422 + key = prepacked_type_key(prepack_type)
423 + if key not in prepack_types_seen:
424 + unique_prepack_types.append(prepack_type)
425 + prepack_types_seen.add(key)
426 +
427 sources.append((
302 - f"machete_prepack_{terse_type_name}",
303 - prepack_dispatch_template.render(
304 - type_name=type_name,
305 - type_config=impl_config.type_config,
306 - ),
428 + "machete_prepack",
429 + prepack_dispatch_template.render(types=unique_prepack_types, ),
430 ))
431
309 - num_schedules = len(impl_config.schedule_configs)
310 - schedules_per_file = math.ceil(num_schedules / num_impl_files)
311 - for part, i in enumerate(range(0, num_schedules, schedules_per_file)):
312 - file_schedules = impl_config.schedule_configs[i:i + schedules_per_file]
432 + # Split up impls across files
433 + num_impls = reduce(lambda x, y: x + len(y.schedules), impl_configs, 0)
434 + num_impls_per_file = math.ceil(num_impls / num_impl_files)
435 +
436 + files_impls: List[List[ImplConfig]] = [[]]
437 +
438 + curr_num_impls_assigned = 0
439 + curr_impl_in_file = 0
440 + curr_impl_configs = deepcopy(list(reversed(impl_configs)))
441 +
442 + while curr_num_impls_assigned < num_impls:
443 + room_left_in_file = num_impls_per_file - curr_impl_in_file
444 + if room_left_in_file == 0:
445 + files_impls.append([])
446 + room_left_in_file = num_impls_per_file
447 + curr_impl_in_file = 0
448 +
449 + curr_ic = curr_impl_configs[-1]
450 + if len(curr_ic.schedules) >= room_left_in_file:
451 + # Break apart the current impl config
452 + tmp_ic = deepcopy(curr_ic)
453 + tmp_ic.schedules = curr_ic.schedules[:room_left_in_file]
454 + curr_ic.schedules = curr_ic.schedules[room_left_in_file:]
455 + files_impls[-1].append(tmp_ic)
456 + else:
457 + files_impls[-1].append(curr_ic)
458 + curr_impl_configs.pop()
459 + curr_num_impls_assigned += len(files_impls[-1][-1].schedules)
460 + curr_impl_in_file += len(files_impls[-1][-1].schedules)
461
462 + for part, file_impls in enumerate(files_impls):
463 sources.append((
315 - f"machete_mm_{terse_type_name}_impl_part{part}",
316 - mm_impl_template.render(
317 - type_name=type_name,
318 - type_config=impl_config.type_config,
319 - schedules=file_schedules,
320 - specializations=impl_config.specializations,
321 - ),
464 + f"machete_mm_impl_part{part+1}",
465 + mm_impl_template.render(impl_configs=file_impls),
466 ))
467 +
468 return sources
469
470
@@ -328,187 +473,169 @@ def generate():
473 # about how this works
474 SCRIPT_DIR = os.path.dirname(__file__)
475
331 - schedule_common_params = dict(
476 + sch_common_params = dict(
477 kernel_schedule=TmaMI,
478 epilogue_schedule=TmaCoop,
479 tile_scheduler=TileSchedulerType.StreamK,
480 )
481
337 - # For now we use the same heuristic for all types
338 - # Heuristic is currently tuned for H100s
339 - default_heuristic = [
482 + # Stored as "condition": ((tile_shape_mn), (cluster_shape_mnk))
483 + default_tile_heuristic_config = {
484 #### M = 257+
341 - (
342 - "M > 256 && K <= 16384 && N <= 4096",
343 - ScheduleConfig(
344 - tile_shape_mn=(128, 128),
345 - cluster_shape_mnk=(2, 1, 1),
346 - **schedule_common_params # type: ignore
347 - )),
348 - (
349 - "M > 256",
350 - ScheduleConfig(
351 - tile_shape_mn=(128, 256),
352 - cluster_shape_mnk=(2, 1, 1),
353 - **schedule_common_params # type: ignore
354 - )),
485 + "M > 256 && K <= 16384 && N <= 4096": ((128, 128), (2, 1, 1)),
486 + "M > 256": ((128, 256), (2, 1, 1)),
487 #### M = 129-256
356 - (
357 - "M > 128 && K <= 4096 && N <= 4096",
358 - ScheduleConfig(
359 - tile_shape_mn=(128, 64),
360 - cluster_shape_mnk=(2, 1, 1),
361 - **schedule_common_params # type: ignore
362 - )),
363 - (
364 - "M > 128 && K <= 8192 && N <= 8192",
365 - ScheduleConfig(
366 - tile_shape_mn=(128, 128),
367 - cluster_shape_mnk=(2, 1, 1),
368 - **schedule_common_params # type: ignore
369 - )),
370 - (
371 - "M > 128",
372 - ScheduleConfig(
373 - tile_shape_mn=(128, 256),
374 - cluster_shape_mnk=(2, 1, 1),
375 - **schedule_common_params # type: ignore
376 - )),
488 + "M > 128 && K <= 4096 && N <= 4096": ((128, 64), (2, 1, 1)),
489 + "M > 128 && K <= 8192 && N <= 8192": ((128, 128), (2, 1, 1)),
490 + "M > 128": ((128, 256), (2, 1, 1)),
491 #### M = 65-128
378 - (
379 - "M > 64 && K <= 4069 && N <= 4069",
380 - ScheduleConfig(
381 - tile_shape_mn=(128, 32),
382 - cluster_shape_mnk=(2, 1, 1),
383 - **schedule_common_params # type: ignore
384 - )),
385 - (
386 - "M > 64 && K <= 4069 && N <= 8192",
387 - ScheduleConfig(
388 - tile_shape_mn=(128, 64),
389 - cluster_shape_mnk=(2, 1, 1),
390 - **schedule_common_params # type: ignore
391 - )),
392 - (
393 - "M > 64 && K >= 8192 && N >= 12288",
394 - ScheduleConfig(
395 - tile_shape_mn=(256, 128),
396 - cluster_shape_mnk=(2, 1, 1),
397 - **schedule_common_params # type: ignore
398 - )),
399 - (
400 - "M > 64",
401 - ScheduleConfig(
402 - tile_shape_mn=(128, 128),
403 - cluster_shape_mnk=(2, 1, 1),
404 - **schedule_common_params # type: ignore
405 - )),
492 + "M > 64 && K <= 4069 && N <= 4069": ((128, 32), (2, 1, 1)),
493 + "M > 64 && K <= 4069 && N <= 8192": ((128, 64), (2, 1, 1)),
494 + "M > 64 && K >= 8192 && N >= 12288": ((256, 128), (2, 1, 1)),
495 + "M > 64": ((128, 128), (2, 1, 1)),
496 #### M = 33-64
407 - (
408 - "M > 32 && K <= 6144 && N <= 6144",
409 - ScheduleConfig(
410 - tile_shape_mn=(128, 16),
411 - cluster_shape_mnk=(1, 1, 1),
412 - **schedule_common_params # type: ignore
413 - )),
414 - (
415 - "M > 32 && K >= 16384 && N >= 12288",
416 - ScheduleConfig(
417 - tile_shape_mn=(256, 64),
418 - cluster_shape_mnk=(2, 1, 1),
419 - **schedule_common_params # type: ignore
420 - )),
421 - (
422 - "M > 32",
423 - ScheduleConfig(
424 - tile_shape_mn=(128, 64),
425 - cluster_shape_mnk=(2, 1, 1),
426 - **schedule_common_params # type: ignore
427 - )),
497 + "M > 32 && K <= 6144 && N <= 6144": ((128, 16), (1, 1, 1)),
498 + "M > 32 && K >= 16384 && N >= 12288": ((256, 64), (2, 1, 1)),
499 + "M > 32": ((128, 64), (2, 1, 1)),
500 #### M = 17-32
429 - (
430 - "M > 16 && K <= 12288 && N <= 8192",
431 - ScheduleConfig(
432 - tile_shape_mn=(128, 32),
433 - cluster_shape_mnk=(2, 1, 1),
434 - **schedule_common_params # type: ignore
435 - )),
436 - (
437 - "M > 16",
438 - ScheduleConfig(
439 - tile_shape_mn=(256, 32),
440 - cluster_shape_mnk=(2, 1, 1),
441 - **schedule_common_params # type: ignore
442 - )),
501 + "M > 16 && K <= 12288 && N <= 8192": ((128, 32), (2, 1, 1)),
502 + "M > 16": ((256, 32), (2, 1, 1)),
503 #### M = 1-16
444 - (
445 - "N >= 26624",
446 - ScheduleConfig(
447 - tile_shape_mn=(256, 16),
448 - cluster_shape_mnk=(1, 1, 1),
449 - **schedule_common_params # type: ignore
450 - )),
451 - (
452 - None,
453 - ScheduleConfig(
454 - tile_shape_mn=(128, 16),
455 - cluster_shape_mnk=(1, 1, 1),
456 - **schedule_common_params # type: ignore
457 - )),
504 + "N >= 26624": ((256, 16), (1, 1, 1)),
505 + None: ((128, 16), (1, 1, 1)),
506 + }
507 +
508 + # For now we use the same heuristic for all types
509 + # Heuristic is currently tuned for H100s
510 + default_heuristic = [
511 + (cond, ScheduleConfig(*tile_config,
512 + **sch_common_params)) # type: ignore
513 + for cond, tile_config in default_tile_heuristic_config.items()
514 ]
515
460 - # Do not use schedules = list(set(...)) because we need to make sure
461 - # the output list is deterministic; otherwise the generated kernel file
462 - # will be non-deterministic and causes ccache miss.
463 - schedules = []
464 - for _, schedule_config in default_heuristic:
465 - if schedule_config not in schedules:
466 - schedules.append(schedule_config)
516 + def get_unique_schedules(heuristic: Dict[str, ScheduleConfig]):
517 + # Do not use schedules = list(set(...)) because we need to make sure
518 + # the output list is deterministic; otherwise the generated kernel file
519 + # will be non-deterministic and causes ccache miss.
520 + schedules = []
521 + for _, schedule_config in heuristic:
522 + if schedule_config not in schedules:
523 + schedules.append(schedule_config)
524 + return schedules
525
526 impl_configs = []
527
528 GPTQ_kernel_type_configs = list(
529 TypeConfig(
472 - element_a=element_a,
473 - element_b=element_b,
474 - element_b_scale=element_a,
475 - element_b_zeropoint=element_a,
476 - element_d=element_a,
530 + a=a,
531 + b=b,
532 + b_group_scale=a,
533 + b_group_zeropoint=DataType.void,
534 + b_channel_scale=DataType.void,
535 + a_token_scale=DataType.void,
536 + out=a,
537 accumulator=DataType.f32,
478 - ) for element_b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
479 - for element_a in (DataType.f16, DataType.bf16))
480 -
481 - GPTQ_kernel_specializations = [
482 - Specialization(with_C=False, with_zeropoints=False, with_scales=True)
483 - ]
538 + ) for b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
539 + for a in (DataType.f16, DataType.bf16))
540
541 impl_configs += [
486 - ImplConfig(x[0], x[1], x[2], x[3])
487 - for x in zip(GPTQ_kernel_type_configs, itertools.repeat(schedules),
488 - itertools.repeat(GPTQ_kernel_specializations),
542 + ImplConfig(x[0], x[1], x[2])
543 + for x in zip(GPTQ_kernel_type_configs,
544 + itertools.repeat(get_unique_schedules(default_heuristic)),
545 itertools.repeat(default_heuristic))
546 ]
547
548 AWQ_kernel_type_configs = list(
549 TypeConfig(
494 - element_a=element_a,
495 - element_b=element_b,
496 - element_b_scale=element_a,
497 - element_b_zeropoint=element_a,
498 - element_d=element_a,
550 + a=a,
551 + b=b,
552 + b_group_scale=a,
553 + b_group_zeropoint=a,
554 + b_channel_scale=DataType.void,
555 + a_token_scale=DataType.void,
556 + out=a,
557 accumulator=DataType.f32,
500 - ) for element_b in (DataType.u4, DataType.u8)
501 - for element_a in (DataType.f16, DataType.bf16))
558 + ) for b in (DataType.u4, DataType.u8)
559 + for a in (DataType.f16, DataType.bf16))
560 +
561 + impl_configs += [
562 + ImplConfig(x[0], x[1], x[2])
563 + for x in zip(AWQ_kernel_type_configs,
564 + itertools.repeat(get_unique_schedules(default_heuristic)),
565 + itertools.repeat(default_heuristic))
566 + ]
567
503 - AWQ_kernel_specializations = [
504 - Specialization(with_C=False, with_zeropoints=True, with_scales=True)
568 + # Stored as "condition": ((tile_shape_mn), (cluster_shape_mnk))
569 + # TODO (LucasWilkinson): Further tuning required
570 + qqq_tile_heuristic_config = {
571 + #### M = 257+
572 + # ((128, 256), (2, 1, 1)) Broken for QQQ types
573 + # TODO (LucasWilkinson): Investigate further
574 + # "M > 256 && K <= 16384 && N <= 4096": ((128, 128), (2, 1, 1)),
575 + # "M > 256": ((128, 256), (2, 1, 1)),
576 + "M > 256": ((128, 128), (2, 1, 1)),
577 + #### M = 129-256
578 + "M > 128 && K <= 4096 && N <= 4096": ((128, 64), (2, 1, 1)),
579 + "M > 128 && K <= 8192 && N <= 8192": ((128, 128), (2, 1, 1)),
580 + # ((128, 256), (2, 1, 1)) Broken for QQQ types
581 + # TODO (LucasWilkinson): Investigate further
582 + # "M > 128": ((128, 256), (2, 1, 1)),
583 + "M > 128": ((128, 128), (2, 1, 1)),
584 + #### M = 65-128
585 + "M > 64 && K <= 4069 && N <= 4069": ((128, 32), (2, 1, 1)),
586 + "M > 64 && K <= 4069 && N <= 8192": ((128, 64), (2, 1, 1)),
587 + "M > 64 && K >= 8192 && N >= 12288": ((256, 128), (2, 1, 1)),
588 + "M > 64": ((128, 128), (2, 1, 1)),
589 + #### M = 33-64
590 + "M > 32 && K <= 6144 && N <= 6144": ((128, 16), (1, 1, 1)),
591 + # Broken for QQQ types
592 + # TODO (LucasWilkinson): Investigate further
593 + #"M > 32 && K >= 16384 && N >= 12288": ((256, 64), (2, 1, 1)),
594 + "M > 32": ((128, 64), (2, 1, 1)),
595 + #### M = 17-32
596 + "M > 16 && K <= 12288 && N <= 8192": ((128, 32), (2, 1, 1)),
597 + "M > 16": ((256, 32), (2, 1, 1)),
598 + #### M = 1-16
599 + "N >= 26624": ((256, 16), (1, 1, 1)),
600 + None: ((128, 16), (1, 1, 1)),
601 + }
602 +
603 + # For now we use the same heuristic for all types
604 + # Heuristic is currently tuned for H100s
605 + qqq_heuristic = [
606 + (cond, ScheduleConfig(*tile_config,
607 + **sch_common_params)) # type: ignore
608 + for cond, tile_config in qqq_tile_heuristic_config.items()
609 + ]
610 +
611 + QQQ_kernel_types = [
612 + *(TypeConfig(
613 + a=DataType.s8,
614 + b=VLLMDataType.u4b8,
615 + b_group_scale=b_group_scale,
616 + b_group_zeropoint=DataType.void,
617 + b_channel_scale=DataType.f32,
618 + a_token_scale=DataType.f32,
619 + out=DataType.f16,
620 + accumulator=DataType.s32,
621 + ) for b_group_scale in (DataType.f16, DataType.void)),
622 + *(TypeConfig(
623 + a=DataType.e4m3,
624 + b=VLLMDataType.u4b8,
625 + b_group_scale=b_group_scale,
626 + b_group_zeropoint=DataType.void,
627 + b_channel_scale=DataType.f32,
628 + a_token_scale=DataType.f32,
629 + out=DataType.f16,
630 + accumulator=DataType.f32,
631 + ) for b_group_scale in (DataType.f16, DataType.void)),
632 ]
633
634 impl_configs += [
508 - ImplConfig(x[0], x[1], x[2], x[3])
509 - for x in zip(AWQ_kernel_type_configs, itertools.repeat(schedules),
510 - itertools.repeat(AWQ_kernel_specializations),
511 - itertools.repeat(default_heuristic))
635 + ImplConfig(x[0], x[1], x[2])
636 + for x in zip(QQQ_kernel_types,
637 + itertools.repeat(get_unique_schedules(qqq_heuristic)),
638 + itertools.repeat(qqq_heuristic))
639 ]
640
641 output_dir = os.path.join(SCRIPT_DIR, "generated")
@@ -521,12 +648,11 @@ def generate():
648 os.makedirs(output_dir)
649
650 # Render each group of configurations into separate files
524 - for impl_config in impl_configs:
525 - for filename, code in create_sources(impl_config):
526 - filepath = os.path.join(output_dir, f"{filename}.cu")
527 - with open(filepath, "w") as output_file:
528 - output_file.write(code)
529 - print(f"Rendered template to {filepath}")
651 + for filename, code in create_sources(impl_configs):
652 + filepath = os.path.join(output_dir, f"{filename}.cu")
653 + with open(filepath, "w") as output_file:
654 + output_file.write(code)
655 + print(f"Rendered template to {filepath}")
656
657
658 if __name__ == "__main__":
csrc/quantization/machete/machete_mainloop.cuh
+11 -14
@@ -171,6 +171,10 @@ struct MacheteCollectiveMma {
171 make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
172 Int<DispatchPolicy::Stages>{})));
173
174 + using SmemLayoutACopy = decltype(GmemLayoutA::TVbNbKL_to_offset_copy(
175 + make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
176 + Int<DispatchPolicy::Stages>{})));
177 +
178 using SmemLayoutAtomARowMajor =
179 decltype(rs_smem_selector<GmmaMajorA, ElementA,
180 decltype(cute::get<0>(TileShape_MNK{})),
@@ -288,14 +292,7 @@ struct MacheteCollectiveMma {
292 static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomScale{})) == 0,
293 "SmemLayoutAtomScale must evenly divide tile k shape.");
294
291 - // Tile along modes in a way that maximizes the TMA box size.
292 - using SmemLayoutACopy = decltype(tile_to_shape(
293 - SmemLayoutAtomARowMajor{},
294 - make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}),
295 - Int<DispatchPolicy::Stages>{}),
296 - conditional_t<::cutlass::gemm::detail::is_major<0, StrideA>(),
297 - Step<_2, _1, _3>, Step<_1, _2, _3>>{}));
298 -
295 + // Tile along modes in a way that maximizes the TMA box size
296 using SmemLayoutB = decltype(tile_to_shape(
297 SmemLayoutAtomB{},
298 make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}),
@@ -428,12 +425,12 @@ struct MacheteCollectiveMma {
425 // clang-format on
426
427 // ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
431 - using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset(
428 + using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset_copy(
429 make_shape(int32_t(0), int32_t(0), int32_t(0)))));
430
431 using ATensor = decltype(make_tensor(
432 get_logical_ptr(static_cast<InternalElementA const*>(nullptr)),
436 - shape(GmemLayoutA::TVbNbKL_to_offset(
433 + shape(GmemLayoutA::TVbNbKL_to_offset_copy(
434 make_shape(int32_t(0), int32_t(0), int32_t(0)))),
435 PrepackedStrideA{}));
436
@@ -450,8 +447,8 @@ struct MacheteCollectiveMma {
447
448 static constexpr auto make_tma_copy_A(ATensor tensor_a = ATensor{}) {
449 return make_tma_copy<TmaElementA>(
453 - GmemTiledCopyA{}, tensor_a, SmemLayoutA{}(_, _, cute::Int<0>{}),
454 - shape(SmemLayoutA{}(_, _, cute::Int<0>{})),
450 + GmemTiledCopyA{}, tensor_a, SmemLayoutACopy{}(_, _, cute::Int<0>{}),
451 + shape(SmemLayoutACopy{}(_, _, cute::Int<0>{})),
452 size<1>(ClusterShape{})); // mcast along N mode for this M load, if any
453 }
454
@@ -584,7 +581,7 @@ struct MacheteCollectiveMma {
581 typename Params::TMA_Scale tma_load_scale;
582 typename Params::TMA_Zero tma_load_zero;
583
587 - auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
584 + auto layout = GmemLayoutA::TVbNbKL_to_offset_copy(make_shape(M, K, L));
585 tma_load_a = make_tma_copy_A(
586 make_logical_tensor(ptr_A, shape(layout), stride(layout)));
587
@@ -722,7 +719,7 @@ struct MacheteCollectiveMma {
719 // (TILE_V,TILE_B,m,k,l)
720 auto make_gA_mkl = [&]() {
721 // ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
725 - auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
722 + auto layout = GmemLayoutA::TVbNbKL_to_offset_copy(make_shape(M, K, L));
723 Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(shape(layout));
724 return local_tile(mA_mkl,
725 make_shape(size<0>(layout), PPBlocksPerTile_MK{}),
csrc/quantization/machete/machete_mm_kernel.cuh
+141 -65
@@ -21,6 +21,8 @@
21
22 #include "cutlass_extensions/cute_utils.cuh"
23 #include "cutlass_extensions/vllm_numeric_conversion.cuh"
24 +#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
25 +#include "cutlass_extensions/torch_utils.hpp"
26 #include "machete_collective_builder.cuh"
27 #include "machete_prepacked_layout.cuh"
28 #include "machete_interleaving_utils.cuh"
@@ -37,27 +39,42 @@ using namespace cute;
39 // W is quantized, in this situation or right-hand operand is quantized so
40 // we compute the transpose to move it to the left-hand side.
41 template <typename ElementA_, typename ElementB_, typename ElementD_,
40 - typename AccumulatorT, typename ScaleT, typename ZeroT,
41 - class KernelSchedule, typename ScheduleConfig, bool with_C,
42 - bool with_scales, bool with_zeropoints>
42 + typename AccumulatorT, typename GroupScaleT, typename GroupZeroT,
43 + typename ChannelScaleT, typename TokenScaleT, class KernelSchedule,
44 + typename ScheduleConfig>
45 struct MacheteKernelTemplate {
46 + static constexpr bool with_C = false; // not ever used
47 + static constexpr bool with_group_scales = !std::is_same_v<GroupScaleT, void>;
48 + static constexpr bool with_group_zeropoints =
49 + !std::is_same_v<GroupZeroT, void>;
50 + static constexpr bool with_channel_scales =
51 + !std::is_same_v<ChannelScaleT, void>;
52 + static constexpr bool with_token_scales = !std::is_same_v<TokenScaleT, void>;
53 +
54 using MmaType = ElementA_;
55 using ElementA = ElementA_;
56 using ElementB = ElementB_;
57 using ElementD = ElementD_;
58 using ElementC = cute::conditional_t<with_C, ElementD, void>;
49 - using ElementZ = ZeroT;
50 - using ElementS = ScaleT;
51 -
52 - using ElementAccumulator =
53 - AccumulatorT; // Element type for internal accumulation
59 + using ElementAccumulator = AccumulatorT;
60 using ElementCompute = AccumulatorT; // For Epilogue
61 + // Use dummy values when we don't have scales or zeropoints
62 + using ElementZGroup =
63 + cute::conditional_t<with_group_zeropoints, GroupZeroT, MmaType>;
64 + using ElementSGroup =
65 + cute::conditional_t<with_group_scales, GroupScaleT, MmaType>;
66 + using ElementConvertGroup =
67 + cute::conditional_t<with_group_scales, GroupScaleT, MmaType>;
68 + using ElementSChannel =
69 + cute::conditional_t<with_channel_scales, ChannelScaleT, AccumulatorT>;
70 + using ElementSToken =
71 + cute::conditional_t<with_token_scales, TokenScaleT, AccumulatorT>;
72
73 using BTypeTuple = cute::conditional_t<
57 - with_scales,
58 - cute::conditional_t<with_zeropoints,
59 - cute::tuple<ElementB, ElementS, ElementZ>,
60 - cute::tuple<ElementB, ElementS>>,
74 + with_group_scales,
75 + cute::conditional_t<with_group_zeropoints,
76 + cute::tuple<ElementB, ElementSGroup, ElementZGroup>,
77 + cute::tuple<ElementB, ElementSGroup>>,
78 ElementB>;
79
80 using LayoutA = cutlass::layout::RowMajor;
@@ -71,8 +88,8 @@ struct MacheteKernelTemplate {
88 using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
89 using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
90 using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
74 - using StrideS = cutlass::detail::TagToStrideA_t<LayoutScale>;
75 - using StrideZ = StrideS;
91 + using StrideSGroup = cutlass::detail::TagToStrideA_t<LayoutScale>;
92 + using StrideZGroup = StrideSGroup;
93
94 using LayoutA_Transpose =
95 typename cutlass::layout::LayoutTranspose<LayoutA>::type;
@@ -85,8 +102,8 @@ struct MacheteKernelTemplate {
102 using OperatorClass = cutlass::arch::OpClassTensorOp;
103
104 using PrepackedLayoutB =
88 - PrepackedLayoutBTemplate<ElementA_, ElementB_, ElementD_, AccumulatorT,
89 - LayoutA_Transpose, KernelSchedule>;
105 + PrepackedLayoutBTemplate<ElementA_, ElementB_, ElementConvertGroup,
106 + AccumulatorT, LayoutA_Transpose, KernelSchedule>;
107
108 static int constexpr TileShapeK =
109 128 * 8 / cutlass::sizeof_bits<MmaType>::value;
@@ -103,12 +120,42 @@ struct MacheteKernelTemplate {
120 using EpilogueTileType = typename ScheduleConfig::EpilogueTileType;
121 using TileScheduler = typename ScheduleConfig::TileScheduler;
122
123 + static_assert(
124 + (!with_channel_scales && !with_token_scales) ||
125 + ((with_channel_scales && with_token_scales) &&
126 + std::is_same_v<ElementSChannel, ElementSToken>),
127 + "Currently token and channel scales (if present) must be the same type");
128 +
129 + using EpilogueDescriptor =
130 + cutlass::epilogue::collective::detail::EpilogueDescriptor<
131 + TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
132 + ElementD, EpilogueSchedule>;
133 +
134 + // Currently only supports float scales
135 + using ChTokScalesEpilogue =
136 + typename vllm::c3x::ScaledEpilogue<ElementAccumulator, ElementD,
137 + EpilogueDescriptor>;
138 + static_assert((with_channel_scales || with_token_scales) ||
139 + (std::is_same_v<ElementSChannel, float> &&
140 + std::is_same_v<ElementSToken, float>),
141 + "Currently token and channel scales (if present) must be float "
142 + "(and if one is present the other must be too)");
143 +
144 + using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT<
145 + cutlass::epilogue::fusion::Sm90AccFetch>;
146 +
147 + using EVTCompute =
148 + std::conditional_t<with_channel_scales || with_token_scales,
149 + typename ChTokScalesEpilogue::EVTCompute,
150 + StoreEpilogueCompute>;
151 +
152 + // EVTCompute
153 using CollectiveEpilogue =
154 typename cutlass::epilogue::collective::CollectiveBuilder<
155 ArchTag, OperatorClass, TileShape, ClusterShape, EpilogueTileType,
109 - ElementAccumulator, ElementAccumulator, ElementC, LayoutC_Transpose,
110 - AlignmentC, ElementD, LayoutD_Transpose, AlignmentD,
111 - EpilogueSchedule>::CollectiveOp;
156 + ElementAccumulator, ElementSChannel, ElementC, LayoutC_Transpose,
157 + AlignmentC, ElementD, LayoutD_Transpose, AlignmentD, EpilogueSchedule,
158 + EVTCompute>::CollectiveOp;
159
160 using CollectiveMainloop =
161 typename cutlass::gemm::collective::VLLMCollectiveBuilder<
@@ -131,26 +178,44 @@ struct MacheteKernelTemplate {
178 using MainloopArguments = typename GemmKernel::MainloopArguments;
179 using EpilogueArguments = typename GemmKernel::EpilogueArguments;
180
134 - template <typename ShapeA, typename ShapeC, typename ShapeD, typename ShapeS,
135 - typename ShapeZ>
181 static Arguments create_arguments(
182 cudaStream_t stream,
138 - ElementA const* A_ptr, // A is an MxK matrix
139 - Layout<ShapeA, StrideA> const& layout_A,
140 - ElementB const* B_ptr, // B is an KxN prepacked matrix
141 - ElementD* D_ptr, // D is an MxN matrix
142 - Layout<ShapeD, StrideD> const& layout_D,
143 - ElementC const* C_ptr, // C is an MxN matrix
144 - std::optional<Layout<ShapeC, StrideC>> const& layout_C,
145 - ElementS const* S_ptr, // S is an scale_KxN matrix
146 - std::optional<Layout<ShapeS, StrideS>> const& layout_S,
147 - ElementZ const* Z_ptr, // Z is an scale_KxN matrix
148 - std::optional<Layout<ShapeZ, StrideZ>> const& layout_Z,
149 - ElementCompute alpha, ElementCompute beta,
150 - std::optional<int> maybe_group_size) {
151 - static_assert(!with_zeropoints || with_scales);
152 -
153 - int M = size<0>(layout_A), N = size<1>(layout_D), K = size<1>(layout_A);
183 + torch::Tensor const& A, // MxK matrix
184 + torch::Tensor const& B, // KxN prepacked matrix
185 + torch::Tensor& D, // MxN matrix
186 + c10::optional<torch::Tensor> const& maybe_g_scales, // scale_KxN matrix
187 + c10::optional<torch::Tensor> const& maybe_g_zeros, // scale_KxN matrix
188 + c10::optional<int64_t> maybe_group_size,
189 + c10::optional<torch::Tensor> const& maybe_ch_scales, // len N vector
190 + c10::optional<torch::Tensor> const& maybe_tok_scales) // len M vector
191 + {
192 + static_assert(!with_group_zeropoints || with_group_scales);
193 +
194 + int M = A.size(0), N = B.size(1), K = A.size(1);
195 + TORCH_CHECK(D.size(0) == M && D.size(1) == N);
196 +
197 + auto layout_A = make_cute_layout<StrideA>(A, "A");
198 + auto layout_D = make_cute_layout<StrideD>(D, "D");
199 + auto layout_S_group =
200 + maybe_make_cute_layout<StrideSGroup>(maybe_g_scales, "group_scales");
201 + auto layout_Z_group =
202 + maybe_make_cute_layout<StrideZGroup>(maybe_g_zeros, "group_zeros");
203 + int64_t numel_S_channel = maybe_ch_scales ? maybe_ch_scales->numel() : 0;
204 + int64_t numel_S_token = maybe_tok_scales ? maybe_tok_scales->numel() : 0;
205 +
206 + auto unwrap = [](auto const& t) {
207 + return t ? t->const_data_ptr() : nullptr;
208 + };
209 + auto A_ptr = static_cast<ElementA const*>(A.const_data_ptr());
210 + auto B_ptr = static_cast<ElementB const*>(B.const_data_ptr());
211 + auto D_ptr = static_cast<ElementD*>(D.mutable_data_ptr());
212 + auto S_group_ptr =
213 + static_cast<ElementSGroup const*>(unwrap(maybe_g_scales));
214 + auto Z_group_ptr = static_cast<ElementZGroup const*>(unwrap(maybe_g_zeros));
215 + auto S_channel_ptr =
216 + static_cast<ElementSChannel const*>(unwrap(maybe_ch_scales));
217 + auto S_token_ptr =
218 + static_cast<ElementSToken const*>(unwrap(maybe_tok_scales));
219
220 int const group_size =
221 maybe_group_size == -1 ? K : maybe_group_size.value_or(K);
@@ -159,26 +224,28 @@ struct MacheteKernelTemplate {
224 TORCH_CHECK(size<0>(layout_A) == M && size<1>(layout_A) == K);
225 TORCH_CHECK(size<0>(layout_D) == M && size<1>(layout_D) == N);
226
162 - if constexpr (with_C) {
163 - TORCH_CHECK(C_ptr && layout_C);
227 + if constexpr (with_group_scales) {
228 + TORCH_CHECK(S_group_ptr && layout_S_group);
229 + TORCH_CHECK((size<0>(*layout_S_group) == scale_k &&
230 + size<1>(*layout_S_group) == N));
231 } else {
165 - TORCH_CHECK(!C_ptr, "C not supported");
232 + TORCH_CHECK(!S_group_ptr, "Scales not supported");
233 }
234
168 - if constexpr (with_scales) {
169 - TORCH_CHECK(S_ptr && layout_S);
170 - TORCH_CHECK((size<0>(*layout_S) == scale_k && size<1>(*layout_S) == N));
235 + if constexpr (with_group_zeropoints) {
236 + TORCH_CHECK(Z_group_ptr && layout_Z_group);
237 + TORCH_CHECK((size<0>(*layout_Z_group) == scale_k &&
238 + size<1>(*layout_Z_group) == N));
239 + TORCH_CHECK(layout_S_group && *layout_Z_group == *layout_S_group,
240 + "Scales and zeros must have the same layout");
241 } else {
172 - TORCH_CHECK(!S_ptr, "Scales not supported");
242 + TORCH_CHECK(!Z_group_ptr, "Zeropoints not supported");
243 }
244
175 - if constexpr (with_zeropoints) {
176 - TORCH_CHECK(Z_ptr && layout_Z);
177 - TORCH_CHECK((size<0>(*layout_Z) == scale_k && size<1>(*layout_Z) == N));
178 - TORCH_CHECK(layout_S && *layout_Z == *layout_S,
179 - "Scales and zeros must have the same layout");
180 - } else {
181 - TORCH_CHECK(!Z_ptr, "Zeropoints not supported");
245 + if constexpr (with_channel_scales || with_token_scales) {
246 + TORCH_CHECK(
247 + (maybe_ch_scales->numel() == N || maybe_ch_scales->numel() == 1) &&
248 + (maybe_tok_scales->numel() == M || maybe_tok_scales->numel() == 1));
249 }
250
251 // Transpose A and D
@@ -186,24 +253,33 @@ struct MacheteKernelTemplate {
253 // for B (which is At)
254 auto stride_At = layout_A.stride();
255 auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
189 - auto stride_Ct = stride_Dt;
190 - if (layout_C) {
191 - stride_Ct = permute_layout<1, 0, 2>(*layout_C).stride();
192 - }
256
257 MainloopArguments mainloop_arguments{};
195 - EpilogueArguments epilogue_arguments{
196 - {alpha, beta}, C_ptr, stride_Ct, D_ptr, stride_Dt};
258 + // {Accum, C, C_layout, D, D}
259 + EpilogueArguments epilogue_arguments{};
260 +
261 + if constexpr (with_channel_scales || with_token_scales) {
262 + epilogue_arguments =
263 + EpilogueArguments{ChTokScalesEpilogue::prepare_args(
264 + *maybe_ch_scales, *maybe_tok_scales),
265 + nullptr,
266 + {},
267 + D_ptr,
268 + stride_Dt};
269 + } else {
270 + epilogue_arguments = EpilogueArguments{{}, nullptr, {}, D_ptr, stride_Dt};
271 + }
272
198 - if constexpr (with_scales && with_zeropoints) {
199 - auto stride_S = permute_layout<1, 0, 2>(*layout_S).stride();
200 - mainloop_arguments =
201 - MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At,
202 - S_ptr, stride_S, group_size, Z_ptr};
203 - } else if constexpr (with_scales) {
204 - auto stride_S = permute_layout<1, 0, 2>(*layout_S).stride();
273 + if constexpr (with_group_scales && with_group_zeropoints) {
274 + auto stride_S_group = permute_layout<1, 0, 2>(*layout_S_group).stride();
275 mainloop_arguments = MainloopArguments{
206 - B_ptr, _StrideB{}, A_ptr, stride_At, S_ptr, stride_S, group_size};
276 + B_ptr, _StrideB{}, A_ptr, stride_At,
277 + S_group_ptr, stride_S_group, group_size, Z_group_ptr};
278 + } else if constexpr (with_group_scales) {
279 + auto stride_S_group = permute_layout<1, 0, 2>(*layout_S_group).stride();
280 + mainloop_arguments =
281 + MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At,
282 + S_group_ptr, stride_S_group, group_size};
283 } else {
284 mainloop_arguments =
285 MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At};
csrc/quantization/machete/machete_mm_launcher.cuh
+35 -55
@@ -5,73 +5,61 @@
5
6 #include "machete_mm_kernel.cuh"
7 #include "cutlass_extensions/torch_utils.hpp"
8 +#include "core/scalar_type.hpp"
9
10 namespace machete {
11
11 -struct PyTorchArguments {
12 +struct MMArgs {
13 torch::Tensor const& A;
14 torch::Tensor const& B;
14 - c10::optional<torch::Tensor> const& scales;
15 - c10::optional<torch::Tensor> const& zeros;
16 - c10::optional<int64_t> group_size;
17 - c10::optional<torch::Tensor> const& C;
18 - c10::optional<double> alpha;
19 - c10::optional<double> beta;
20 - c10::optional<std::string> schedule;
15 + vllm::ScalarType const& b_type;
16 + c10::optional<at::ScalarType> const& maybe_out_type;
17 + c10::optional<torch::Tensor> const& maybe_group_scales;
18 + c10::optional<torch::Tensor> const& maybe_group_zeros;
19 + c10::optional<int64_t> maybe_group_size;
20 + c10::optional<torch::Tensor> const& maybe_channel_scales;
21 + c10::optional<torch::Tensor> const& maybe_token_scales;
22 + c10::optional<std::string> maybe_schedule;
23 };
24
25 +struct SupportedSchedulesArgs {
26 + at::ScalarType a_type;
27 + vllm::ScalarType b_type;
28 + c10::optional<at::ScalarType> maybe_group_scales_type;
29 + c10::optional<at::ScalarType> maybe_group_zeros_type;
30 + c10::optional<at::ScalarType> maybe_channel_scales_type;
31 + c10::optional<at::ScalarType> maybe_token_scales_type;
32 + c10::optional<at::ScalarType> maybe_out_type;
33 +};
34 +
35 +torch::Tensor mm_dispatch(MMArgs args);
36 +
37 +std::vector<std::string> supported_schedules_dispatch(
38 + SupportedSchedulesArgs args);
39 +
40 template <typename MacheteKernel>
24 -torch::Tensor run_impl(PyTorchArguments args) {
41 +torch::Tensor run_impl(MMArgs args) {
42 const at::cuda::OptionalCUDAGuard device_guard(device_of(args.A));
43
44 auto device = args.A.device();
45 auto stream = at::cuda::getCurrentCUDAStream(device.index());
46
30 - using EleA = typename MacheteKernel::ElementA;
31 - using EleB = typename MacheteKernel::ElementB;
32 - using EleC = typename MacheteKernel::ElementC;
33 - using EleD = typename MacheteKernel::ElementD;
34 - using EleScale = typename MacheteKernel::ElementS;
35 - using EleZero = typename MacheteKernel::ElementZ;
36 -
37 - using StrideA = typename MacheteKernel::StrideA;
38 - using StrideC = typename MacheteKernel::StrideC;
39 - using StrideD = typename MacheteKernel::StrideD;
40 - using StrideS = typename MacheteKernel::StrideS;
41 - using StrideZ = typename MacheteKernel::StrideZ;
42 -
47 int M = args.A.size(0);
48 int N = args.B.size(1);
49 int K = args.A.size(1);
50
51 // Allocate output
48 - torch::Tensor D =
49 - torch::empty({M, N}, torch::TensorOptions()
50 - .dtype(equivalent_scalar_type_v<EleD>)
51 - .device(device));
52 -
53 - auto const &A = args.A, &B = args.B;
54 - auto const &C = args.C, &scales = args.scales, &zeros = args.zeros;
55 -
56 - auto layout_A = make_cute_layout<StrideA>(A, "A");
57 - auto layout_D = make_cute_layout<StrideD>(D, "D");
58 - auto layout_C = maybe_make_cute_layout<StrideC>(C, "C");
59 - auto layout_S = maybe_make_cute_layout<StrideS>(scales, "scales");
60 - auto layout_Z = maybe_make_cute_layout<StrideZ>(zeros, "zeros");
61 -
62 - auto A_ptr = static_cast<EleA const*>(A.const_data_ptr());
63 - auto B_ptr = static_cast<EleB const*>(B.const_data_ptr());
64 - auto D_ptr = static_cast<EleD*>(D.mutable_data_ptr());
65 - auto C_ptr = static_cast<EleC const*>(C ? C->const_data_ptr() : nullptr);
66 - auto S_ptr =
67 - static_cast<EleScale const*>(scales ? scales->const_data_ptr() : nullptr);
68 - auto Z_ptr =
69 - static_cast<EleZero const*>(zeros ? zeros->const_data_ptr() : nullptr);
52 + torch::Tensor D = torch::empty(
53 + {M, N},
54 + torch::TensorOptions()
55 + .dtype(equivalent_scalar_type_v<typename MacheteKernel::ElementD>)
56 + .device(device));
57
58 auto arguments = MacheteKernel::create_arguments(
72 - stream, A_ptr, layout_A, B_ptr, D_ptr, layout_D, C_ptr, layout_C, S_ptr,
73 - layout_S, Z_ptr, layout_Z, args.alpha.value_or(1), args.beta.value_or(0),
74 - args.group_size);
59 + stream, //
60 + args.A, args.B, D, args.maybe_group_scales, args.maybe_group_zeros,
61 + args.maybe_group_size, args.maybe_channel_scales,
62 + args.maybe_token_scales);
63 TORCH_CHECK(MacheteKernel::can_implement(arguments),
64 "Machete kernel cannot be run with these arguments");
65
@@ -84,12 +72,4 @@ torch::Tensor run_impl(PyTorchArguments args) {
72 return D;
73 };
74
87 -template <typename ElementA, typename ElementB, typename ElementD = ElementA,
88 - typename AccumulatorT = float, typename ScaleT = ElementA,
89 - typename ZeroT = ElementA>
90 -struct GemmDispatcher {
91 - static torch::Tensor dispatch(PyTorchArguments args);
92 - static std::vector<std::string> supported_schedules();
93 -};
94 -
75 }; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepack_kernel.cuh
+39 -24
@@ -6,31 +6,49 @@
6
7 namespace machete {
8
9 -template <typename TileShapeNKL, typename ElementB, typename BInTensor,
10 - typename BTiledOutTensor>
11 -static __global__ void prepack_B_kernel(BInTensor B_in,
12 - BTiledOutTensor B_tiled_out) {
13 - auto tB_in = local_tile(B_in, TileShapeNKL{},
14 - make_coord(blockIdx.x, blockIdx.y, blockIdx.z));
15 - auto tB_out = B_tiled_out(make_coord(_, _),
16 - make_coord(blockIdx.x, blockIdx.y), blockIdx.z);
9 +template <int threads, typename PrepackedLayoutB, typename BInTensor,
10 + typename ElementB>
11 +static __global__ void prepack_B_kernel(BInTensor B_in, ElementB* B_out_ptr) {
12 + auto constexpr block_size =
13 + Int<size(typename PrepackedLayoutB::PPBlockShape_NK{})>{};
14 + auto constexpr eles_per_thread = Int<block_size / threads>{};
15 + static_assert(block_size % threads == 0,
16 + "block_size must be divisible by the number of threads");
17
18 - auto tiled_copy = make_tiled_copy(Copy_Atom<DefaultCopy, ElementB>{},
19 - Layout<Shape<_4, _32>, Stride<_32, _1>>{},
20 - Layout<Shape<_1, _2>>{});
18 + // Which pre-packed are we responsible for
19 + auto blk_coord = make_coord(blockIdx.x, blockIdx.y, blockIdx.z);
20 + auto tB_in = local_tile(
21 + B_in, append(typename PrepackedLayoutB::PPBlockShape_NK{}, _1{}),
22 + blk_coord);
23
22 - auto thr_copy = tiled_copy.get_thread_slice(threadIdx.x);
24 + // Find the start offset in the output for this pre-packed block
25 + auto bNbKL_to_offset = PrepackedLayoutB::bNbKL_to_offset(shape(B_in));
26
24 - Tensor thr_tile_S = thr_copy.partition_S(tB_in);
25 - Tensor thr_tile_D = thr_copy.partition_D(tB_out);
27 + // Tensor representing a 1:1 mapping to the output space in 1D
28 + auto tB_out_linear =
29 + make_tensor(get_logical_ptr(B_out_ptr) + bNbKL_to_offset(blk_coord),
30 + make_layout(make_shape(block_size)));
31 + // Mapping from output space (1D) to input space
32 + auto tB_in_linear = make_tensor(
33 + tB_in.data(),
34 + tB_in.layout()
35 + .compose(right_inverse(PrepackedLayoutB::ppblock_ilvd_NK_to_offset()))
36 + .with_shape(make_shape(block_size)));
37 +
38 + // Tile for this specific thread (could have used a TiledCopy but these work
39 + // best with 2d layouts, this is a simple 1d layout so local_tile is enough,
40 + // we are also not that concerned with performance for this kernel)
41 + auto thr_tB_in_linear =
42 + local_tile(tB_in_linear, make_shape(eles_per_thread), threadIdx.x);
43 + auto thr_tB_out_linear =
44 + local_tile(tB_out_linear, make_shape(eles_per_thread), threadIdx.x);
45
46 // Construct a register-backed Tensor with the same shape as each thread's
47 // partition
29 - auto fragment = make_tensor<ElementB>(shape(thr_tile_D));
48 + auto fragment = make_tensor<ElementB>(shape(thr_tB_in_linear));
49
31 - // Copy from GMEM to RMEM and from RMEM to GMEM
32 - copy(tiled_copy, thr_tile_S, fragment);
33 - copy(Copy_Atom<DefaultCopy, uint8_t>{}, fragment, thr_tile_D);
50 + copy(thr_tB_in_linear, fragment);
51 + copy(Copy_Atom<DefaultCopy, uint8_t>{}, fragment, thr_tB_out_linear);
52 }
53
54 template <typename PrepackedLayoutB, typename InLayout>
@@ -44,18 +62,15 @@ static void prepack_B_template(
62
63 TORCH_CHECK(size<0>(B_layout) % size<0>(TileShapeNKL{}) == 0);
64 TORCH_CHECK(size<1>(B_layout) % size<1>(TileShapeNKL{}) == 0);
47 - TORCH_CHECK(size<2>(B_layout) % size<2>(TileShapeNKL{}) == 0);
65
66 auto N_tiles = size<0>(B_layout) / size<0>(TileShapeNKL{});
67 auto K_tiles = size<1>(B_layout) / size<1>(TileShapeNKL{});
51 - auto L_tiles = size<2>(B_layout) / size<2>(TileShapeNKL{});
68 + auto L_tiles = size<2>(B_layout);
69
70 auto B_in = make_tensor(get_logical_ptr(B_in_ptr), B_layout);
54 - auto B_tiled_out =
55 - make_tensor(get_logical_ptr(B_out_ptr), ilvd_NKbNbKL_to_offset);
71
57 - prepack_B_kernel<TileShapeNKL, typename PrepackedLayoutB::ElementB>
58 - <<<dim3(N_tiles, K_tiles, L_tiles), 128, 0, stream>>>(B_in, B_tiled_out);
72 + prepack_B_kernel<128, PrepackedLayoutB>
73 + <<<dim3(N_tiles, K_tiles, L_tiles), 128, 0, stream>>>(B_in, B_out_ptr);
74 }
75
76 }; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepack_launcher.cuh
+9 -6
@@ -2,9 +2,17 @@
2
3 #include "machete_prepack_kernel.cuh"
4 #include "cutlass_extensions/torch_utils.hpp"
5 +#include "core/scalar_type.hpp"
6
7 namespace machete {
8
9 +struct PrepackBArgs {
10 + torch::Tensor const& B;
11 + at::ScalarType a_type;
12 + vllm::ScalarType b_type;
13 + c10::optional<at::ScalarType> maybe_group_scales_type;
14 +};
15 +
16 template <typename PrepackedLayoutB>
17 torch::Tensor prepack_impl(torch::Tensor const B) {
18 const at::cuda::OptionalCUDAGuard device_guard(device_of(B));
@@ -61,11 +69,6 @@ torch::Tensor prepack_impl(torch::Tensor const B) {
69 return D;
70 };
71
64 -template <typename ElementA, typename ElementB, typename ElementD,
65 - typename AccumulatorT = float, typename ScaleT = cutlass::half_t,
66 - typename ZeroT = cutlass::half_t>
67 -struct PrepackBDispatcher {
68 - static torch::Tensor dispatch(torch::Tensor B);
69 -};
72 +torch::Tensor prepack_B_dispatch(PrepackBArgs args);
73
74 }; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepacked_layout.cuh
+42 -12
@@ -41,7 +41,7 @@ struct IlvBlkLayoutAuto {};
41 // The contract here is that the `TiledMma` determined below matches the one
42 // ultimately used in the kernel. (this is also why the other element types are
43 // required along with the kernel schedule)
44 -template <typename ElementA_, typename ElementB_, typename ElementD_,
44 +template <typename ElementA_, typename ElementB_, typename ElementConvert_,
45 typename AccumulatorT, class LayoutB, class KernelSchedule,
46 typename IlvBlkLayout_ = IlvBlkLayoutAuto>
47 // clang-format on
@@ -49,20 +49,27 @@ struct PrepackedLayoutBTemplate {
49 using MmaType = ElementA_;
50 using ElementA = ElementA_;
51 using ElementB = ElementB_;
52 - using ElementD = ElementD_;
53 - using ElementAccumulator =
54 - AccumulatorT; // Element type for internal accumulation
52 + using ElementAccumulator = AccumulatorT;
53 using ElementMma = MmaType;
54
57 - // Only use interleaved layouts for subbyte weights, prmt instructions makes
58 - // non-interleaved layouts for 8bit+ weights efficient enough we don't need
59 - // iterleaved layouts
55 + // Interleave for 4bit bit types when we are not upconverting to fp8 or int8,
56 + // in those cases case we use a LUT using prmt instructions to upconvert and
57 + // is more efficient if the data is not interleaved For 8bit+ prmt
58 + // instructions makes non-interleaved layouts efficient enough we don't need
59 + // iterleaved layouts (and can reuse more of the existing cutlass converts)
60 + static constexpr bool should_interleave =
61 + sizeof_bits_v<ElementB> <= 4 &&
62 + !std::is_same_v<ElementConvert_, cutlass::float_e4m3_t> &&
63 + !std::is_same_v<ElementConvert_, int8_t>;
64 +
65 + // Only use interleaved layouts for subbyte weights,
66 using IlvdBlkLayout = std::conditional_t<
67 std::is_same_v<IlvBlkLayout_, IlvBlkLayoutAuto>,
62 - std::conditional_t<sizeof_bits_v<ElementB> <= 4,
63 - decltype(get_interleaved_blk_layout<
64 - ElementB, sizeof_bits_v<ElementA>, 32>()),
65 - void>,
68 + std::conditional_t<
69 + should_interleave,
70 + decltype(get_interleaved_blk_layout<
71 + ElementB, sizeof_bits_v<ElementConvert_>, 32>()),
72 + void>,
73 IlvBlkLayout_>;
74
75 // TODO (LucasWilkinson): compare the performance for other sizes
@@ -135,7 +142,8 @@ struct PrepackedLayoutBTemplate {
142 // then ((IlvBlk), FrgB) is {A, C, B, D, C, G, D, H}
143 auto frgV = get<1, 0>(layout_no_interleave);
144 auto ilvdBlk = IlvdBlkLayout{};
138 - static_assert(size(frgV) % 4 == 0, "FrgV must be divisible by 4");
145 + static_assert(size(frgV) % size(ilvdBlk) == 0,
146 + "FrgV must be divisible by size(ilvdBlk)");
147 auto ilvd_FrgV = make_layout(
148 make_shape(shape(ilvdBlk), Int<size(frgV) / size(ilvdBlk)>{}),
149 make_stride(stride(ilvdBlk), size(ilvdBlk)));
@@ -175,6 +183,15 @@ struct PrepackedLayoutBTemplate {
183 return group<1, 3>(result(_, repeat<rank<1>(result)>(_)));
184 }
185
186 + // ((athrid_val), (BlocksN, BlocksK, L)) -> (N, K, L)
187 + template <typename Shape_NKL>
188 + CUTE_HOST_DEVICE static constexpr auto TVbNbKL_to_offset_copy(
189 + Shape_NKL shape_mkl) {
190 + auto layout = TVbNbKL_to_offset(shape_mkl);
191 + return make_layout(coalesce(get<0>(layout)), get<1>(layout),
192 + get<2>(layout));
193 + }
194 +
195 // ((BlockN, BlockK), (BlocksN, BlocksK), L) -> (storage_idx)
196 template <typename Shape_NKL>
197 CUTE_HOST_DEVICE static constexpr auto ilvd_NKbNbKL_to_offset(
@@ -197,6 +214,19 @@ struct PrepackedLayoutBTemplate {
214 return group<1, 3>(result(_, repeat<rank<1>(result)>(_)));
215 }
216
217 + // (BlocksN, BlocksK, L) -> (storage_idx)
218 + template <typename Shape_NKL>
219 + CUTE_HOST_DEVICE static constexpr auto bNbKL_to_offset(Shape_NKL shape_mkl) {
220 + // (BlocksN, BlocksK, L)
221 + auto blocks_shape =
222 + cute::transform(shape_mkl, append(PPBlockShape_NK{}, _1{}),
223 + [](auto x, auto y) { return x / y; });
224 + auto stride = size(PPBlockShape_NK{});
225 +
226 + // (BlocksN, BlocksK, L) -> (storage_idx)
227 + return make_layout(blocks_shape, compact_col_major(blocks_shape, stride));
228 + }
229 +
230 // ((athrid, val), (BlocksN, BlocksK, L)) -> (N, K, L)
231 template <class Shape_NKL>
232 CUTE_HOST_DEVICE static auto TVbNbK_to_NKL(Shape_NKL shape_mkl) {
csrc/quantization/machete/machete_pytorch.cu
+46 -74
@@ -8,89 +8,61 @@ namespace machete {
8
9 using namespace vllm;
10
11 -//
12 -// Utils (type dispatching)
13 -//
14 -
15 -template <typename Fn>
16 -static auto scalar_type_dispatch(ScalarType const& type, Fn fn) {
17 - if (type == vllm::kU4) {
18 - return fn(cutlass::uint4b_t{});
19 - } else if (type == vllm::kU8) {
20 - return fn(cutlass::uint8_t{});
21 - } else if (type == vllm::kU4B8) {
22 - return fn(cutlass::vllm_uint4b8_t{});
23 - } else if (type == vllm::kU8B128) {
24 - return fn(cutlass::vllm_uint8b128_t{});
25 - } else {
26 - TORCH_CHECK(false, "Unsupported type ", type.str());
27 - }
28 -}
29 -
30 -#define AT_DISPATCH_CASE_SUPPORTED_COMPUTE_TYPES(...) \
31 - AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__)
32 -
33 -#define AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(TYPE, NAME, ...) \
34 - AT_DISPATCH_SWITCH(TYPE, NAME, \
35 - AT_DISPATCH_CASE_SUPPORTED_COMPUTE_TYPES(__VA_ARGS__))
36 -
37 -//
38 -// Interface
39 -//
40 -
41 -std::vector<std::string> supported_schedules(ScalarTypeId const btype_id) {
42 -#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
43 - vllm::ScalarType b_type = ScalarType::from_id(btype_id);
44 - return scalar_type_dispatch(b_type, [&](auto BType) {
45 - return GemmDispatcher<half_t, decltype(BType)>::supported_schedules();
11 +std::vector<std::string> supported_schedules(
12 + at::ScalarType a_type, int64_t b_type_id,
13 + c10::optional<at::ScalarType> maybe_group_scales_type,
14 + c10::optional<at::ScalarType> maybe_group_zeros_type,
15 + c10::optional<at::ScalarType> maybe_channel_scales_type,
16 + c10::optional<at::ScalarType> maybe_token_scales_type,
17 + c10::optional<at::ScalarType> maybe_out_type) {
18 + ScalarType const b_type = ScalarType::from_id(b_type_id);
19 + return supported_schedules_dispatch({
20 + .a_type = a_type,
21 + .b_type = b_type,
22 + .maybe_group_scales_type = maybe_group_scales_type,
23 + .maybe_group_zeros_type = maybe_group_zeros_type,
24 + .maybe_channel_scales_type = maybe_channel_scales_type,
25 + .maybe_token_scales_type = maybe_token_scales_type,
26 + .maybe_out_type = maybe_out_type,
27 });
47 -#else
48 - TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
49 -#endif
28 }
29
52 -torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
53 - ScalarTypeId const btype_id,
54 - c10::optional<torch::Tensor> const& scales,
55 - c10::optional<torch::Tensor> const& zeros,
56 - c10::optional<int64_t> group_size,
57 - c10::optional<torch::Tensor> const& C,
58 - c10::optional<double> alpha, c10::optional<double> beta,
59 - c10::optional<std::string> schedule) {
60 -#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
61 - ScalarType const btype = ScalarType::from_id(btype_id);
62 - auto args = PyTorchArguments{.A = A,
63 - .B = B,
64 - .scales = scales,
65 - .zeros = zeros,
66 - .group_size = group_size,
67 - .C = C,
68 - .alpha = alpha,
69 - .beta = beta,
70 - .schedule = schedule};
71 -
72 - return scalar_type_dispatch(btype, [&](auto BType) {
73 - return AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(
74 - A.scalar_type(), "machete_gemm", [&] {
75 - using ComputeType = equivalent_cutlass_type_t<scalar_t>;
76 - return GemmDispatcher<ComputeType, decltype(BType)>::dispatch(args);
77 - });
78 - });
79 -#else
80 - TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
81 -#endif
30 +torch::Tensor mm(torch::Tensor const& A, torch::Tensor const& B,
31 + int64_t b_type_id,
32 + c10::optional<at::ScalarType> const& maybe_out_type,
33 + c10::optional<torch::Tensor> const& maybe_group_scales,
34 + c10::optional<torch::Tensor> const& maybe_group_zeros,
35 + c10::optional<int64_t> maybe_group_size,
36 + c10::optional<torch::Tensor> const& maybe_channel_scales,
37 + c10::optional<torch::Tensor> const& maybe_token_scales,
38 + c10::optional<std::string> maybe_schedule) {
39 + ScalarType const b_type = ScalarType::from_id(b_type_id);
40 + return mm_dispatch({.A = A,
41 + .B = B,
42 + .b_type = b_type,
43 + .maybe_out_type = maybe_out_type,
44 + .maybe_group_scales = maybe_group_scales,
45 + .maybe_group_zeros = maybe_group_zeros,
46 + .maybe_group_size = maybe_group_size,
47 + .maybe_channel_scales = maybe_channel_scales,
48 + .maybe_token_scales = maybe_token_scales,
49 + .maybe_schedule = maybe_schedule});
50 }
51
84 -torch::Tensor prepack_B(torch::Tensor const& B, ScalarTypeId const btype_id) {
85 - ScalarType const btype = ScalarType::from_id(btype_id);
86 - return scalar_type_dispatch(btype, [&](auto BType) {
87 - return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
88 - });
52 +torch::Tensor prepack_B(
53 + torch::Tensor const& B, at::ScalarType const& a_type, int64_t b_type_id,
54 + c10::optional<at::ScalarType> const& maybe_group_scales_type) {
55 + ScalarType const b_type = ScalarType::from_id(b_type_id);
56 + return prepack_B_dispatch(
57 + {.B = B,
58 + .a_type = a_type,
59 + .b_type = b_type,
60 + .maybe_group_scales_type = maybe_group_scales_type});
61 }
62
63 TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
64 m.impl("machete_prepack_B", &prepack_B);
93 - m.impl("machete_gemm", &gemm);
65 + m.impl("machete_mm", &mm);
66 }
67
68 // use CatchAll since supported_schedules has no tensor arguments
csrc/torch_bindings.cpp
+29 -6
@@ -203,13 +203,36 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
203 // conditionally compiled so impl in source file
204
205 // Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
206 - ops.def("machete_supported_schedules(int btype) -> str[]");
206 ops.def(
208 - "machete_gemm(Tensor A, Tensor B, int btype, "
209 - " Tensor? scales, Tensor? zeros, int? group_size, "
210 - " Tensor? C, float? alpha, float? beta, str? schedule)"
211 - "-> Tensor");
212 - ops.def("machete_prepack_B(Tensor B, int btype) -> Tensor");
207 + "machete_supported_schedules("
208 + " ScalarType a_type,"
209 + " int b_type,"
210 + " ScalarType? maybe_group_scales_type,"
211 + " ScalarType? maybe_group_zeros_type,"
212 + " ScalarType? maybe_channel_scales_type,"
213 + " ScalarType? maybe_token_scales_type,"
214 + " ScalarType? maybe_out_type"
215 + ") -> str[]");
216 + ops.def(
217 + "machete_mm("
218 + " Tensor A,"
219 + " Tensor B,"
220 + " int b_type,"
221 + " ScalarType? out_type,"
222 + " Tensor? group_scales,"
223 + " Tensor? group_zeros,"
224 + " int? group_size,"
225 + " Tensor? channel_scales,"
226 + " Tensor? token_scales,"
227 + " str? schedule"
228 + ") -> Tensor");
229 + ops.def(
230 + "machete_prepack_B("
231 + " Tensor B,"
232 + " ScalarType a_type,"
233 + " int b_type,"
234 + " ScalarType? group_scales_type"
235 + ") -> Tensor");
236 // conditionally compiled so impl registration is in source file
237
238 ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
tests/kernels/test_machete_gemm.py deleted
-284
@@ -1,284 +0,0 @@
1 -"""Tests for the machete kernel.
2 -
3 -Run `pytest tests/kernels/test_machete_gemm.py`.
4 -"""
5 -
6 -import math
7 -from typing import Optional, Tuple
8 -
9 -import pytest
10 -import torch
11 -
12 -from tests.kernels.utils import opcheck
13 -from vllm import _custom_ops as ops
14 -from vllm.model_executor.layers.quantization.utils.quant_utils import (
15 - pack_rows, quantize_weights)
16 -from vllm.platforms import current_platform
17 -from vllm.scalar_type import ScalarType, scalar_types
18 -
19 -CUDA_DEVICES = [
20 - f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
21 -]
22 -
23 -MNK_SHAPES = [
24 - (1, 128, 128),
25 - (1, 512, 1024),
26 - (1, 4096, 4096),
27 - (1, 8192, 28672),
28 - (13, 8192, 4096),
29 - (26, 4096, 8192),
30 - (64, 4096, 4096),
31 - (64, 8192, 28672),
32 - (257, 128, 4096),
33 - (257, 4224, 4160),
34 - (257, 4096, 4096),
35 - (1024, 4096, 8192),
36 - (1024, 8192, 4096),
37 -]
38 -
39 -ACT_TYPES = [torch.float16, torch.bfloat16]
40 -WTYPE_ZEROPOINTS = [
41 - # GPTQ style
42 - (scalar_types.uint4b8, False),
43 - (scalar_types.uint8b128, False),
44 - # AWQ style
45 - (scalar_types.uint4, True),
46 - (scalar_types.uint8, True),
47 -]
48 -
49 -# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
50 -# unit tests to a common utility function. Currently the use of
51 -# `is_quant_method_supported` conflates kernels with quantization methods
52 -# an assumption which is breaking down as quantizations methods can have
53 -# have kernels and some kernels support multiple quantization methods.
54 -IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
55 -
56 -
57 -def rand_data(shape, dtype=torch.float16):
58 - return 10 * (torch.rand(shape, dtype=dtype, device="cuda") - 0.3)
59 -
60 -
61 -def maybe_convert_zeropoints(zps: Optional[torch.Tensor], s: torch.Tensor):
62 - return zps if zps is None else -1 * s * (zps.to(s.dtype))
63 -
64 -
65 -def machete_quantize_and_pack(w: torch.Tensor,
66 - wtype: ScalarType,
67 - group_size: int,
68 - zero_points: bool = False):
69 - assert wtype.is_integer(), "TODO: support floating point weights"
70 -
71 - w_ref, w_q, w_s, w_zp = quantize_weights(
72 - w,
73 - wtype,
74 - group_size,
75 - zero_points=zero_points,
76 - # to match how the kernel applies zps
77 - ref_zero_points_after_scales=True)
78 -
79 - w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
80 - w_q = w_q.t().contiguous().t() # convert to col major
81 - w_q_machete = ops.machete_prepack_B(w_q, wtype)
82 -
83 - opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype.id))
84 -
85 - return w_ref, w_q_machete, w_s, w_zp
86 -
87 -
88 -def machete_gemm_test_helper(a: torch.Tensor, b: torch.Tensor,
89 - wtype: ScalarType, group_size: int,
90 - zero_points: bool):
91 - w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
92 - b, wtype, group_size, zero_points)
93 -
94 - output_ref = torch.matmul(a, w_ref)
95 -
96 - output = ops.machete_gemm(
97 - a=a,
98 - b_q=w_q_packed,
99 - b_type=wtype,
100 - b_scales=w_s,
101 - b_zeros=maybe_convert_zeropoints(w_zp, w_s),
102 - b_group_size=group_size,
103 - )
104 -
105 - # Relax atol as our reduction dim becomes larger (more rounding error)
106 - # Relax atol when we have zeropoints since the way machete applies
107 - # zeropoints (after scales) causes noise around 0
108 - atol = 1 if zero_points else min(5e-2 * math.sqrt(a.shape[1]), 1)
109 - torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
110 -
111 -
112 -@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
113 - reason="Machete is not supported on this GPU type.")
114 -@pytest.mark.parametrize("shape",
115 - MNK_SHAPES,
116 - ids=lambda x: "x".join(str(v) for v in x))
117 -@pytest.mark.parametrize("atype", ACT_TYPES, ids=lambda x: str(x))
118 -@pytest.mark.parametrize("wtype_zeropoints", WTYPE_ZEROPOINTS)
119 -@pytest.mark.parametrize("group_size", [128, None])
120 -def test_machete_all_schedules(shape, atype: torch.dtype,
121 - wtype_zeropoints: Tuple[ScalarType, bool],
122 - group_size: Optional[int]):
123 - m, n, k = shape
124 - wtype, zero_points = wtype_zeropoints
125 -
126 - if group_size is not None and k % group_size != 0:
127 - return
128 -
129 - print(f"MNK = {m} {n} {k}")
130 -
131 - # Normalize group_size
132 - if group_size is None:
133 - group_size = k
134 - assert group_size <= k
135 -
136 - a = rand_data((m, k), atype)
137 - w = rand_data((k, n), atype)
138 -
139 - w_ref, w_q_machete, w_s, w_zp = machete_quantize_and_pack(
140 - w, wtype, group_size, zero_points)
141 -
142 - output_ref = torch.matmul(a, w_ref)
143 -
144 - for schedule in ops.machete_supported_schedules(wtype):
145 - print(f"Testing schedule {schedule}")
146 - output = ops.machete_gemm(
147 - a,
148 - b_q=w_q_machete,
149 - b_type=wtype,
150 - b_scales=w_s,
151 - b_zeros=maybe_convert_zeropoints(w_zp, w_s),
152 - b_group_size=group_size,
153 - schedule=schedule,
154 - )
155 -
156 - opcheck(
157 - torch.ops._C.machete_gemm,
158 - (a, w_q_machete, wtype.id, w_s, maybe_convert_zeropoints(
159 - w_zp, w_s), group_size, None, None, None, schedule))
160 -
161 - # Relax atol as our reduction dim becomes larger (more rounding error)
162 - # Relax atol when we have zeropoints since the way machete applies
163 - # zeropoints (after scales) causes noise around 0
164 - atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
165 - torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol),\
166 - f"Schedule failed {schedule}"
167 -
168 -
169 -@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
170 - reason="Machete is not supported on this GPU type.")
171 -@pytest.mark.parametrize("shape",
172 - MNK_SHAPES,
173 - ids=lambda x: "x".join(str(v) for v in x))
174 -@pytest.mark.parametrize("atype", ACT_TYPES, ids=lambda x: str(x))
175 -@pytest.mark.parametrize("wtype_zeropoints", WTYPE_ZEROPOINTS)
176 -@pytest.mark.parametrize("group_size", [128, None])
177 -def test_machete_heuristic(shape, atype: torch.dtype,
178 - wtype_zeropoints: Tuple[ScalarType, bool],
179 - group_size: Optional[int]):
180 - m, n, k = shape
181 - wtype, zero_points = wtype_zeropoints
182 -
183 - if group_size is not None and k % group_size != 0:
184 - return
185 -
186 - # Normalize group_size
187 - if group_size is None:
188 - group_size = k
189 - assert group_size <= k
190 -
191 - a = rand_data((m, k), atype)
192 - b = rand_data((k, n), atype)
193 -
194 - machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
195 -
196 -
197 -# Test working on other devices
198 -@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
199 - reason="Machete is not supported on this GPU type.")
200 -@pytest.mark.parametrize("device", CUDA_DEVICES)
201 -def test_machete_devices(device: str):
202 - m, n, k = 512, 4096, 4096
203 - wtype = scalar_types.uint4b8
204 - group_size = 128
205 - zero_points = False
206 -
207 - print(f"MNK = {m} {n} {k}, device = {device}")
208 -
209 - a = rand_data((m, k), torch.float16).to(device)
210 - b = rand_data((k, n), torch.float16).to(device)
211 -
212 - machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
213 -
214 -
215 -# Test working with a subset of A and B
216 -@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
217 - reason="Machete is not supported on this GPU type.")
218 -def test_machete_subset():
219 - big_m, big_n, big_k = 1024, 1024, 1024
220 - m, n, k = 512, 512, 512
221 - wtype = scalar_types.uint4b8
222 - group_size = 128
223 - zero_points = False
224 -
225 - whole_a = rand_data((big_m, big_k), torch.float16)
226 - whole_b = rand_data((big_k, big_n), torch.float16)
227 -
228 - a = whole_a[0:m, 0:k]
229 - b = whole_b[0:k, 0:n]
230 -
231 - machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
232 -
233 -
234 -# Test to make sure cuda graphs work
235 -class MacheteLayer(torch.nn.Module):
236 -
237 - def __init__(self, **kwargs):
238 - super().__init__()
239 - self.kwargs = kwargs
240 -
241 - def forward(self, a):
242 - return ops.machete_gemm(**self.kwargs)
243 -
244 -
245 -@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
246 - reason="Machete is not supported on this GPU type.")
247 -def test_machete_cuda_graph():
248 - m, n, k = 512, 4096, 4096
249 -
250 - a = rand_data((m, k), torch.float16)
251 - b = rand_data((k, n), torch.float16)
252 - wtype = scalar_types.uint4b8
253 - group_size = 128
254 - zero_points = False
255 -
256 - w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
257 - b, wtype, group_size, zero_points)
258 -
259 - # Construct a trivial model with a single layer that calls a machete kernel
260 - model = MacheteLayer(
261 - a=a,
262 - b_q=w_q_packed,
263 - b_type=wtype,
264 - b_scales=w_s,
265 - b_zeros=maybe_convert_zeropoints(w_zp, w_s),
266 - b_group_size=group_size,
267 - )
268 -
269 - output_ref = torch.matmul(a, w_ref)
270 -
271 - # Run the model with a cuda graph
272 - stream = torch.cuda.Stream()
273 - with torch.cuda.stream(stream):
274 - g = torch.cuda.CUDAGraph()
275 - with torch.cuda.graph(g):
276 - output = model(a)
277 - output.zero_()
278 - g.replay()
279 -
280 - # Relax atol as our reduction dim becomes larger (more rounding error)
281 - # Relax atol when we have zeropoints since the way machete applies
282 - # zeropoints (after scales) causes noise around 0
283 - atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
284 - torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
tests/kernels/test_machete_mm.py new
+406
@@ -0,0 +1,406 @@
1 +"""Tests for the machete kernel.
2 +
3 +Run `pytest tests/kernels/test_machete_mm.py`.
4 +"""
5 +
6 +import math
7 +from dataclasses import dataclass, fields
8 +from typing import List, Optional, Tuple
9 +
10 +import pytest
11 +import torch
12 +
13 +from tests.kernels.utils import opcheck
14 +from vllm import _custom_ops as ops
15 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
16 + pack_rows, quantize_weights)
17 +from vllm.platforms import current_platform
18 +from vllm.scalar_type import ScalarType, scalar_types
19 +
20 +CUDA_DEVICES = [
21 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
22 +]
23 +
24 +# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
25 +# unit tests to a common utility function. Currently the use of
26 +# `is_quant_method_supported` conflates kernels with quantization methods
27 +# an assumption which is breaking down as quantizations methods can have
28 +# have kernels and some kernels support multiple quantization methods.
29 +IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
30 +
31 +MNK_SHAPES = [
32 + (1, 128, 128),
33 + (1, 512, 1024),
34 + (1, 4096, 4096),
35 + (1, 8192, 28672),
36 + (13, 8192, 4096),
37 + (26, 4096, 8192),
38 + (64, 4096, 4096),
39 + (64, 8192, 28672),
40 + (257, 128, 4096),
41 + (257, 4224, 4160),
42 + (257, 4096, 4096),
43 + (1024, 4096, 8192),
44 + (1024, 8192, 4096),
45 +]
46 +
47 +GROUP_SIZES_TO_TEST: List[Optional[int]] = [128, -1]
48 +
49 +
50 +@dataclass
51 +class TypeConfig:
52 + act_type: torch.dtype
53 + weight_type: ScalarType
54 + output_type: Optional[torch.dtype]
55 + group_scale_type: Optional[torch.dtype]
56 + group_zero_type: Optional[torch.dtype]
57 + channel_scale_type: Optional[torch.dtype]
58 + token_scale_type: Optional[torch.dtype]
59 +
60 +
61 +@dataclass
62 +class Tensors:
63 + w_ref: torch.Tensor
64 + a_ref: torch.Tensor
65 + a: torch.Tensor
66 + w_q: torch.Tensor
67 + w_g_s: Optional[torch.Tensor]
68 + w_g_zp: Optional[torch.Tensor]
69 + w_ch_s: Optional[torch.Tensor]
70 + w_tok_s: Optional[torch.Tensor]
71 +
72 +
73 +# (Act Type, Weight Type, Output Type, Scale Type, ZeroPoints,
74 +# Ch Scales Type, Tok Scales Type)
75 +# NOTE: None "Scale Type" means the act type is floating point
76 +# None "Output Type" means the output type is the same as the act type
77 +TestTypeTuple = Tuple[List[torch.dtype], ScalarType, Optional[torch.dtype],
78 + Optional[torch.dtype], bool]
79 +TEST_TYPES = [
80 + # GPTQ style
81 + *(TypeConfig(act_type=a_type,
82 + weight_type=w_type,
83 + output_type=None,
84 + group_scale_type=a_type,
85 + group_zero_type=None,
86 + channel_scale_type=None,
87 + token_scale_type=None)
88 + for w_type in [scalar_types.uint4b8, scalar_types.uint8b128]
89 + for a_type in [torch.float16, torch.bfloat16]),
90 + # AWQ style
91 + *(TypeConfig(act_type=a_type,
92 + weight_type=w_type,
93 + output_type=None,
94 + group_scale_type=a_type,
95 + group_zero_type=a_type,
96 + channel_scale_type=None,
97 + token_scale_type=None)
98 + for w_type in [scalar_types.uint4, scalar_types.uint8]
99 + for a_type in [torch.float16, torch.bfloat16]),
100 + # QQQ style
101 + *(TypeConfig(act_type=torch.int8,
102 + weight_type=scalar_types.uint4b8,
103 + output_type=torch.float16,
104 + group_scale_type=group_scale_type,
105 + group_zero_type=None,
106 + channel_scale_type=torch.float,
107 + token_scale_type=torch.float)
108 + for group_scale_type in [None, torch.float16]),
109 + *(TypeConfig(act_type=torch.float8_e4m3fn,
110 + weight_type=scalar_types.uint4b8,
111 + output_type=torch.float16,
112 + group_scale_type=group_scale_type,
113 + group_zero_type=None,
114 + channel_scale_type=torch.float,
115 + token_scale_type=torch.float)
116 + for group_scale_type in [None, torch.float16]),
117 +]
118 +
119 +# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
120 +# unit tests to a common utility function. Currently the use of
121 +# `is_quant_method_supported` conflates kernels with quantization methods
122 +# an assumption which is breaking down as quantizations methods can have
123 +# have kernels and some kernels support multiple quantization methods.
124 +IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
125 +
126 +
127 +def rand_data(shape, dtype=torch.float16, scale=1, offset=0):
128 + if dtype.is_floating_point:
129 + return (scale * torch.rand(shape, device="cuda") - offset).to(dtype)
130 + else:
131 + return torch.randint(-8, 7, shape, dtype=dtype, device="cuda")
132 +
133 +
134 +def maybe_convert_zeropoints(zps: Optional[torch.Tensor], s: torch.Tensor):
135 + return zps if zps is None else -1 * s * (zps.to(s.dtype))
136 +
137 +
138 +def group_size_valid(shape: Tuple[int, int, int],
139 + group_size: Optional[int]) -> bool:
140 + return group_size is None or group_size == -1 or group_size % shape[2] == 0
141 +
142 +
143 +def machete_quantize_and_pack(atype: torch.dtype,
144 + w: torch.Tensor,
145 + wtype: ScalarType,
146 + stype: Optional[torch.dtype],
147 + group_size: Optional[int],
148 + zero_points: bool = False):
149 + assert wtype.is_integer(), "TODO: support floating point weights"
150 +
151 + w_ref, w_q, w_s, w_zp = quantize_weights(
152 + w,
153 + wtype,
154 + group_size=group_size,
155 + zero_points=zero_points,
156 + # to match how the kernel applies zps
157 + ref_zero_points_after_scales=True)
158 +
159 + w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
160 + w_q = w_q.t().contiguous().t() # convert to col major
161 +
162 + w_q_machete = ops.machete_prepack_B(w_q, atype, wtype, stype)
163 + opcheck(torch.ops._C.machete_prepack_B, (w_q, atype, wtype.id, stype))
164 +
165 + return w_ref, w_q_machete, w_s, w_zp
166 +
167 +
168 +def create_test_tensors(shape: Tuple[int, int, int],
169 + types: TypeConfig,
170 + group_size: Optional[int],
171 + subset_stride_factor: Optional[int] = None) -> Tensors:
172 + m, n, k = shape
173 + factor = subset_stride_factor or 1
174 +
175 + print("create_test_tensors, shape:", shape, "types:", types, "group_size:",
176 + group_size)
177 +
178 + a = rand_data((m * factor, k * factor), types.act_type, scale=3, offset=2)
179 + w = rand_data((k * factor, n * factor), types.act_type, scale=3, offset=1)
180 +
181 + if factor > 1:
182 + a = a[0:m, 0:k]
183 + w = w[0:k, 0:n]
184 +
185 + if types.group_scale_type is not None:
186 + w = w.to(types.group_scale_type)
187 + if w.dtype.itemsize == 1:
188 + w = w.to(torch.float16)
189 +
190 + w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
191 + a.dtype, w, types.weight_type, types.group_scale_type, group_size,
192 + types.group_zero_type is not None)
193 +
194 + if not a.dtype.is_floating_point:
195 + aiinfo = torch.iinfo(a.dtype)
196 + w_ref = w_ref.round().clamp(aiinfo.min, aiinfo.max)
197 +
198 + a_ref = a.to(torch.float32)
199 + w_ref = w_ref.to(torch.float32)
200 +
201 + w_ch_s = None if types.channel_scale_type is None else\
202 + rand_data((n,), types.channel_scale_type)
203 + w_tok_s = None if types.token_scale_type is None else\
204 + rand_data((m,), types.token_scale_type)
205 +
206 + return Tensors(w_ref=w_ref,
207 + a_ref=a_ref,
208 + a=a,
209 + w_q=w_q_packed,
210 + w_g_s=w_s,
211 + w_g_zp=maybe_convert_zeropoints(w_zp, w_s),
212 + w_ch_s=w_ch_s,
213 + w_tok_s=w_tok_s)
214 +
215 +
216 +# None stype means scales use the same dtype as a
217 +def machete_mm_test_helper(types: TypeConfig,
218 + tensors: Tensors,
219 + group_size: Optional[int] = None,
220 + schedule: Optional[str] = None):
221 + output_ref = torch.matmul(tensors.a_ref, tensors.w_ref)
222 + output_ref_type = output_ref.dtype
223 +
224 + if tensors.w_ch_s is not None:
225 + output_ref = (output_ref.to(tensors.w_ch_s.dtype) *
226 + tensors.w_ch_s.unsqueeze(0)).to(output_ref_type)
227 + if tensors.w_tok_s is not None:
228 + output_ref = (output_ref.to(tensors.w_tok_s.dtype) *
229 + tensors.w_tok_s.unsqueeze(1)).to(output_ref_type)
230 +
231 + output = ops.machete_mm(
232 + a=tensors.a,
233 + b_q=tensors.w_q,
234 + b_type=types.weight_type,
235 + b_group_scales=tensors.w_g_s,
236 + b_group_zeros=tensors.w_g_zp,
237 + b_group_size=group_size,
238 + b_channel_scales=tensors.w_ch_s,
239 + a_token_scales=tensors.w_tok_s,
240 + out_type=types.output_type,
241 + schedule=schedule,
242 + )
243 +
244 + print(output)
245 + print(output_ref)
246 +
247 + # Relax atol as our reduction dim becomes larger (more rounding error)
248 + # Relax atol when we have zeropoints since the way machete applies
249 + # zeropoints (after scales) causes noise around 0
250 + atol = 1 if tensors.w_g_zp is not None\
251 + else min(5e-2 * math.sqrt(tensors.a.shape[1]), 1)
252 + rtol = 1e-1 if tensors.a.element_size() >= 2 else 2e-1
253 + torch.testing.assert_close(output,
254 + output_ref.to(output.dtype),
255 + rtol=rtol,
256 + atol=atol)
257 +
258 +
259 +@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
260 + reason="Machete is not supported on this GPU type.")
261 +@pytest.mark.parametrize("shape",
262 + MNK_SHAPES,
263 + ids=lambda x: "x".join(str(v) for v in x))
264 +@pytest.mark.parametrize("types", TEST_TYPES)
265 +def test_machete_all_schedules(shape, types: TypeConfig):
266 +
267 + group_sizes: List[Optional[int]] = []
268 + if types.group_scale_type is None:
269 + group_sizes = [None]
270 + else:
271 + group_sizes = GROUP_SIZES_TO_TEST
272 +
273 + for group_size in group_sizes:
274 + if not group_size_valid(shape, group_size):
275 + continue
276 +
277 + tensors = create_test_tensors(shape, types, group_size)
278 + print(f"MNK = {shape}")
279 + for schedule in ops.machete_supported_schedules(
280 + types.act_type,
281 + types.weight_type,
282 + group_scales_type=types.group_scale_type,
283 + group_zeros_type=types.group_scale_type,
284 + out_type=types.output_type):
285 + print(f"Testing schedule {schedule}")
286 + machete_mm_test_helper(types, tensors, group_size, schedule)
287 +
288 +
289 +@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
290 + reason="Machete is not supported on this GPU type.")
291 +@pytest.mark.parametrize("shape",
292 + MNK_SHAPES,
293 + ids=lambda x: "x".join(str(v) for v in x))
294 +@pytest.mark.parametrize("types", TEST_TYPES)
295 +def test_machete_heuristic(shape, types: TypeConfig):
296 + group_sizes: List[Optional[int]] = []
297 + if types.group_scale_type is None:
298 + group_sizes = [None]
299 + else:
300 + group_sizes = GROUP_SIZES_TO_TEST
301 +
302 + for group_size in group_sizes:
303 + if not group_size_valid(shape, group_size):
304 + continue
305 +
306 + tensors = create_test_tensors(shape, types, group_size)
307 + machete_mm_test_helper(types, tensors, group_size)
308 +
309 +
310 +# Test working on other devices
311 +@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
312 + reason="Machete is not supported on this GPU type.")
313 +@pytest.mark.parametrize("device", CUDA_DEVICES)
314 +def test_machete_devices(device: str):
315 + group_size = 128
316 +
317 + type_config = TypeConfig(act_type=torch.float16,
318 + weight_type=scalar_types.uint4b8,
319 + output_type=None,
320 + group_scale_type=torch.float16,
321 + group_zero_type=None,
322 + channel_scale_type=None,
323 + token_scale_type=None)
324 +
325 + tensors = create_test_tensors((512, 4096, 4096), type_config, group_size)
326 +
327 + for field in fields(Tensors):
328 + tensor = getattr(tensors, field.name)
329 + if isinstance(tensor, torch.Tensor):
330 + setattr(tensors, field.name, tensor.to(device))
331 +
332 + machete_mm_test_helper(type_config, tensors, group_size)
333 +
334 +
335 +# Test working with a subset of A and B
336 +@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
337 + reason="Machete is not supported on this GPU type.")
338 +def test_machete_subset():
339 + group_size = 128
340 +
341 + type_config = TypeConfig(act_type=torch.float16,
342 + weight_type=scalar_types.uint4b8,
343 + output_type=None,
344 + group_scale_type=torch.float16,
345 + group_zero_type=None,
346 + channel_scale_type=None,
347 + token_scale_type=None)
348 +
349 + tensors = create_test_tensors((512, 4096, 4096),
350 + type_config,
351 + group_size,
352 + subset_stride_factor=2)
353 + machete_mm_test_helper(type_config, tensors, group_size)
354 +
355 +
356 +# Test to make sure cuda graphs work
357 +class MacheteLayer(torch.nn.Module):
358 +
359 + def __init__(self, **kwargs):
360 + super().__init__()
361 + self.kwargs = kwargs
362 +
363 + def forward(self, a):
364 + return ops.machete_mm(a=a, **self.kwargs)
365 +
366 +
367 +@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
368 + reason="Machete is not supported on this GPU type.")
369 +def test_machete_cuda_graph():
370 + m, n, k = 512, 4096, 4096
371 +
372 + a = rand_data((m, k), torch.float16)
373 + b = rand_data((k, n), torch.float16)
374 + wtype = scalar_types.uint4b8
375 + stype = torch.float16
376 + group_size = 128
377 + zero_points = False
378 +
379 + w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
380 + a.dtype, b, wtype, stype, group_size, zero_points)
381 +
382 + # Construct a trivial model with a single layer that calls a machete kernel
383 + model = MacheteLayer(
384 + b_q=w_q_packed,
385 + b_type=wtype,
386 + b_group_scales=w_s,
387 + b_group_zeros=maybe_convert_zeropoints(w_zp, w_s),
388 + b_group_size=group_size,
389 + )
390 +
391 + output_ref = torch.matmul(a, w_ref)
392 +
393 + # Run the model with a cuda graph
394 + stream = torch.cuda.Stream()
395 + with torch.cuda.stream(stream):
396 + g = torch.cuda.CUDAGraph()
397 + with torch.cuda.graph(g):
398 + output = model(a)
399 + output.zero_()
400 + g.replay()
401 +
402 + # Relax atol as our reduction dim becomes larger (more rounding error)
403 + # Relax atol when we have zeropoints since the way machete applies
404 + # zeropoints (after scales) causes noise around 0
405 + atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
406 + torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
vllm/_custom_ops.py
+44 -31
@@ -444,18 +444,18 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
444 size_k: torch.SymInt) -> torch.Tensor:
445 return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
446
447 - @register_fake("_C::machete_gemm")
448 - def machete_gemm_fake(
447 + @register_fake("_C::machete_mm")
448 + def machete_mm_fake(
449 a: torch.Tensor,
450 - # Should be the tensor returned by machete_prepack_B
450 + # b_q Should be the tensor returned by machete_prepack_B
451 b_q: torch.Tensor,
452 b_type: ScalarType,
453 - b_scales: Optional[torch.Tensor] = None,
454 - b_zeros: Optional[torch.Tensor] = None,
453 + out_type: Optional[torch.dtype] = None,
454 + b_group_scales: Optional[torch.Tensor] = None,
455 + b_group_zeros: Optional[torch.Tensor] = None,
456 b_group_size: Optional[int] = None,
456 - c: Optional[torch.Tensor] = None,
457 - alpha: Optional[float] = None,
458 - beta: Optional[float] = None,
457 + b_channel_scales: Optional[torch.Tensor] = None,
458 + a_token_scales: Optional[torch.Tensor] = None,
459 schedule: Optional[str] = None,
460 ) -> torch.Tensor:
461 m = a.size(0)
@@ -463,8 +463,9 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
463 return torch.empty((m, n), device=a.device, dtype=a.dtype)
464
465 @register_fake("_C::machete_prepack_B")
466 - def machete_prepack_B_fake(b_q_weight: torch.Tensor,
467 - b_type: ScalarType) -> torch.Tensor:
466 + def machete_prepack_B_fake(
467 + b_q_weight: torch.Tensor, a_type: torch.dtype, b_type: ScalarType,
468 + group_scales_type: Optional[torch.dtype]) -> torch.Tensor:
469 return torch.empty_like(b_q_weight,
470 memory_format=torch.contiguous_format)
471
@@ -617,29 +618,41 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
618
619
620 # machete
620 -def machete_supported_schedules(b_type: ScalarType) -> List[str]:
621 - return torch.ops._C.machete_supported_schedules(b_type.id)
622 -
623 -
624 -def machete_gemm(
625 - a: torch.Tensor,
626 - b_q: torch.Tensor, # Should be the tensor returned by machete_prepack_B
627 - b_type: ScalarType,
628 - b_scales: Optional[torch.Tensor] = None,
629 - b_zeros: Optional[torch.Tensor] = None,
630 - b_group_size: Optional[int] = None,
631 - c: Optional[torch.Tensor] = None,
632 - alpha: Optional[float] = None,
633 - beta: Optional[float] = None,
634 - schedule: Optional[str] = None,
635 -) -> torch.Tensor:
636 - return torch.ops._C.machete_gemm(a, b_q, b_type.id, b_scales, b_zeros,
637 - b_group_size, c, alpha, beta, schedule)
621 +def machete_supported_schedules(
622 + a_type: torch.dtype,
623 + b_type: ScalarType,
624 + group_scales_type: Optional[torch.dtype],
625 + group_zeros_type: Optional[torch.dtype] = None,
626 + channel_scales_type: Optional[torch.dtype] = None,
627 + token_scales_type: Optional[torch.dtype] = None,
628 + out_type: Optional[torch.dtype] = None) -> List[str]:
629 + return torch.ops._C.machete_supported_schedules(
630 + a_type, b_type.id, group_scales_type, group_zeros_type,
631 + channel_scales_type, token_scales_type, out_type)
632
633
640 -def machete_prepack_B(b_q_weight: torch.Tensor,
641 - b_type: ScalarType) -> torch.Tensor:
642 - return torch.ops._C.machete_prepack_B(b_q_weight, b_type.id)
634 +def machete_mm(
635 + a: torch.Tensor,
636 + # b_q Should be the tensor returned by machete_prepack_B
637 + b_q: torch.Tensor,
638 + b_type: ScalarType,
639 + out_type: Optional[torch.dtype] = None,
640 + b_group_scales: Optional[torch.Tensor] = None,
641 + b_group_zeros: Optional[torch.Tensor] = None,
642 + b_group_size: Optional[int] = None,
643 + b_channel_scales: Optional[torch.Tensor] = None,
644 + a_token_scales: Optional[torch.Tensor] = None,
645 + schedule: Optional[str] = None) -> torch.Tensor:
646 + return torch.ops._C.machete_mm(a, b_q, b_type.id, out_type, b_group_scales,
647 + b_group_zeros, b_group_size,
648 + b_channel_scales, a_token_scales, schedule)
649 +
650 +
651 +def machete_prepack_B(
652 + b_q_weight: torch.Tensor, a_type: torch.dtype, b_type: ScalarType,
653 + group_scales_type: Optional[torch.dtype]) -> torch.Tensor:
654 + return torch.ops._C.machete_prepack_B(b_q_weight, a_type, b_type.id,
655 + group_scales_type)
656
657
658 if hasattr(torch.ops._C, "permute_cols"):
vllm/model_executor/layers/quantization/kernels/machete.py
+9 -7
@@ -79,7 +79,9 @@ class MacheteLinearKernel(MPLinearKernel):
79 c.weight_type,
80 packed_dim=0)
81 x.data = ops.machete_prepack_B(x.data.t().contiguous().t(),
82 - self.config.weight_type)
82 + a_type=c.act_type,
83 + b_type=c.weight_type,
84 + group_scales_type=c.act_type)
85 return x
86
87 def transform_w_s(x):
@@ -105,12 +107,12 @@ class MacheteLinearKernel(MPLinearKernel):
107 if c.has_g_idx:
108 x_2d = self.act_perm(x_2d)
109
108 - output = ops.machete_gemm(a=x_2d,
109 - b_q=w_q,
110 - b_type=c.weight_type,
111 - b_zeros=None,
112 - b_scales=w_s,
113 - b_group_size=c.group_size)
110 + output = ops.machete_mm(a=x_2d,
111 + b_q=w_q,
112 + b_type=c.weight_type,
113 + b_group_zeros=None,
114 + b_group_scales=w_s,
115 + b_group_size=c.group_size)
116
117 if bias is not None:
118 output.add_(bias) # In-place add
vllm/model_executor/layers/quantization/utils/quant_utils.py
+24 -21
@@ -126,11 +126,14 @@ def permute_rows(q_w: torch.Tensor,
126
127 def quantize_weights(w: torch.Tensor,
128 quant_type: ScalarType,
129 - group_size: int,
129 + group_size: Optional[int],
130 zero_points: bool = False,
131 ref_zero_points_after_scales: bool = False):
132 assert quant_type.is_integer(), \
133 "Floating point quantization may work but has not been tested"
134 + assert not zero_points or group_size is not None, \
135 + "to have group zero points, group_size must be provided "\
136 + "(-1 group_size is channelwise)"
137
138 orig_device = w.device
139 orig_type = w.dtype
@@ -140,10 +143,9 @@ def quantize_weights(w: torch.Tensor,
143
144 if group_size == -1:
145 group_size = size_k
143 - assert group_size <= size_k
146
147 # Reshape to [groupsize, -1]
146 - if group_size < size_k:
148 + if group_size is not None and group_size < size_k:
149 w = w.reshape((-1, group_size, size_n))
150 w = w.permute(1, 0, 2)
151 w = w.reshape((group_size, -1))
@@ -155,18 +157,20 @@ def quantize_weights(w: torch.Tensor,
157 max_q_val = quant_type.max()
158 min_q_val = quant_type.min()
159
158 - if zero_points:
159 - assert not quant_type.is_signed() and quant_type.max() > 0
160 - w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
161 - maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
162 - .clamp(min_q_val, max_q_val).int()
163 - else:
164 - # If the bias is such that there are no possible negative/positive
165 - # values, set the max value to inf to avoid divide by 0
166 - w_s = torch.max(
167 - abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
168 - abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
169 - maybe_w_zp = None
160 + w_s = torch.Tensor([1.0]).to(w.device) # unscaled case
161 + maybe_w_zp = None
162 + if group_size is not None:
163 + if zero_points:
164 + assert not quant_type.is_signed() and quant_type.max() > 0
165 + w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
166 + maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
167 + .clamp(min_q_val, max_q_val).int()
168 + else:
169 + # If the bias is such that there are no possible negative/positive
170 + # values, set the max value to inf to avoid divide by 0
171 + w_s = torch.max(
172 + abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
173 + abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
174
175 # Quantize
176 w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0)
@@ -176,7 +180,7 @@ def quantize_weights(w: torch.Tensor,
180 # For some kernels (namely Machete) the zero-points are applied after the
181 # scales are applied, for this case computing the reference in similar way
182 # allows us to use tighter error tolerances in our unit tests.
179 - if ref_zero_points_after_scales and zero_points:
183 + if ref_zero_points_after_scales and maybe_w_zp is not None:
184 w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s
185 else:
186 w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
@@ -185,7 +189,7 @@ def quantize_weights(w: torch.Tensor,
189 w_q += quant_type.bias
190
191 # Restore original shapes
188 - if group_size < size_k:
192 + if group_size is not None and group_size < size_k:
193
194 def reshape_w(w):
195 w = w.reshape((group_size, -1, size_n))
@@ -195,17 +199,16 @@ def quantize_weights(w: torch.Tensor,
199
200 w_q = reshape_w(w_q)
201 w_ref = reshape_w(w_ref)
202 + w_s = w_s.reshape((-1, size_n)).contiguous()
203
199 - w_s = w_s.reshape((-1, size_n)).contiguous()
200 -
201 - if zero_points:
204 + if maybe_w_zp is not None:
205 maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous()
206 maybe_w_zp = maybe_w_zp.to(device=orig_device)
207
208 return (
209 w_ref.to(device=orig_device),
210 w_q.to(device=orig_device),
208 - w_s.to(device=orig_device),
211 + w_s if group_size is not None else None,
212 maybe_w_zp,
213 )
214