A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Kernel] Initial Machete W4A8 support + Refactors (#9855)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Nov 18, 2024 at 14:59 UTC
96d999fbe8d610fa4c5b7cad6bb0d0158d1d5b8b
28 files changed
+2616
-1694
benchmarks/kernels/benchmark_machete.py
+385
-134
@@ -2,8 +2,10 @@ import argparse
2
import copy
3
import itertools
4
import math
5
+import os
6
import pickle as pkl
7
import time
8
+from dataclasses import dataclass
9
from itertools import product
10
from typing import Callable, Iterable, List, Optional, Tuple
11
@@ -15,11 +17,12 @@ from weight_shapes import WEIGHT_SHAPES
17
18
from vllm import _custom_ops as ops
19
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, marlin_permute_scales)
20
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, marlin_permute_scales,
21
+ marlin_zero_points)
22
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
23
MarlinWorkspace)
24
from vllm.model_executor.layers.quantization.utils.quant_utils import (
22
- gptq_pack, pack_rows, quantize_weights)
25
+ pack_rows, quantize_weights)
26
from vllm.scalar_type import ScalarType, scalar_types
27
from vllm.utils import FlexibleArgumentParser
28
@@ -27,149 +30,349 @@ DEFAULT_MODELS = ["meta-llama/Llama-3-8b", "meta-llama/Llama-2-70b-hf"]
30
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512, 1024]
31
DEFAULT_TP_SIZES = [1]
32
33
+NVTX_PROFILE = os.environ.get("NVTX_PROFILE", False)
34
+
35
+if NVTX_PROFILE:
36
+ import nvtx
37
+
38
+
39
+def terse_type_name(dt):
40
+ return {
41
+ torch.bfloat16: "bf16",
42
+ torch.float16: "fp16",
43
+ torch.int8: "int8",
44
+ torch.float8_e4m3fn: "fp8",
45
+ torch.bfloat16: "bf16",
46
+ torch.float: "float",
47
+ torch.int: "int",
48
+ }[dt]
49
+
50
+
51
+@dataclass
52
+class BenchmarkTensors:
53
+ w_ref: torch.Tensor
54
+ a: torch.Tensor
55
+
56
+ w_q: torch.Tensor
57
+ group_size: Optional[int]
58
+ wtype: ScalarType
59
+ w_g_s: torch.Tensor
60
+ w_g_zp: Optional[torch.Tensor]
61
+ w_ch_s: Optional[torch.Tensor]
62
+ w_tok_s: Optional[torch.Tensor]
63
+
64
+
65
+@dataclass
66
+class TypeConfig:
67
+ act_type: torch.dtype
68
+ weight_type: ScalarType
69
+ output_type: Optional[torch.dtype]
70
+ group_scale_type: Optional[torch.dtype]
71
+ group_zero_type: Optional[torch.dtype]
72
+ channel_scale_type: Optional[torch.dtype]
73
+ token_scale_type: Optional[torch.dtype]
74
+
75
+
76
+def rand_data(shape, dtype=torch.float16, scale=1):
77
+ if dtype.is_floating_point:
78
+ return (scale * torch.rand(shape, device="cuda") - 0.3).to(dtype)
79
+ else:
80
+ return torch.randint(-15, 15, shape, dtype=dtype, device="cuda")
81
+
82
+
83
+def quantize_and_pack(atype: torch.dtype,
84
+ w: torch.Tensor,
85
+ wtype: ScalarType,
86
+ stype: Optional[torch.dtype],
87
+ group_size: Optional[int],
88
+ zero_points: bool = False):
89
+ assert wtype.is_integer(), "TODO: support floating point weights"
90
+
91
+ w_ref, w_q, w_s, w_zp = quantize_weights(
92
+ w,
93
+ wtype,
94
+ group_size=group_size,
95
+ zero_points=zero_points,
96
+ # to match how the kernel applies zps
97
+ ref_zero_points_after_scales=True)
98
31
-def machete_pack_weights(w_q: torch.tensor, wtype: ScalarType) -> torch.tensor:
99
w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
33
- w_q = w_q.t().contiguous().t() # make col major
34
- return ops.machete_prepack_B(w_q, wtype)
100
+ return w_ref, w_q, w_s, w_zp
101
102
37
-def make_bench_tensors(
38
- atype: torch.dtype, wtype: ScalarType, group_size: int, m: int, n: int,
39
- k: int
40
-) -> Tuple[torch.tensor, List[Tuple[torch.tensor, torch.tensor, torch.tensor,
41
- torch.tensor]]]:
42
- assert wtype.is_integer(), "TODO: support floating point weights"
103
+def create_bench_tensors(shape: Tuple[int, int, int], types: TypeConfig,
104
+ group_size: Optional[int]) -> List[BenchmarkTensors]:
105
+ m, n, k = shape
106
107
# we want to make sure that weights don't fit into L2 cache between runs so
108
# we construct enough weights to exceed L2 cache, which is 50mb on a H100
109
# so we target total weight size > 2*50mb
47
- num_weights = math.ceil(2 * 50 * 1024**2 * 8 / (k * n * wtype.size_bits))
48
-
49
- a = torch.randn((m, k), device="cuda", dtype=atype) * 5
50
- weights = [
51
- torch.randn((k, n), device="cuda", dtype=atype)
52
- for _ in range(num_weights)
53
- ]
54
- quanitized_weights = [
55
- quantize_weights(w, wtype, group_size) for w in weights
56
- ]
57
-
58
- return a, quanitized_weights
110
+ num_weights = math.ceil(2 * 50 * 1024**2 * 8 /
111
+ (k * n * types.weight_type.size_bits))
112
+
113
+ a = rand_data((m, k), types.act_type, scale=5)
114
+
115
+ benchmark_tensors: List[BenchmarkTensors] = []
116
+ for _ in range(num_weights):
117
+ w = rand_data((k, n), types.act_type, scale=5)
118
+
119
+ if types.group_scale_type is not None:
120
+ w = w.to(types.group_scale_type)
121
+ if w.dtype.itemsize == 1:
122
+ w = w.to(torch.float16)
123
+
124
+ w_ref, w_q_packed, w_s, w_zp = quantize_and_pack(
125
+ a.dtype, w, types.weight_type, types.group_scale_type, group_size,
126
+ types.group_zero_type is not None)
127
+
128
+ if not a.dtype.is_floating_point:
129
+ aiinfo = torch.iinfo(a.dtype)
130
+ w_ref = w_ref.round().clamp(aiinfo.min, aiinfo.max)
131
+
132
+ w_ref = w_ref.to(torch.float32)
133
+
134
+ w_ch_s = None if types.channel_scale_type is None else\
135
+ rand_data((n,), types.channel_scale_type)
136
+ w_tok_s = None if types.token_scale_type is None else\
137
+ rand_data((m,), types.token_scale_type)
138
+
139
+ benchmark_tensors.append(
140
+ BenchmarkTensors(w_ref=w_ref,
141
+ a=a,
142
+ w_q=w_q_packed,
143
+ wtype=types.weight_type,
144
+ w_g_s=w_s,
145
+ w_g_zp=w_zp,
146
+ group_size=group_size,
147
+ w_ch_s=w_ch_s,
148
+ w_tok_s=w_tok_s))
149
+
150
+ return benchmark_tensors
151
+
152
+
153
+def torch_matmul_f16_create_bench_fn(bt: BenchmarkTensors) -> Callable:
154
+ a = bt.a
155
+ w = bt.w_ref.to(bt.a.dtype) # use float reference tensor
156
+ if a.dtype not in [torch.float16, torch.bfloat16]:
157
+ a = a.to(torch.float16)
158
+ w = w.to(torch.float16)
159
+ return lambda: torch.matmul(a, w)
160
+
161
+
162
+def cutlass_scaled_mm_create_bench_fn(bt: BenchmarkTensors) -> Callable:
163
+ if bt.w_ch_s is not None and bt.w_tok_s is not None:
164
+ scale_a = bt.w_tok_s.to(torch.float32)
165
+ scale_b = bt.w_ch_s.to(torch.float32)
166
+ else:
167
+ scale_a = torch.tensor(1.0, dtype=torch.float32, device=bt.a.device)
168
+ scale_b = torch.tensor(1.0, dtype=torch.float32, device=bt.a.device)
169
+ w_col_major = bt.w_ref.to(bt.a.dtype).t().contiguous().t()
170
+ return lambda: ops.cutlass_scaled_mm(
171
+ bt.a, w_col_major, scale_a, scale_b, out_dtype=torch.float16)
172
+
173
+
174
+def marlin_create_bench_fn(bt: BenchmarkTensors) -> Callable:
175
+ device = bt.a.device
176
+
177
+ workspace = MarlinWorkspace(bt.w_ref.shape[1], GPTQ_MARLIN_MIN_THREAD_N,
178
+ GPTQ_MARLIN_MAX_PARALLEL)
179
+
180
+ if bt.w_g_zp is None:
181
+ w_zp = torch.empty(0, dtype=torch.int, device=device)
182
+ else:
183
+ w_zp = marlin_zero_points(bt.w_g_zp, bt.w_ref.shape[0],
184
+ bt.w_ref.shape[1], bt.wtype.size_bits)
185
+
186
+ if bt.group_size is None:
187
+ w_s = torch.tensor([], device="cuda", dtype=torch.half)
188
+ else:
189
+ w_s = marlin_permute_scales(bt.w_g_s, bt.w_ref.shape[0],
190
+ bt.w_ref.shape[1], bt.group_size)
191
+
192
+ sort_indices = torch.empty(0, dtype=torch.int, device=device)
193
+ g_idx = torch.empty(0, dtype=torch.int, device=device)
194
+ w_q = ops.gptq_marlin_repack(bt.w_q, sort_indices, bt.w_ref.shape[0],
195
+ bt.w_ref.shape[1], bt.wtype.size_bits)
196
+
197
+ if bt.a.dtype.is_floating_point:
198
+ assert bt.w_ch_s is None
199
+ assert bt.w_tok_s is None
200
+ assert bt.group_size is not None
201
+
202
+ fn = lambda: ops.gptq_marlin_gemm(a=bt.a,
203
+ b_q_weight=w_q,
204
+ b_scales=w_s,
205
+ b_zeros=w_zp,
206
+ g_idx=g_idx,
207
+ perm=sort_indices,
208
+ workspace=workspace.scratch,
209
+ b_q_type=bt.wtype,
210
+ size_m=bt.a.shape[0],
211
+ size_n=bt.w_ref.shape[1],
212
+ size_k=bt.w_ref.shape[0],
213
+ is_k_full=True)
214
+ else:
215
+ assert bt.a.dtype == torch.int8
216
+ assert bt.wtype == scalar_types.uint4b8
217
+
218
+ if bt.w_ch_s is not None:
219
+ s_ch = bt.w_ch_s.to(torch.float32)
220
+ else:
221
+ s_ch = torch.ones(bt.w_ref.shape[1],
222
+ dtype=torch.float32,
223
+ device=device)
224
+
225
+ if bt.w_tok_s is not None:
226
+ s_tok = bt.w_tok_s.to(torch.float32)
227
+ else:
228
+ s_tok = torch.ones(bt.a.shape[0],
229
+ dtype=torch.float32,
230
+ device=device)
231
+
232
+ fn = lambda: ops.marlin_qqq_gemm(a=bt.a,
233
+ b_q_weight=w_q,
234
+ s_group=w_s,
235
+ s_tok=s_tok,
236
+ s_ch=s_ch,
237
+ workspace=workspace.scratch,
238
+ size_m=bt.a.shape[0],
239
+ size_n=bt.w_ref.shape[1],
240
+ size_k=bt.w_ref.shape[0])
241
+
242
+ return fn
243
+
244
+
245
+def machete_create_bench_fn(bt: BenchmarkTensors,
246
+ out_type=torch.dtype,
247
+ schedule=None) -> Callable:
248
+ w_q = bt.w_q.t().contiguous().t() # make col major
249
+ w_q = ops.machete_prepack_B(w_q, bt.a.dtype, bt.wtype,
250
+ None if bt.w_g_s is None else bt.w_g_s.dtype)
251
+
252
+ w_g_zp = bt.w_g_zp
253
+ if w_g_zp is not None:
254
+ w_g_zp = -1 * bt.w_g_s * (w_g_zp.to(bt.w_g_s.dtype))
255
+
256
+ return lambda: ops.machete_mm(
257
+ a=bt.a,
258
+ b_q=bt.w_q,
259
+ b_type=bt.wtype,
260
+ b_group_scales=bt.w_g_s,
261
+ b_group_zeros=w_g_zp,
262
+ b_group_size=bt.group_size,
263
+ b_channel_scales=bt.w_ch_s,
264
+ a_token_scales=bt.w_tok_s,
265
+ out_type=out_type,
266
+ schedule=schedule,
267
+ )
268
269
270
# impl
271
63
-
272
# bench
65
-def bench_fn(label: str, sub_label: str, description: str,
66
- fn: Callable) -> TMeasurement:
273
68
- min_run_time = 1
69
- return TBenchmark.Timer(
70
- stmt="fn()",
274
+
275
+def bench_fns(label: str, sub_label: str, description: str,
276
+ fns: List[Callable]):
277
+
278
+ min_run_time = 1 if not NVTX_PROFILE else 0.1
279
+ res = TBenchmark.Timer(
280
+ stmt="""
281
+ for fn in fns:
282
+ fn()
283
+ """,
284
globals={
72
- "fn": fn
285
+ "fns": fns
286
},
287
label=label,
288
sub_label=sub_label,
289
description=description,
290
).blocked_autorange(min_run_time=min_run_time)
291
292
+ if NVTX_PROFILE:
293
+ with nvtx.annotate("mm-bench"), nvtx.annotate(
294
+ f"{label}|{sub_label}|{description}"):
295
+ fns[0]()
296
80
-def loop_over_weights(
81
- a: torch.tensor, weights: List[Tuple[torch.tensor, torch.tensor,
82
- torch.tensor, torch.tensor]],
83
- fn: Callable[[torch.tensor, torch.tensor, torch.tensor, torch.tensor],
84
- None]):
85
- for w_ref, w_q, w_s, _ in weights:
86
- fn(a, w_ref, w_q, w_s)
297
+ return res
298
299
300
_SWEEP_SCHEDULES_RESULTS: Optional[pd.DataFrame] = None
301
_SWEEP_SCHEDULES_RESULTS_CSV: Optional[str] = None
302
303
93
-def bench(atype: torch.dtype,
94
- wtype: ScalarType,
304
+def bench(types: TypeConfig,
305
group_size: int,
306
m: int,
307
k: int,
308
n: int,
309
label: str,
310
sub_label: str,
101
- benchmark_marlinv1: bool = True,
102
- sweep_schedules: bool = True) -> Iterable[TMeasurement]:
103
- global _SWEEP_SCHEDULES_RESULTS
104
-
105
- a, weights = make_bench_tensors(atype, wtype, group_size, m, n, k)
106
- sub_label += f", L={len(weights)}"
107
-
108
- weights_machete = [(w_ref, machete_pack_weights(w_q, wtype), w_s, w_zp)
109
- for w_ref, w_q, w_s, w_zp in weights]
311
+ sweep_schedules: bool = True) -> List[TMeasurement]:
312
+ benchmark_tensors = create_bench_tensors((m, n, k), types, group_size)
313
+ sub_label += f", L={len(benchmark_tensors)}"
314
+
315
+ name_type_string = f"W{types.weight_type}"+\
316
+ f"-A{terse_type_name(types.act_type)}"
317
+ if types.group_scale_type is not None:
318
+ name_type_string += f"-GS{terse_type_name(types.group_scale_type)}"
319
+ if types.group_zero_type is not None:
320
+ name_type_string += f"-GZ{terse_type_name(types.group_zero_type)}"
321
+ if group_size is not None:
322
+ name_type_string += f"-G{group_size}"
323
+ if types.channel_scale_type is not None:
324
+ name_type_string += f"-CS{terse_type_name(types.channel_scale_type)}"
325
+ if types.token_scale_type is not None:
326
+ name_type_string += f"-TS{terse_type_name(types.token_scale_type)}"
327
328
timers = []
329
# pytorch impl
330
timers.append(
114
- bench_fn(
115
- label, sub_label, "torch.matmul", lambda: loop_over_weights(
116
- a,
117
- weights,
118
- lambda a, w_ref, w_q, w_s: torch.matmul(a, w_ref),
119
- )))
331
+ bench_fns(
332
+ label, sub_label, "torch.matmul (fp16)",
333
+ [torch_matmul_f16_create_bench_fn(bt)
334
+ for bt in benchmark_tensors]))
335
121
- if benchmark_marlinv1:
122
- w_ref = weights[0][0]
123
-
124
- w_zp_empty = torch.empty(0, dtype=torch.int, device=w_ref.device)
125
- sort_indices = torch.empty(0, dtype=torch.int, device=w_ref.device)
126
- g_idx = torch.empty(0, dtype=torch.int, device=w_ref.device)
127
-
128
- def marlinv1_pack_weights(w_q: torch.tensor) -> torch.tensor:
129
- w_q_gptq = gptq_pack(w_q, wtype.size_bits, *w_ref.shape)
130
- return ops.gptq_marlin_repack(w_q_gptq, sort_indices, *w_ref.shape,
131
- wtype.size_bits)
132
-
133
- def marlinv1_permute_scales(w_s: torch.tensor) -> torch.tensor:
134
- return marlin_permute_scales(w_s, *w_ref.shape, group_size)
135
-
136
- weights_marlinv1 = [(w_ref, marlinv1_pack_weights(w_q),
137
- marlinv1_permute_scales(w_s), w_zp)
138
- for w_ref, w_q, w_s, w_zp in weights]
139
-
140
- workspace = MarlinWorkspace(w_ref.shape[1], GPTQ_MARLIN_MIN_THREAD_N,
141
- GPTQ_MARLIN_MAX_PARALLEL)
142
-
143
- # marlinv1
336
+ if types.act_type == torch.int8 or types.act_type == torch.float8_e4m3fn:
337
+ timers.append(
338
+ bench_fns(
339
+ label, sub_label,
340
+ f"cutlass_scaled_mm ({terse_type_name(types.act_type)})", [
341
+ cutlass_scaled_mm_create_bench_fn(bt)
342
+ for bt in benchmark_tensors
343
+ ]))
344
+
345
+ if types.act_type != torch.float8_e4m3fn:
346
timers.append(
145
- bench_fn(
146
- label, sub_label, "marlin_orig", lambda: loop_over_weights(
147
- a, weights_marlinv1, lambda a, w_ref, w_q, w_s: ops.
148
- gptq_marlin_gemm(a,
149
- w_q,
150
- w_s,
151
- w_zp_empty,
152
- g_idx,
153
- sort_indices,
154
- workspace.scratch,
155
- wtype,
156
- size_m=a.shape[0],
157
- size_n=w_ref.shape[1],
158
- size_k=w_ref.shape[0],
159
- is_k_full=True))))
347
+ bench_fns(label, sub_label, f"marlin ({name_type_string})",
348
+ [marlin_create_bench_fn(bt)
349
+ for bt in benchmark_tensors]))
350
351
# machete
352
timers.append(
163
- bench_fn(
164
- label, sub_label, "machete_heuristic", lambda: loop_over_weights(
165
- a, weights_machete, lambda a, _, w_q, w_s: ops.machete_gemm(
166
- a, w_q, wtype, b_scales=w_s, b_group_size=group_size))))
353
+ bench_fns(label, sub_label, f"machete ({name_type_string})", [
354
+ machete_create_bench_fn(bt, out_type=types.output_type)
355
+ for bt in benchmark_tensors
356
+ ]))
357
358
if sweep_schedules:
359
+ global _SWEEP_SCHEDULES_RESULTS
360
+
361
print("Finding best schedule for machete")
362
best = None
363
best_schedule = None
172
- schedules = ops.machete_supported_schedules(wtype)
364
+ schedules = ops.machete_supported_schedules(
365
+ a_type=types.act_type,
366
+ b_type=types.weight_type,
367
+ group_scales_type=types.group_scale_type,
368
+ group_zeros_type=types.group_zero_type,
369
+ token_scales_type=types.token_scale_type,
370
+ channel_scales_type=types.channel_scale_type,
371
+ out_type=types.output_type)
372
+
373
+ if schedules is None or len(schedules) == 0:
374
+ raise ValueError("No schedules found to sweep")
375
+
376
for schedule in reversed(schedules):
377
schedule_M = int(schedule.split("_")[0].split("x")[1])
378
@@ -177,16 +380,11 @@ def bench(atype: torch.dtype,
380
if schedule_M >= 2 * max(m, 16) or schedule_M < m // 4:
381
continue
382
180
- def run(a, _, w_q, w_s, schedule=schedule):
181
- ops.machete_gemm(a,
182
- w_q,
183
- wtype,
184
- w_s,
185
- b_group_size=group_size,
186
- schedule=schedule)
187
-
188
- res = bench_fn(label, sub_label, "machete_best",
189
- lambda: loop_over_weights(a, weights_machete, run))
383
+ res = bench_fns(label, sub_label, "machete_best", [
384
+ machete_create_bench_fn(
385
+ bt, out_type=types.output_type, schedule=schedule)
386
+ for bt in benchmark_tensors
387
+ ])
388
389
results_row = {
390
"M": m,
@@ -213,25 +411,33 @@ def bench(atype: torch.dtype,
411
412
413
# runner
216
-def print_timers(timers: Iterable[TMeasurement]):
414
+def print_timers(timers: List[TMeasurement]):
415
compare = TBenchmark.Compare(timers)
416
compare.print()
417
418
221
-def run(dtype: torch.dtype, sweep_schedules: bool,
222
- MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
419
+def run(args, MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
420
+ types = TypeConfig(
421
+ act_type=args.act_type,
422
+ weight_type=scalar_types.uint4b8 if args.group_zero_type is None \
423
+ else scalar_types.uint4,
424
+ output_type=args.out_type,
425
+ group_scale_type=args.group_scale_type,
426
+ group_zero_type=args.group_zero_type,
427
+ channel_scale_type=args.channel_scale_type,
428
+ token_scale_type=args.token_scale_type,
429
+ )
430
224
- results = []
431
+ results: List[TMeasurement] = []
432
for m, k, n in MKNs:
226
- timers = bench(dtype,
227
- scalar_types.uint4b8,
228
- 128,
433
+ timers = bench(types,
434
+ args.group_size,
435
m,
436
k,
437
n,
232
- f"{dtype}-gemm",
438
+ f"{args.act_type}-gemm",
439
f"MKN=({m}x{k}x{n})",
234
- sweep_schedules=sweep_schedules)
440
+ sweep_schedules=args.sweep_schedules)
441
print_timers(timers)
442
results.extend(timers)
443
@@ -240,7 +446,7 @@ def run(dtype: torch.dtype, sweep_schedules: bool,
446
447
# output makers
448
def make_output(
243
- data: Iterable[TMeasurement],
449
+ data: List[TMeasurement],
450
MKNs: Iterable[Tuple[int, int, int]],
451
base_description: str,
452
timestamp=None,
@@ -262,7 +468,6 @@ def run_square_bench(args):
468
dim_sizes = list(
469
range(args.dim_start, args.dim_end + 1, args.dim_increment))
470
MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
265
-
471
data = run(args.dtype, args.sweep_schedules, MKNs)
472
473
make_output(data, MKNs, f"square_bench-{args.dtype}")
@@ -306,33 +511,49 @@ def run_model_bench(args):
511
for k, n in KNs:
512
MKNs.append((m, k, n))
513
309
- data = run(args.dtype, args.sweep_schedules, MKNs)
514
+ data = run(args, MKNs)
515
model_bench_data.append(data)
516
517
+ type_string = f"{args.act_type}"
518
+
519
# Print all results
520
for data, model_tp in zip(model_bench_data, models_tps):
521
model, tp_size = model_tp
315
- print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
522
+ print(f"== Results {type_string} {model}-TP{tp_size} ====")
523
print_timers(data)
524
318
- timestamp = int(time.time())
525
+ timestr = time.strftime("%Y%m%d-%H%M%S")
526
320
- all_data = []
527
+ all_results = []
528
for d in model_bench_data:
322
- all_data.extend(d)
529
+ all_results.extend(d)
530
+
531
# pickle all data
324
- with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
325
- pkl.dump(all_data, f)
532
+ with open(f"model_bench-{type_string}-{timestr}.pkl", "wb") as f:
533
+ args_dict = vars(args)
534
+ args_dict.pop("func")
535
+ pkl.dump({
536
+ "args": args_dict,
537
+ "results": all_results,
538
+ }, f)
539
540
541
if __name__ == "__main__":
542
543
def to_torch_dtype(dt):
331
- if dt == "bfloat16":
332
- return torch.bfloat16
333
- if dt == "float16":
334
- return torch.float16
335
- raise ValueError("unsupported dtype")
544
+ return {
545
+ "bfloat16": torch.bfloat16,
546
+ "float16": torch.float16,
547
+ "int8": torch.int8,
548
+ "float8_e4m3fn": torch.float8_e4m3fn,
549
+ "int": torch.int,
550
+ "float": torch.float,
551
+ }[dt]
552
+
553
+ class ToTorchDtype(argparse.Action):
554
+
555
+ def __call__(self, parser, namespace, values, option_string=None):
556
+ setattr(namespace, self.dest, to_torch_dtype(values))
557
558
parser = FlexibleArgumentParser(
559
description="""
@@ -352,12 +573,42 @@ Benchmark Machete GEMM.
573
""", # noqa: E501
574
formatter_class=argparse.RawTextHelpFormatter,
575
)
355
-
576
parser.add_argument(
357
- "--dtype",
358
- type=to_torch_dtype,
577
+ "--act-type",
578
+ action=ToTorchDtype,
579
required=True,
360
- help="Available options are ['bfloat16', 'float16']",
580
+ choices=['bfloat16', 'float16', 'int8', 'float8_e4m3fn'],
581
+ )
582
+ parser.add_argument(
583
+ "--group-scale-type",
584
+ action=ToTorchDtype,
585
+ choices=['bfloat16', 'float16'],
586
+ )
587
+ parser.add_argument(
588
+ "--group-zero-type",
589
+ type=to_torch_dtype,
590
+ choices=['bfloat16', 'float16'],
591
+ )
592
+ parser.add_argument(
593
+ "--channel-scale-type",
594
+ action=ToTorchDtype,
595
+ choices=['float'],
596
+ )
597
+ parser.add_argument(
598
+ "--token-scale-type",
599
+ action=ToTorchDtype,
600
+ choices=['float'],
601
+ )
602
+ parser.add_argument(
603
+ "--out-type",
604
+ action=ToTorchDtype,
605
+ choices=['bfloat16', 'float16'],
606
+ )
607
+ parser.add_argument(
608
+ "--group-size",
609
+ type=int,
610
+ help="Available options are ['None', '-1', '128'], default=128",
611
+ default=128,
612
)
613
parser.add_argument(
614
"--sweep-schedules",
benchmarks/kernels/graph_machete_bench.py
+3
-2
@@ -20,10 +20,11 @@ if __name__ == "__main__":
20
args = parser.parse_args()
21
22
with open(args.filename, 'rb') as f:
23
- data: List[TMeasurement] = pickle.load(f)
23
+ data = pickle.load(f)
24
+ raw_results: List[TMeasurement] = data["results"]
25
26
results = defaultdict(lambda: list())
26
- for v in data:
27
+ for v in raw_results:
28
result = re.search(r"MKN=\(\d+x(\d+x\d+)\)", v.task_spec.sub_label)
29
if result is not None:
30
KN = result.group(1)
benchmarks/kernels/weight_shapes.py
+6
@@ -40,4 +40,10 @@ WEIGHT_SHAPES = {
40
([8192, 57344], 1),
41
([28672, 8192], 0),
42
],
43
+ "meta-llama/Llama-3.1-405b-hf": [
44
+ ([16384, 18432], 1),
45
+ ([16384, 16384], 0),
46
+ ([16384, 106496], 1),
47
+ ([53248, 16384], 0),
48
+ ],
49
}
csrc/cutlass_extensions/cute_utils.cuh
+2
-2
@@ -20,9 +20,9 @@ CUTE_HOST_DEVICE static constexpr auto permute_layout(Layout l) {
20
// is the layout f(x) = x
21
template <typename Layout>
22
CUTE_HOST_DEVICE static constexpr bool is_identity_layout() {
23
- if constexpr (std::is_same_v<Layout, void>)
23
+ if constexpr (std::is_same_v<Layout, void>) {
24
return true;
25
- else {
25
+ } else {
26
constexpr auto coalesced_layout = coalesce(Layout{});
27
if constexpr (rank(coalesced_layout) == 1 &&
28
stride<0>(coalesced_layout) == 1) {
csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c2x.hpp
renamed
+1
@@ -52,6 +52,7 @@
52
// clang-format off
53
54
#include "cutlass/epilogue/threadblock/fusion/visitor_2x.hpp"
55
+#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
56
#include "cute/tensor.hpp"
57
58
namespace cutlass::epilogue::threadblock {
csrc/cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp
renamed
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
new
+317
@@ -0,0 +1,317 @@
1
+#include "cutlass_extensions/epilogue/broadcast_load_epilogue_c2x.hpp"
2
+
3
+/*
4
+ This file defines custom epilogues for fusing channel scales, token scales,
5
+ bias, and activation zero-points onto a GEMM operation using the
6
+ CUTLASS 2.x API, for sm80 (Ampere) NVIDIA GPUs.
7
+
8
+ Epilogues must contain a public type named EVTCompute of type Sm80EVT,
9
+ as well as a static prepare_args function that constructs an
10
+ EVTCompute::Arguments struct.
11
+*/
12
+
13
+namespace vllm::c2x {
14
+
15
+using namespace cute;
16
+
17
+/*
18
+ * This class provides the common load descriptors for the
19
+ * ScaledEpilogue[...] classes
20
+ */
21
+template <typename ElementD, typename OutputTileThreadMap>
22
+struct ScaledEpilogueBase {
23
+ protected:
24
+ using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
25
+
26
+ template <typename T>
27
+ using ColOrScalarLoad =
28
+ cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
29
+ OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
30
+
31
+ template <typename T>
32
+ using RowOrScalarLoad =
33
+ cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
34
+ OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
35
+
36
+ template <typename T>
37
+ using ColLoad = cutlass::epilogue::threadblock::VisitorColBroadcast<
38
+ OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
39
+
40
+ template <typename T>
41
+ using RowLoad = cutlass::epilogue::threadblock::VisitorRowBroadcast<
42
+ OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
43
+
44
+ template <typename T>
45
+ using RowOrZeroLoad =
46
+ cutlass::epilogue::threadblock::VisitorRowOrZeroBroadcast<
47
+ OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
48
+
49
+ // This utility function constructs the arguments for the load descriptors
50
+ // from a tensor. It can handle both row and column, as well as row/column or
51
+ // scalar cases.
52
+ template <typename Descriptor, typename T>
53
+ static auto args_from_tensor(torch::Tensor const& tensor) {
54
+ using Arguments = typename Descriptor::Arguments;
55
+ auto* data_ptr = static_cast<T*>(tensor.data_ptr());
56
+ if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
57
+ std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
58
+ return Arguments{data_ptr, tensor.numel() != 1};
59
+ } else {
60
+ // it would technically work but no use case as data_ptr is never nullptr
61
+ static_assert(!std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
62
+ return Arguments{data_ptr};
63
+ }
64
+ }
65
+
66
+ // This overload handles the case where there might not be a tensor, in which
67
+ // case a nullptr is passed and a constant (0) is used.
68
+ template <typename Descriptor, typename T>
69
+ static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
70
+ static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
71
+ using Arguments = typename Descriptor::Arguments;
72
+ auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
73
+ return Arguments{data_ptr};
74
+ }
75
+};
76
+
77
+/*
78
+ This epilogue function defines a quantized GEMM operation similar to
79
+ torch._scaled_mm.
80
+
81
+ A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
82
+ per-row. B can be quantized per-tensor or per-column.
83
+ Any combination of per-tensor and per-row or column is supported.
84
+ A and B must have symmetric quantization (zero point == 0).
85
+
86
+ So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
87
+ scales are applied elementwise with numpy-style broadcasting.
88
+
89
+ ScaleA and ScaleB define the epilogue functions that apply the scales for
90
+ the A and B operands respectively. These scales may be either per-tensor or
91
+ per row or column.
92
+*/
93
+template <typename ElementD, typename OutputTileThreadMap>
94
+struct ScaledEpilogue
95
+ : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
96
+ private:
97
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
98
+ using Accum = typename SUPER::Accum;
99
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
100
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
101
+
102
+ using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
103
+ cutlass::multiplies, float, float,
104
+ cutlass::FloatRoundStyle::round_to_nearest>;
105
+
106
+ using EVTCompute0 =
107
+ cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
108
+
109
+ using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
110
+ cutlass::multiplies, ElementD, float,
111
+ cutlass::FloatRoundStyle::round_to_nearest>;
112
+
113
+ public:
114
+ using EVTCompute =
115
+ cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
116
+ using ArgumentType = typename EVTCompute::Arguments;
117
+
118
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
119
+ torch::Tensor const& b_scales) {
120
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
121
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
122
+
123
+ typename EVTCompute0::Arguments evt0_args{b_args};
124
+ return ArgumentType{a_args, evt0_args};
125
+ }
126
+};
127
+
128
+/*
129
+ * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
130
+ * This bias can also be used in the per-tensor azp case, where the activation
131
+ * zero point (azp) is used to compute an azp correction term,
132
+ * which is folded into the bias.
133
+ *
134
+ * The bias tensor must be per-output channel.
135
+ * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
136
+ */
137
+template <typename ElementD, typename OutputTileThreadMap>
138
+struct ScaledEpilogueBias
139
+ : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
140
+ protected:
141
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
142
+ using Accum = typename SUPER::Accum;
143
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
144
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
145
+ using Bias = typename SUPER::template RowLoad<ElementD>;
146
+ using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
147
+ cutlass::multiplies, float, float,
148
+ cutlass::FloatRoundStyle::round_to_nearest>;
149
+
150
+ using EVTCompute0 =
151
+ cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
152
+
153
+ using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
154
+ cutlass::multiply_add, ElementD, float,
155
+ cutlass::FloatRoundStyle::round_to_nearest>;
156
+
157
+ public:
158
+ using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
159
+ EVTCompute0, Bias>;
160
+ using ArgumentType = typename EVTCompute::Arguments;
161
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
162
+ torch::Tensor const& b_scales,
163
+ torch::Tensor const& bias) {
164
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
165
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
166
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
167
+
168
+ typename EVTCompute0::Arguments evt0_args{b_args};
169
+ return ArgumentType{a_args, evt0_args, bias_args};
170
+ }
171
+};
172
+
173
+/*
174
+ * This epilogue directly supports per-tensor azp in int32 form.
175
+ * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
176
+ * term, which should already be multiplied with the scalar azp.
177
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
178
+ *
179
+ * This epilogue also supports bias, which remains per-channel.
180
+ */
181
+template <typename ElementD, typename OutputTileThreadMap>
182
+struct ScaledEpilogueBiasAzp
183
+ : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
184
+ private:
185
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
186
+ using Accum = typename SUPER::Accum;
187
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
188
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
189
+ using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
190
+
191
+ // This is the full AZP term, azp * J @ B, shape (1,n)
192
+ using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
193
+
194
+ // Compute float(accum - azp_adj), both operands are int32_t
195
+ using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
196
+ cutlass::minus, float, int32_t,
197
+ cutlass::FloatRoundStyle::round_to_nearest>;
198
+
199
+ using EVTComputeAzp =
200
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Accum, AzpWithAdj>;
201
+
202
+ using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
203
+ cutlass::multiplies, float, float,
204
+ cutlass::FloatRoundStyle::round_to_nearest>;
205
+
206
+ using EVTComputeScaleB =
207
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
208
+ EVTComputeAzp>;
209
+
210
+ using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
211
+ cutlass::multiply_add, ElementD, float,
212
+ cutlass::FloatRoundStyle::round_to_nearest>;
213
+
214
+ public:
215
+ using EVTCompute =
216
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
217
+ EVTComputeScaleB, Bias>;
218
+
219
+ using ArgumentType = typename EVTCompute::Arguments;
220
+
221
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
222
+ torch::Tensor const& b_scales,
223
+ torch::Tensor const& azp_adj,
224
+ c10::optional<torch::Tensor> const& bias) {
225
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
226
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
227
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
228
+ auto azp_adj_args =
229
+ SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
230
+
231
+ typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
232
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
233
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
234
+ }
235
+};
236
+
237
+/*
238
+ * This epilogue supports per-token azp by computing and applying
239
+ * the correction term using a rank-1 update. If the term were materialized,
240
+ * it would require O(m*n) space, and this way it only requires O(m+n) space.
241
+ * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
242
+ * point for each row of A.
243
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
244
+ *
245
+ * This epilogue also supports bias, which remains per-channel.
246
+ */
247
+template <typename ElementD, typename OutputTileThreadMap>
248
+struct ScaledEpilogueBiasAzpToken
249
+ : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
250
+ private:
251
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
252
+ using Accum = typename SUPER::Accum;
253
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
254
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
255
+ using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
256
+
257
+ // Per-token azp term, shape (m,1)
258
+ using Azp = typename SUPER::template ColLoad<int32_t>;
259
+
260
+ // This is the AZP adjustment term, J @ B, shape (1,n)
261
+ using AzpAdj = typename SUPER::template RowLoad<int32_t>;
262
+
263
+ // Compute azp * azp_adj
264
+ using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
265
+ cutlass::multiplies, int32_t, int32_t,
266
+ cutlass::FloatRoundStyle::round_to_nearest>;
267
+
268
+ using EVTComputeAzp =
269
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Azp, AzpAdj>;
270
+
271
+ // Compute float(accum - azp*azp_adj), all operands are int32_t
272
+ using ComputeAcc = cutlass::epilogue::threadblock::VisitorCompute<
273
+ cutlass::minus, float, int32_t,
274
+ cutlass::FloatRoundStyle::round_to_nearest>;
275
+
276
+ using EVTComputeAcc =
277
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeAcc, Accum, EVTComputeAzp>;
278
+
279
+ using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
280
+ cutlass::multiplies, float, float,
281
+ cutlass::FloatRoundStyle::round_to_nearest>;
282
+
283
+ using EVTComputeScaleB =
284
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
285
+ EVTComputeAcc>;
286
+
287
+ using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
288
+ cutlass::multiply_add, ElementD, float,
289
+ cutlass::FloatRoundStyle::round_to_nearest>;
290
+
291
+ public:
292
+ using EVTCompute =
293
+ cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
294
+ EVTComputeScaleB, Bias>;
295
+
296
+ using ArgumentType = typename EVTCompute::Arguments;
297
+
298
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
299
+ torch::Tensor const& b_scales,
300
+ torch::Tensor const& azp_adj,
301
+ torch::Tensor const& azp,
302
+ c10::optional<torch::Tensor> const& bias) {
303
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
304
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
305
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
306
+ auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
307
+ auto azp_adj_args =
308
+ SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
309
+
310
+ typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
311
+ typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
312
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
313
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
314
+ }
315
+};
316
+
317
+}; // namespace vllm::c2x
\ No newline at end of file
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
new
+315
@@ -0,0 +1,315 @@
1
+#include "cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp"
2
+
3
+/*
4
+ This file defines custom epilogues for fusing channel scales, token scales,
5
+ bias, and activation zero-points onto a GEMM operation using the
6
+ CUTLASS 3.x API, for NVIDIA GPUs with sm90a (Hopper) or later.
7
+
8
+ Epilogues must contain a public type named EVTCompute of type Sm90EVT,
9
+ as well as a static prepare_args function that constructs an
10
+ EVTCompute::Arguments struct.
11
+*/
12
+
13
+namespace vllm::c3x {
14
+
15
+using namespace cute;
16
+
17
+/*
18
+ * This class provides the common load descriptors for the
19
+ * ScaledEpilogue[...] classes
20
+ */
21
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
22
+struct ScaledEpilogueBase {
23
+ protected:
24
+ using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
25
+
26
+ template <typename T>
27
+ using ColOrScalarLoad = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
28
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
29
+ Stride<Int<1>, Int<0>, Int<0>>>;
30
+
31
+ template <typename T>
32
+ using RowOrScalarLoad = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
33
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
34
+ Stride<Int<0>, Int<1>, Int<0>>>;
35
+
36
+ // Don't want to support nullptr by default
37
+ template <typename T, bool EnableNullPtr = false>
38
+ using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
39
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
40
+ Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
41
+
42
+ // Don't want to support nullptr by default
43
+ template <typename T, bool EnableNullPtr = false>
44
+ using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
45
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
46
+ Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
47
+
48
+ // This utility function constructs the arguments for the load descriptors
49
+ // from a tensor. It can handle both row and column, as well as row/column or
50
+ // scalar cases.
51
+ template <typename Descriptor, typename T>
52
+ static auto args_from_tensor(torch::Tensor const& tensor) {
53
+ using Arguments = typename Descriptor::Arguments;
54
+ auto* data_ptr = static_cast<T*>(tensor.data_ptr());
55
+ if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
56
+ std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
57
+ return Arguments{data_ptr, tensor.numel() != 1};
58
+ } else {
59
+ static_assert(!std::is_same_v<Descriptor, ColLoad<T, true>> &&
60
+ !std::is_same_v<Descriptor, RowLoad<T, true>>);
61
+ return Arguments{data_ptr};
62
+ }
63
+ }
64
+
65
+ // This overload handles the case where there might not be a tensor, in which
66
+ // case a nullptr is passed and a constant (0) is used.
67
+ template <typename Descriptor, typename T>
68
+ static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
69
+ using Arguments = typename Descriptor::Arguments;
70
+ auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
71
+ static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
72
+ std::is_same_v<Descriptor, RowLoad<T, true>>);
73
+ return Arguments{data_ptr};
74
+ }
75
+};
76
+
77
+/*
78
+ This epilogue function defines a quantized GEMM operation similar to
79
+ torch.scaled_mm_.
80
+
81
+ A and B may be both either int8 or fp8_e4m3. A can be
82
+ quantized per-tensor or per-row. B can be quantized per-tensor or per-column.
83
+ Any combination of per-tensor and per-row or column is supported.
84
+ A and B must have symmetric quantization (zero point == 0).
85
+
86
+ So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
87
+ scales are applied elementwise with numpy-style broadcasting.
88
+
89
+ ScaleA and ScaleB define the epilogue functions that apply the scales for
90
+ the A and B operands respectively. These scales may be either per-tensor or
91
+ per row or column.
92
+*/
93
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
94
+struct ScaledEpilogue
95
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
96
+ private:
97
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
98
+ using Accum = typename SUPER::Accum;
99
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
100
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
101
+
102
+ using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
103
+ cutlass::multiplies, float, float,
104
+ cutlass::FloatRoundStyle::round_to_nearest>;
105
+
106
+ using EVTCompute0 =
107
+ cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
108
+
109
+ using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
110
+ cutlass::multiplies, ElementD, float,
111
+ cutlass::FloatRoundStyle::round_to_nearest>;
112
+
113
+ public:
114
+ using EVTCompute =
115
+ cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
116
+ using ArgumentType = typename EVTCompute::Arguments;
117
+
118
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
119
+ torch::Tensor const& b_scales) {
120
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
121
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
122
+
123
+ typename EVTCompute0::Arguments evt0_args{b_args};
124
+ return ArgumentType{a_args, evt0_args};
125
+ }
126
+};
127
+
128
+/*
129
+ * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
130
+ * This bias can also be used in the per-tensor azp case, where the activation
131
+ * zero point (azp) is used to compute an azp correction term,
132
+ * which is folded into the bias.
133
+ *
134
+ * The bias tensor must be per-output channel.
135
+ * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
136
+ */
137
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
138
+struct ScaledEpilogueBias
139
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
140
+ private:
141
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
142
+ using Accum = typename SUPER::Accum;
143
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
144
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
145
+ using Bias = typename SUPER::template RowLoad<ElementD>;
146
+
147
+ using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
148
+ cutlass::multiplies, float, float,
149
+ cutlass::FloatRoundStyle::round_to_nearest>;
150
+
151
+ using EVTCompute0 =
152
+ cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
153
+
154
+ using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
155
+ cutlass::multiply_add, ElementD, float,
156
+ cutlass::FloatRoundStyle::round_to_nearest>;
157
+
158
+ public:
159
+ using EVTCompute =
160
+ cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
161
+
162
+ using ArgumentType = typename EVTCompute::Arguments;
163
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
164
+ torch::Tensor const& b_scales,
165
+ torch::Tensor const& bias) {
166
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
167
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
168
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
169
+
170
+ typename EVTCompute0::Arguments evt0_args{b_args};
171
+ return ArgumentType{a_args, evt0_args, bias_args};
172
+ }
173
+};
174
+
175
+/*
176
+ * This epilogue directly supports per-tensor azp in int32 form.
177
+ * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
178
+ * term, which should already be multiplied with the scalar azp.
179
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
180
+ *
181
+ * This epilogue also supports bias, which remains per-channel.
182
+ */
183
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
184
+struct ScaledEpilogueBiasAzp
185
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
186
+ private:
187
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
188
+ using Accum = typename SUPER::Accum;
189
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
190
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
191
+ using Bias = typename SUPER::template RowLoad<ElementD, true>;
192
+
193
+ // This is the full AZP term, azp * J @ B, shape (1,n)
194
+ using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
195
+
196
+ // Compute float(accum - azp_adj), both operands are int32_t
197
+ using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
198
+ cutlass::minus, float, int32_t,
199
+ cutlass::FloatRoundStyle::round_to_nearest>;
200
+
201
+ using EVTComputeAzp =
202
+ cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Accum, AzpWithAdj>;
203
+
204
+ using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
205
+ cutlass::multiplies, float, float,
206
+ cutlass::FloatRoundStyle::round_to_nearest>;
207
+
208
+ using EVTComputeScaleB =
209
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAzp>;
210
+
211
+ using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
212
+ cutlass::multiply_add, ElementD, float,
213
+ cutlass::FloatRoundStyle::round_to_nearest>;
214
+
215
+ public:
216
+ using EVTCompute =
217
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
218
+ EVTComputeScaleB, Bias>;
219
+ using ArgumentType = typename EVTCompute::Arguments;
220
+
221
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
222
+ torch::Tensor const& b_scales,
223
+ torch::Tensor const& azp_adj,
224
+ c10::optional<torch::Tensor> const& bias) {
225
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
226
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
227
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
228
+ auto azp_adj_args =
229
+ SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
230
+
231
+ typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
232
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
233
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
234
+ }
235
+};
236
+
237
+/*
238
+ * This epilogue supports per-token azp by computing and applying
239
+ * the correction term using a rank-1 update. If the term were materialized,
240
+ * it would require O(m*n) space, and this way it only requires O(m+n) space.
241
+ * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
242
+ * point for each row of A.
243
+ * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
244
+ *
245
+ * This epilogue also supports bias, which remains per-channel.
246
+ */
247
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
248
+struct ScaledEpilogueBiasAzpToken
249
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
250
+ private:
251
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
252
+ using Accum = typename SUPER::Accum;
253
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
254
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
255
+ using Bias = typename SUPER::template RowLoad<ElementD, true>;
256
+
257
+ // Per-token azp term, shape (m,1)
258
+ using Azp = typename SUPER::template ColLoad<int32_t>;
259
+
260
+ // This is the AZP adjustment term, J @ B, shape (1,n)
261
+ using AzpAdj = typename SUPER::template RowLoad<int32_t>;
262
+
263
+ // Compute azp * azp_adj
264
+ using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
265
+ cutlass::multiplies, int32_t, int32_t,
266
+ cutlass::FloatRoundStyle::round_to_nearest>;
267
+
268
+ using EVTComputeAzp =
269
+ cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Azp, AzpAdj>;
270
+
271
+ // Compute float(accum - azp*azp_adj), all operands are int32_t
272
+ using ComputeAcc = cutlass::epilogue::fusion::Sm90Compute<
273
+ cutlass::minus, float, int32_t,
274
+ cutlass::FloatRoundStyle::round_to_nearest>;
275
+
276
+ using EVTComputeAcc =
277
+ cutlass::epilogue::fusion::Sm90EVT<ComputeAcc, Accum, EVTComputeAzp>;
278
+
279
+ using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
280
+ cutlass::multiplies, float, float,
281
+ cutlass::FloatRoundStyle::round_to_nearest>;
282
+
283
+ using EVTComputeScaleB =
284
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAcc>;
285
+
286
+ using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
287
+ cutlass::multiply_add, ElementD, float,
288
+ cutlass::FloatRoundStyle::round_to_nearest>;
289
+
290
+ public:
291
+ using EVTCompute =
292
+ cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
293
+ EVTComputeScaleB, Bias>;
294
+ using ArgumentType = typename EVTCompute::Arguments;
295
+
296
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
297
+ torch::Tensor const& b_scales,
298
+ torch::Tensor const& azp_adj,
299
+ torch::Tensor const& azp,
300
+ c10::optional<torch::Tensor> const& bias) {
301
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
302
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
303
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
304
+ auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
305
+ auto azp_adj_args =
306
+ SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
307
+
308
+ typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
309
+ typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
310
+ typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
311
+ return ArgumentType{a_args, evt_scale_b_args, bias_args};
312
+ }
313
+};
314
+
315
+}; // namespace vllm::c3x
\ No newline at end of file
csrc/cutlass_extensions/vllm_cutlass_library_extension.py
+29
@@ -35,6 +35,35 @@ VLLMDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
35
}
36
}
37
38
+VLLMDataTypeSize: Dict[Union[VLLMDataType, DataType], int] = {
39
+ **DataTypeSize, # type: ignore
40
+ **{
41
+ VLLMDataType.u4b8: 4,
42
+ VLLMDataType.u8b128: 8,
43
+ }
44
+}
45
+
46
+VLLMDataTypeVLLMScalarTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
47
+ VLLMDataType.u4b8: "vllm::kU4B8",
48
+ VLLMDataType.u8b128: "vllm::kU8B128",
49
+ DataType.u4: "vllm::kU4",
50
+ DataType.u8: "vllm::kU8",
51
+ DataType.s4: "vllm::kS4",
52
+ DataType.s8: "vllm::kS8",
53
+ DataType.f16: "vllm::kFloat16",
54
+ DataType.bf16: "vllm::kBfloat16",
55
+}
56
+
57
+VLLMDataTypeTorchDataTypeTag: Dict[Union[VLLMDataType, DataType], str] = {
58
+ DataType.u8: "at::ScalarType::Byte",
59
+ DataType.s8: "at::ScalarType::Char",
60
+ DataType.e4m3: "at::ScalarType::Float8_e4m3fn",
61
+ DataType.s32: "at::ScalarType::Int",
62
+ DataType.f16: "at::ScalarType::Half",
63
+ DataType.bf16: "at::ScalarType::BFloat16",
64
+ DataType.f32: "at::ScalarType::Float",
65
+}
66
+
67
VLLMKernelScheduleTag: Dict[Union[
68
MixedInputKernelScheduleType, KernelScheduleType], str] = {
69
**KernelScheduleTag, # type: ignore
csrc/cutlass_extensions/vllm_numeric_conversion.cuh
+218
-21
@@ -3,6 +3,7 @@
3
#include "cutlass/numeric_conversion.h"
4
#include "cutlass_extensions/vllm_custom_types.cuh"
5
#include "cutlass_extensions/cute_utils.cuh"
6
+#include "cutlass_extensions/vllm_type_utils.cuh"
7
8
// this file extends:
9
// https://github.com/NVIDIA/cutlass/blob/cutlass-3.5.0/include/cutlass/numeric_conversion.h
@@ -28,8 +29,19 @@ struct InterleavedNumericArrayConverter {
29
30
CUTLASS_DEVICE
31
static result_type convert(source_type const& source) {
31
- CUTE_INVALID_CONTROL_PATH(
32
- "InterleavedNumericArrayConverter not implemented\n");
32
+ if (cute::elect_one_sync()) {
33
+ if constexpr (std::is_same_v<IlvBlkLayout, void>) {
34
+ printf(
35
+ "Convert %s <= %s (N = %d, IlvBlkLayout = void), not implemented\n",
36
+ nameof_v<T>, nameof_v<S>, N);
37
+ } else {
38
+ printf(
39
+ "Convert %s <= %s (N = %d, size(IlvBlkLayout{}) = %d), not "
40
+ "implemented\n",
41
+ nameof_v<T>, nameof_v<S>, N, size(IlvBlkLayout{}));
42
+ }
43
+ __brkpt();
44
+ }
45
return {};
46
}
47
@@ -56,11 +68,6 @@ struct InterleavedNumericArrayConverter<
68
result_type operator()(source_type const& s) const { return convert(s); }
69
};
70
59
-// TODO (LucasWilkinson): Implement
60
-// for Array<cutlass::float8_e4m3fn, N> <= Array<vllm_uint4b8_t, N>
61
-
62
-// ....
63
-
71
template <typename RegConvert32bit, typename T, typename S, int N>
72
struct ArrayConverterPacked32Bit {
73
using result_type = Array<T, N>;
@@ -86,14 +93,16 @@ struct ArrayConverterPacked32Bit {
93
using ScalarConverter = NumericConverter<T, S>;
94
95
template <typename PackedSrc>
89
- CUTLASS_DEVICE static uint32_t to_reg(PackedSrc const& source) {
96
+ CUTLASS_DEVICE static auto to_regs(PackedSrc const& src) {
97
if constexpr (sizeof(PackedSrc) == 1) {
91
- return static_cast<uint32_t>(reinterpret_cast<const uint8_t&>(source));
98
+ return Array<uint32_t, 1>{reinterpret_cast<uint8_t const&>(src)};
99
} else if constexpr (sizeof(PackedSrc) == 2) {
93
- return static_cast<uint32_t>(reinterpret_cast<const uint16_t&>(source));
100
+ return Array<uint32_t, 1>{reinterpret_cast<uint16_t const&>(src)};
101
+ } else if constexpr (sizeof(PackedSrc) == 4) {
102
+ return Array<uint32_t, 1>{reinterpret_cast<uint32_t const&>(src)};
103
} else {
95
- static_assert(sizeof(PackedSrc) == 4);
96
- return reinterpret_cast<const uint32_t&>(source);
104
+ static_assert(sizeof(PackedSrc) == 8);
105
+ return reinterpret_cast<Array<uint32_t, 2> const&>(src);
106
}
107
}
108
@@ -110,7 +119,7 @@ struct ArrayConverterPacked32Bit {
119
static_assert(std::is_same_v<typename PackedSrcType::Element, S>);
120
static_assert(std::is_same_v<typename PackedResultType::Element, T>);
121
113
- return RegConvert32bit::template convert<PackedResultType>(to_reg(source));
122
+ return RegConvert32bit::template convert<PackedResultType>(to_regs(source));
123
}
124
125
friend class detail::VectorizedConverter;
@@ -140,6 +149,131 @@ struct ArrayConverterPacked32Bit {
149
}
150
};
151
152
+// Convert 8 4bit values packed into a 32bit register to 8 8bit values packed
153
+// into 2 32bit register.
154
+template <uint8_t LUT0, uint8_t LUT1, uint8_t LUT2, uint8_t LUT3, //
155
+ uint8_t LUT4, uint8_t LUT5, uint8_t LUT6, uint8_t LUT7, //
156
+ uint8_t LUT8, uint8_t LUT9, uint8_t LUT10, uint8_t LUT11, //
157
+ uint8_t LUT12, uint8_t LUT13, uint8_t LUT14, uint8_t LUT15>
158
+CUTLASS_DEVICE cutlass::AlignedArray<uint32_t, 2> lut_4bit_to_8bit_convert(
159
+ uint32_t src) {
160
+ cutlass::AlignedArray<uint32_t, 2> r;
161
+ // Determines if the value is in the top half of the LUT if set or
162
+ // (i.e. LUT[8:15]) in the bottom half (i.e. LUT[0:7]) if not set. Then move
163
+ // into bit position 0x4 of each nibble so when or'd with final_prmt_base it
164
+ // selects the correct candidate. When elements in final_prmt_base
165
+ // are >= 0x4, the high candidate is selected (i.e. LUT[8:15]), when elements
166
+ // are < 0x4, the low candidate is selected (i.e. LUT[0:7])
167
+ uint32_t high_bit = (src & 0x88888888) >> 1;
168
+
169
+ // `high_bit` is OR'd with 0x31203120 to find the correct value in the LUT
170
+ // (selects correct high or low candidate)
171
+ const uint32_t final_prmt_base = 0x32103210;
172
+
173
+ // Ignore the high bit when indexing into LUT, for each 4bit value
174
+ // we index into both the high and low candidates then use
175
+ // high_bit | final_prmt_base to select the correct candidate
176
+ uint32_t lut_idx = (src & 0x77777777);
177
+
178
+ auto pack = [](uint8_t a, uint8_t b, uint8_t c, uint8_t d) {
179
+ return uint32_t(a) | (uint32_t(b) << 8) | (uint32_t(c) << 16) |
180
+ (uint32_t(d) << 24);
181
+ };
182
+
183
+ static constexpr uint32_t LOW_0 = pack(LUT0, LUT1, LUT2, LUT3);
184
+ static constexpr uint32_t LOW_1 = pack(LUT4, LUT5, LUT6, LUT7);
185
+ static constexpr uint32_t HIGH_0 = pack(LUT8, LUT9, LUT10, LUT11);
186
+ static constexpr uint32_t HIGH_1 = pack(LUT12, LUT13, LUT14, LUT15);
187
+
188
+ CUTLASS_PRAGMA_UNROLL
189
+ for (int ii = 0; ii < 2; ++ii, lut_idx >>= 16, high_bit >>= 16) {
190
+ uint32_t final_prmt_idx = final_prmt_base | high_bit;
191
+
192
+ // This uses a look up table to convert packed int4s to packed int8s,
193
+ // using the int4 value as the index to prmt. It first select both the
194
+ // high and low candidates, then uses the high bit (i.e. `high_bit`) to
195
+ // select the correct candidate.
196
+ asm volatile(
197
+ "{\n"
198
+ " .reg .b32 low, high;\n"
199
+ " prmt.b32 low, %1, %2, %5;\n"
200
+ " prmt.b32 high, %3, %4, %5;\n"
201
+ " prmt.b32 %0, low, high, %6;\n"
202
+ "}\n"
203
+ : "=r"(r[ii])
204
+ : "n"(LOW_0), "n"(LOW_1), "n"(HIGH_0), "n"(HIGH_1), "r"(lut_idx),
205
+ "r"(final_prmt_idx));
206
+ }
207
+
208
+ return r;
209
+};
210
+
211
+// for Array<int8_t, N> <= Array<vllm_uint4b8_t, N>
212
+template <FloatRoundStyle Round, int N>
213
+struct NumericArrayConverter<int8_t, vllm_uint4b8_t, N, Round> {
214
+ using result_type = Array<int8_t, N>;
215
+ using source_type = Array<vllm_uint4b8_t, N>;
216
+
217
+ static FloatRoundStyle const round_style = Round;
218
+
219
+ private:
220
+ struct RegConvert {
221
+ template <typename PackedResultType>
222
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
223
+ // [-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7] as int8s
224
+ auto r = lut_4bit_to_8bit_convert<0xF8, 0xF9, 0xFA, 0xFB, //
225
+ 0xFC, 0xFD, 0xFE, 0xFF, //
226
+ 0x00, 0x01, 0x02, 0x03, //
227
+ 0x04, 0x05, 0x06, 0x07>(src_[0]);
228
+ return reinterpret_cast<PackedResultType&>(r);
229
+ };
230
+ };
231
+
232
+ public:
233
+ CUTLASS_DEVICE
234
+ static result_type convert(source_type const& source) {
235
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
236
+ typename source_type::Element,
237
+ N>::convert(source);
238
+ }
239
+
240
+ CUTLASS_DEVICE
241
+ result_type operator()(source_type const& s) const { return convert(s); }
242
+};
243
+
244
+// for Array<cutlass::float_e4m3_t, N> <= Array<vllm_uint4b8_t, N>
245
+template <FloatRoundStyle Round, int N>
246
+struct NumericArrayConverter<cutlass::float_e4m3_t, vllm_uint4b8_t, N, Round> {
247
+ using result_type = Array<cutlass::float_e4m3_t, N>;
248
+ using source_type = Array<vllm_uint4b8_t, N>;
249
+
250
+ static FloatRoundStyle const round_style = Round;
251
+
252
+ private:
253
+ struct RegConvert {
254
+ template <typename PackedResultType>
255
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
256
+ // [-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7] as fp8s
257
+ auto r = lut_4bit_to_8bit_convert<0xD0, 0xCE, 0xCC, 0xCA, //
258
+ 0xC8, 0xC4, 0xC0, 0xB8, //
259
+ 0x00, 0x38, 0x40, 0x44, //
260
+ 0x48, 0x4A, 0x4C, 0x4E>(src_[0]);
261
+ return reinterpret_cast<PackedResultType&>(r);
262
+ };
263
+ };
264
+
265
+ public:
266
+ CUTLASS_DEVICE
267
+ static result_type convert(source_type const& source) {
268
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
269
+ typename source_type::Element,
270
+ N>::convert(source);
271
+ }
272
+
273
+ CUTLASS_DEVICE
274
+ result_type operator()(source_type const& s) const { return convert(s); }
275
+};
276
+
277
// for Array<cutlass::half_t, N> <= Array<vllm_uint4b8_t, N>
278
template <FloatRoundStyle Round, int N>
279
struct NumericArrayConverter<cutlass::half_t, vllm_uint4b8_t, N, Round> {
@@ -148,7 +282,8 @@ struct NumericArrayConverter<cutlass::half_t, vllm_uint4b8_t, N, Round> {
282
283
struct RegConvert {
284
template <typename PackedResultType>
151
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
285
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
286
+ uint32_t src = src_[0];
287
using RegArray =
288
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
289
sizeof(PackedResultType)>;
@@ -249,7 +384,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
384
private:
385
struct RegConvert {
386
template <typename PackedResultType>
252
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
387
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
388
+ uint32_t src = src_[0];
389
using RegArray =
390
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
391
sizeof(PackedResultType)>;
@@ -338,7 +474,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
474
private:
475
struct RegConvert {
476
template <typename PackedResultType>
341
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
477
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
478
+ uint32_t src = src_[0];
479
using RegArray =
480
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
481
sizeof(PackedResultType)>;
@@ -417,7 +554,8 @@ struct NumericArrayConverter<cutlass::half_t, vllm_uint8b128_t, N, Round> {
554
555
struct RegConvert {
556
template <typename PackedResultType>
420
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
557
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
558
+ uint32_t src = src_[0];
559
// Hold output FP16s in reg. We need 1 reg for every 2 elements
560
using RegArray =
561
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
@@ -469,7 +607,8 @@ struct NumericArrayConverter<float, vllm_uint8b128_t, N, Round> {
607
private:
608
struct RegConvert {
609
template <typename PackedResultType>
472
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
610
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
611
+ uint32_t src = src_[0];
612
PackedResultType r;
613
614
// __byte_perm simulates the add.u32 0x4B000000 to every u8 element of
@@ -513,7 +652,8 @@ struct NumericArrayConverter<cutlass::bfloat16_t, vllm_uint4b8_t, N, Round> {
652
private:
653
struct RegConvert {
654
template <typename PackedResultType>
516
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src_reg) {
655
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
656
+ uint32_t src_reg = src_[0];
657
// Hold output BF16s in reg. We need 1 reg for every 2 elements
658
using RegArray =
659
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
@@ -603,7 +743,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
743
private:
744
struct RegConvert {
745
template <typename PackedResultType>
606
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
746
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
747
+ uint32_t src = src_[0];
748
using RegArray =
749
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
750
sizeof(PackedResultType)>;
@@ -671,7 +812,8 @@ struct InterleavedNumericArrayConverter<Layout<Shape<_2, _4>, Stride<_4, _1>>,
812
private:
813
struct RegConvert {
814
template <typename PackedResultType>
674
- CUTLASS_DEVICE static PackedResultType convert(uint32_t src) {
815
+ CUTLASS_DEVICE static PackedResultType convert(Array<uint32_t, 1> src_) {
816
+ uint32_t src = src_[0];
817
using RegArray =
818
cutlass::AlignedArray<uint32_t, PackedResultType::kElements / 2,
819
sizeof(PackedResultType)>;
@@ -788,6 +930,61 @@ struct NumericArrayConverter<cutlass::bfloat16_t, vllm_uint8b128_t, N, Round> {
930
931
#endif
932
933
+// for Array<int8_t, N> <= Array<cutlass::half_t, N>
934
+// FastFP16toINT8 from https://arxiv.org/pdf/2406.09904
935
+template <FloatRoundStyle Round, int N>
936
+struct NumericArrayConverter<int8_t, cutlass::half_t, N, Round> {
937
+ using result_type = Array<int8_t, N>;
938
+ using source_type = Array<cutlass::half_t, N>;
939
+
940
+ struct RegConvert {
941
+ // FastFP16toINT8 from https://arxiv.org/pdf/2406.09904
942
+ template <typename PackedResultType, int src_regs>
943
+ CUTLASS_DEVICE static PackedResultType convert(
944
+ Array<uint32_t, src_regs> src) {
945
+ // Hold output int8s in reg. We need 1 reg for every 4 elements
946
+ using RegArray = cutlass::AlignedArray<
947
+ uint32_t, std::max(PackedResultType::kElements / 4, size_t(1))>;
948
+ RegArray r;
949
+
950
+ static constexpr uint32_t MAGIC_BIAS_ = 0x64806480;
951
+ auto MAGIC_BIAS = *reinterpret_cast<const half2*>(&MAGIC_BIAS_);
952
+
953
+ *reinterpret_cast<half2*>(&src[0]) =
954
+ __hadd2(*reinterpret_cast<half2*>(&src[0]), MAGIC_BIAS);
955
+
956
+ if constexpr (src_regs > 1) {
957
+ *reinterpret_cast<half2*>(&src[1]) =
958
+ __hadd2(*reinterpret_cast<half2*>(&src[1]), MAGIC_BIAS);
959
+ }
960
+
961
+ static_assert(PackedResultType::kElements <= 4);
962
+ uint32_t uint8s;
963
+ static constexpr uint32_t MASK_0246 = 0x6420;
964
+ static constexpr uint32_t UINT8s_TO_INT8s_MASK = 0x80808080;
965
+ asm volatile("prmt.b32 %0,%1,%2,%3;\n"
966
+ : "=r"(uint8s)
967
+ : "r"(src[0]), "r"((src_regs > 1) ? src[1] : src[0]),
968
+ "n"(MASK_0246));
969
+
970
+ uint32_t int8s = (uint8s ^ UINT8s_TO_INT8s_MASK);
971
+
972
+ return reinterpret_cast<PackedResultType&>(int8s);
973
+ };
974
+ };
975
+
976
+ public:
977
+ CUTLASS_DEVICE
978
+ static result_type convert(source_type const& source) {
979
+ return ArrayConverterPacked32Bit<RegConvert, typename result_type::Element,
980
+ typename source_type::Element,
981
+ N>::convert(source);
982
+ }
983
+
984
+ CUTLASS_DEVICE
985
+ result_type operator()(source_type const& s) const { return convert(s); }
986
+};
987
+
988
/////////////////////////////////////////////////////////////////////////////////////////////////
989
990
} // namespace cutlass
csrc/cutlass_extensions/vllm_type_utils.cuh
new
+42
@@ -0,0 +1,42 @@
1
+#include "cutlass/bfloat16.h"
2
+#include "cutlass/half.h"
3
+#include "cuda_bf16.h"
4
+
5
+#include "cutlass_extensions/vllm_custom_types.cuh"
6
+
7
+namespace cutlass {
8
+
9
+template <typename T>
10
+struct nameof {
11
+ static constexpr char const* value = "unknown";
12
+};
13
+
14
+template <typename T>
15
+inline constexpr auto nameof_v = nameof<T>::value;
16
+
17
+#define NAMEOF_TYPE(T) \
18
+ template <> \
19
+ struct nameof<T> { \
20
+ static constexpr char const* value = #T; \
21
+ };
22
+
23
+NAMEOF_TYPE(float_e4m3_t)
24
+NAMEOF_TYPE(float_e5m2_t)
25
+NAMEOF_TYPE(half_t)
26
+NAMEOF_TYPE(nv_bfloat16)
27
+NAMEOF_TYPE(bfloat16_t)
28
+NAMEOF_TYPE(float)
29
+
30
+NAMEOF_TYPE(int4b_t)
31
+NAMEOF_TYPE(int8_t)
32
+NAMEOF_TYPE(int32_t)
33
+NAMEOF_TYPE(int64_t)
34
+
35
+NAMEOF_TYPE(vllm_uint4b8_t)
36
+NAMEOF_TYPE(uint4b_t)
37
+NAMEOF_TYPE(uint8_t)
38
+NAMEOF_TYPE(vllm_uint8b128_t)
39
+NAMEOF_TYPE(uint32_t)
40
+NAMEOF_TYPE(uint64_t)
41
+
42
+}; // namespace cutlass
\ No newline at end of file
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+27
-26
@@ -8,6 +8,10 @@
8
#include "scaled_mm_c2x_sm89_fp8_dispatch.cuh"
9
#include "scaled_mm_c2x_sm89_int8_dispatch.cuh"
10
11
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp"
12
+
13
+using namespace vllm;
14
+
15
/*
16
This file defines quantized GEMM operations using the CUTLASS 2.x API, for
17
NVIDIA GPUs with SM versions prior to sm90 (Hopper).
@@ -22,12 +26,11 @@ void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
26
TORCH_CHECK(b.dtype() == torch::kInt8);
27
28
if (out.dtype() == torch::kBFloat16) {
25
- return vllm::cutlass_gemm_sm75_dispatch<int8_t, cutlass::bfloat16_t,
26
- Epilogue>(
29
+ return cutlass_gemm_sm75_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
30
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
31
} else {
32
TORCH_CHECK(out.dtype() == torch::kFloat16);
30
- return vllm::cutlass_gemm_sm75_dispatch<int8_t, cutlass::half_t, Epilogue>(
33
+ return cutlass_gemm_sm75_dispatch<int8_t, cutlass::half_t, Epilogue>(
34
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
35
}
36
}
@@ -42,10 +45,10 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
45
if (bias) {
46
TORCH_CHECK(bias->dtype() == out.dtype(),
47
"currently bias dtype must match output dtype ", out.dtype());
45
- return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBias>(
48
+ return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogueBias>(
49
out, a, b, a_scales, b_scales, *bias);
50
} else {
48
- return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogue>(
51
+ return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogue>(
52
out, a, b, a_scales, b_scales);
53
}
54
}
@@ -61,10 +64,10 @@ void cutlass_scaled_mm_azp_sm75(torch::Tensor& out, torch::Tensor const& a,
64
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
65
66
if (azp) {
64
- return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
67
+ return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogueBiasAzpToken>(
68
out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
69
} else {
67
- return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBiasAzp>(
70
+ return cutlass_scaled_mm_sm75_epilogue<c2x::ScaledEpilogueBiasAzp>(
71
out, a, b, a_scales, b_scales, azp_adj, bias);
72
}
73
}
@@ -78,12 +81,11 @@ void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
81
TORCH_CHECK(b.dtype() == torch::kInt8);
82
83
if (out.dtype() == torch::kBFloat16) {
81
- return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
82
- Epilogue>(
84
+ return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
85
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
86
} else {
87
TORCH_CHECK(out.dtype() == torch::kFloat16);
86
- return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
88
+ return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
89
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
90
}
91
}
@@ -98,10 +100,10 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
100
if (bias) {
101
TORCH_CHECK(bias->dtype() == out.dtype(),
102
"currently bias dtype must match output dtype ", out.dtype());
101
- return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBias>(
103
+ return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogueBias>(
104
out, a, b, a_scales, b_scales, *bias);
105
} else {
104
- return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogue>(
106
+ return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogue>(
107
out, a, b, a_scales, b_scales);
108
}
109
}
@@ -117,10 +119,10 @@ void cutlass_scaled_mm_azp_sm80(torch::Tensor& out, torch::Tensor const& a,
119
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
120
121
if (azp) {
120
- return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
122
+ return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogueBiasAzpToken>(
123
out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
124
} else {
123
- return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBiasAzp>(
125
+ return cutlass_scaled_mm_sm80_epilogue<c2x::ScaledEpilogueBiasAzp>(
126
out, a, b, a_scales, b_scales, azp_adj, bias);
127
}
128
}
@@ -134,13 +136,12 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
136
TORCH_CHECK(b.dtype() == torch::kInt8);
137
138
if (out.dtype() == torch::kBFloat16) {
137
- return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::bfloat16_t,
138
- Epilogue>(
139
+ return cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::bfloat16_t,
140
+ Epilogue>(
141
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
142
} else {
143
assert(out.dtype() == torch::kFloat16);
142
- return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::half_t,
143
- Epilogue>(
144
+ return cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
145
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
146
}
147
} else {
@@ -148,13 +149,13 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
149
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
150
151
if (out.dtype() == torch::kBFloat16) {
151
- return vllm::cutlass_gemm_sm89_fp8_dispatch<
152
- cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue>(
152
+ return cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
153
+ cutlass::bfloat16_t, Epilogue>(
154
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
155
} else {
156
TORCH_CHECK(out.dtype() == torch::kFloat16);
156
- return vllm::cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
157
- cutlass::half_t, Epilogue>(
157
+ return cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
158
+ cutlass::half_t, Epilogue>(
159
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
160
}
161
}
@@ -170,10 +171,10 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
171
if (bias) {
172
TORCH_CHECK(bias->dtype() == out.dtype(),
173
"currently bias dtype must match output dtype ", out.dtype());
173
- return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBias>(
174
+ return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogueBias>(
175
out, a, b, a_scales, b_scales, *bias);
176
} else {
176
- return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogue>(
177
+ return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogue>(
178
out, a, b, a_scales, b_scales);
179
}
180
}
@@ -189,10 +190,10 @@ void cutlass_scaled_mm_azp_sm89(torch::Tensor& out, torch::Tensor const& a,
190
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
191
192
if (azp) {
192
- return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzpToken>(
193
+ return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogueBiasAzpToken>(
194
out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
195
} else {
195
- return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBiasAzp>(
196
+ return cutlass_scaled_mm_sm89_epilogue<c2x::ScaledEpilogueBiasAzp>(
197
out, a, b, a_scales, b_scales, azp_adj, bias);
198
}
199
}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
-302
@@ -21,7 +21,6 @@
21
#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
22
#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
23
24
-#include "broadcast_load_epilogue_c2x.hpp"
24
#include "common.hpp"
25
// clang-format on
26
@@ -71,307 +70,6 @@ struct enable_sm89_to_sm90 : Kernel {
70
#endif
71
}
72
};
74
-
75
-/*
76
- * This class provides the common load descriptors for the
77
- * ScaledEpilogue[...] classes
78
- */
79
-template <typename ElementD, typename OutputTileThreadMap>
80
-struct ScaledEpilogueBase {
81
- protected:
82
- using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
83
-
84
- template <typename T>
85
- using ColOrScalarLoad =
86
- cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
87
- OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
88
-
89
- template <typename T>
90
- using RowOrScalarLoad =
91
- cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92
- OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
93
-
94
- template <typename T>
95
- using ColLoad = cutlass::epilogue::threadblock::VisitorColBroadcast<
96
- OutputTileThreadMap, T, Stride<Int<1>, Int<0>, Int<0>>>;
97
-
98
- template <typename T>
99
- using RowLoad = cutlass::epilogue::threadblock::VisitorRowBroadcast<
100
- OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
101
-
102
- template <typename T>
103
- using RowOrZeroLoad =
104
- cutlass::epilogue::threadblock::VisitorRowOrZeroBroadcast<
105
- OutputTileThreadMap, T, Stride<Int<0>, Int<1>, Int<0>>>;
106
-
107
- // This utility function constructs the arguments for the load descriptors
108
- // from a tensor. It can handle both row and column, as well as row/column or
109
- // scalar cases.
110
- template <typename Descriptor, typename T>
111
- static auto args_from_tensor(torch::Tensor const& tensor) {
112
- using Arguments = typename Descriptor::Arguments;
113
- auto* data_ptr = static_cast<T*>(tensor.data_ptr());
114
- if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
115
- std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
116
- return Arguments{data_ptr, tensor.numel() != 1};
117
- } else {
118
- // it would technically work but no use case as data_ptr is never nullptr
119
- static_assert(!std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
120
- return Arguments{data_ptr};
121
- }
122
- }
123
-
124
- // This overload handles the case where there might not be a tensor, in which
125
- // case a nullptr is passed and a constant (0) is used.
126
- template <typename Descriptor, typename T>
127
- static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
128
- static_assert(std::is_same_v<Descriptor, RowOrZeroLoad<T>>);
129
- using Arguments = typename Descriptor::Arguments;
130
- auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
131
- return Arguments{data_ptr};
132
- }
133
-};
134
-
135
-/*
136
- This epilogue function defines a quantized GEMM operation similar to
137
- torch._scaled_mm.
138
-
139
- A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
140
- per-row. B can be quantized per-tensor or per-column.
141
- Any combination of per-tensor and per-row or column is supported.
142
- A and B must have symmetric quantization (zero point == 0).
143
-
144
- So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
145
- scales are applied elementwise with numpy-style broadcasting.
146
-
147
- ScaleA and ScaleB define the epilogue functions that apply the scales for
148
- the A and B operands respectively. These scales may be either per-tensor or
149
- per row or column.
150
-*/
151
-template <typename ElementD, typename OutputTileThreadMap>
152
-struct ScaledEpilogue
153
- : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
154
- private:
155
- using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156
- using Accum = typename SUPER::Accum;
157
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
158
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
159
-
160
- using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161
- cutlass::multiplies, float, float,
162
- cutlass::FloatRoundStyle::round_to_nearest>;
163
-
164
- using EVTCompute0 =
165
- cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
166
-
167
- using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
168
- cutlass::multiplies, ElementD, float,
169
- cutlass::FloatRoundStyle::round_to_nearest>;
170
-
171
- public:
172
- using EVTCompute =
173
- cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
174
- using ArgumentType = typename EVTCompute::Arguments;
175
-
176
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
177
- torch::Tensor const& b_scales) {
178
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
179
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
180
-
181
- typename EVTCompute0::Arguments evt0_args{b_args};
182
- return ArgumentType{a_args, evt0_args};
183
- }
184
-};
185
-
186
-/*
187
- * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
188
- * This bias can also be used in the per-tensor azp case, where the activation
189
- * zero point (azp) is used to compute an azp correction term,
190
- * which is folded into the bias.
191
- *
192
- * The bias tensor must be per-output channel.
193
- * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
194
- */
195
-template <typename ElementD, typename OutputTileThreadMap>
196
-struct ScaledEpilogueBias
197
- : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
198
- protected:
199
- using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
200
- using Accum = typename SUPER::Accum;
201
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
202
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
203
- using Bias = typename SUPER::template RowLoad<ElementD>;
204
- using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
205
- cutlass::multiplies, float, float,
206
- cutlass::FloatRoundStyle::round_to_nearest>;
207
-
208
- using EVTCompute0 =
209
- cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
210
-
211
- using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
212
- cutlass::multiply_add, ElementD, float,
213
- cutlass::FloatRoundStyle::round_to_nearest>;
214
-
215
- public:
216
- using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
217
- EVTCompute0, Bias>;
218
- using ArgumentType = typename EVTCompute::Arguments;
219
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
220
- torch::Tensor const& b_scales,
221
- torch::Tensor const& bias) {
222
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
223
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
224
- auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
225
-
226
- typename EVTCompute0::Arguments evt0_args{b_args};
227
- return ArgumentType{a_args, evt0_args, bias_args};
228
- }
229
-};
230
-
231
-/*
232
- * This epilogue directly supports per-tensor azp in int32 form.
233
- * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
234
- * term, which should already be multiplied with the scalar azp.
235
- * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
236
- *
237
- * This epilogue also supports bias, which remains per-channel.
238
- */
239
-template <typename ElementD, typename OutputTileThreadMap>
240
-struct ScaledEpilogueBiasAzp
241
- : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
242
- private:
243
- using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
244
- using Accum = typename SUPER::Accum;
245
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
246
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
247
- using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
248
-
249
- // This is the full AZP term, azp * J @ B, shape (1,n)
250
- using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
251
-
252
- // Compute float(accum - azp_adj), both operands are int32_t
253
- using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
254
- cutlass::minus, float, int32_t,
255
- cutlass::FloatRoundStyle::round_to_nearest>;
256
-
257
- using EVTComputeAzp =
258
- cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Accum, AzpWithAdj>;
259
-
260
- using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
261
- cutlass::multiplies, float, float,
262
- cutlass::FloatRoundStyle::round_to_nearest>;
263
-
264
- using EVTComputeScaleB =
265
- cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
266
- EVTComputeAzp>;
267
-
268
- using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
269
- cutlass::multiply_add, ElementD, float,
270
- cutlass::FloatRoundStyle::round_to_nearest>;
271
-
272
- public:
273
- using EVTCompute =
274
- cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
275
- EVTComputeScaleB, Bias>;
276
-
277
- using ArgumentType = typename EVTCompute::Arguments;
278
-
279
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
280
- torch::Tensor const& b_scales,
281
- torch::Tensor const& azp_adj,
282
- c10::optional<torch::Tensor> const& bias) {
283
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
284
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
285
- auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
286
- auto azp_adj_args =
287
- SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
288
-
289
- typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
290
- typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
291
- return ArgumentType{a_args, evt_scale_b_args, bias_args};
292
- }
293
-};
294
-
295
-/*
296
- * This epilogue supports per-token azp by computing and applying
297
- * the correction term using a rank-1 update. If the term were materialized,
298
- * it would require O(m*n) space, and this way it only requires O(m+n) space.
299
- * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
300
- * point for each row of A.
301
- * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
302
- *
303
- * This epilogue also supports bias, which remains per-channel.
304
- */
305
-template <typename ElementD, typename OutputTileThreadMap>
306
-struct ScaledEpilogueBiasAzpToken
307
- : protected ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
308
- private:
309
- using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
310
- using Accum = typename SUPER::Accum;
311
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
312
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
313
- using Bias = typename SUPER::template RowOrZeroLoad<ElementD>;
314
-
315
- // Per-token azp term, shape (m,1)
316
- using Azp = typename SUPER::template ColLoad<int32_t>;
317
-
318
- // This is the AZP adjustment term, J @ B, shape (1,n)
319
- using AzpAdj = typename SUPER::template RowLoad<int32_t>;
320
-
321
- // Compute azp * azp_adj
322
- using ComputeAzp = cutlass::epilogue::threadblock::VisitorCompute<
323
- cutlass::multiplies, int32_t, int32_t,
324
- cutlass::FloatRoundStyle::round_to_nearest>;
325
-
326
- using EVTComputeAzp =
327
- cutlass::epilogue::threadblock::Sm80EVT<ComputeAzp, Azp, AzpAdj>;
328
-
329
- // Compute float(accum - azp*azp_adj), all operands are int32_t
330
- using ComputeAcc = cutlass::epilogue::threadblock::VisitorCompute<
331
- cutlass::minus, float, int32_t,
332
- cutlass::FloatRoundStyle::round_to_nearest>;
333
-
334
- using EVTComputeAcc =
335
- cutlass::epilogue::threadblock::Sm80EVT<ComputeAcc, Accum, EVTComputeAzp>;
336
-
337
- using ComputeScaleB = cutlass::epilogue::threadblock::VisitorCompute<
338
- cutlass::multiplies, float, float,
339
- cutlass::FloatRoundStyle::round_to_nearest>;
340
-
341
- using EVTComputeScaleB =
342
- cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleB, ScaleB,
343
- EVTComputeAcc>;
344
-
345
- using ComputeScaleBiasA = cutlass::epilogue::threadblock::VisitorCompute<
346
- cutlass::multiply_add, ElementD, float,
347
- cutlass::FloatRoundStyle::round_to_nearest>;
348
-
349
- public:
350
- using EVTCompute =
351
- cutlass::epilogue::threadblock::Sm80EVT<ComputeScaleBiasA, ScaleA,
352
- EVTComputeScaleB, Bias>;
353
-
354
- using ArgumentType = typename EVTCompute::Arguments;
355
-
356
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
357
- torch::Tensor const& b_scales,
358
- torch::Tensor const& azp_adj,
359
- torch::Tensor const& azp,
360
- c10::optional<torch::Tensor> const& bias) {
361
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
362
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
363
- auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
364
- auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
365
- auto azp_adj_args =
366
- SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
367
-
368
- typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
369
- typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
370
- typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
371
- return ArgumentType{a_args, evt_scale_b_args, bias_args};
372
- }
373
-};
374
-
73
template <typename Arch, template <typename> typename ArchGuard,
74
typename ElementAB_, typename ElementD_,
75
template <typename, typename> typename Epilogue_, typename TileShape,
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+7
-305
@@ -23,11 +23,12 @@
23
#include "cutlass/epilogue/collective/collective_builder.hpp"
24
#include "cutlass/gemm/collective/collective_builder.hpp"
25
26
-#include "broadcast_load_epilogue_c3x.hpp"
26
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
27
#include "common.hpp"
28
// clang-format on
29
30
using namespace cute;
31
+using namespace vllm;
32
33
/*
34
This file defines quantized GEMM operations using the CUTLASS 3.x API, for
@@ -56,305 +57,6 @@ struct enable_sm90_or_later : Kernel {
57
#endif
58
}
59
};
59
-
60
-/*
61
- * This class provides the common load descriptors for the
62
- * ScaledEpilogue[...] classes
63
- */
64
-template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
65
-struct ScaledEpilogueBase {
66
- protected:
67
- using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
68
-
69
- template <typename T>
70
- using ColOrScalarLoad = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
71
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
72
- Stride<Int<1>, Int<0>, Int<0>>>;
73
-
74
- template <typename T>
75
- using RowOrScalarLoad = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
76
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
77
- Stride<Int<0>, Int<1>, Int<0>>>;
78
-
79
- // Don't want to support nullptr by default
80
- template <typename T, bool EnableNullPtr = false>
81
- using ColLoad = cutlass::epilogue::fusion::Sm90ColBroadcast<
82
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
83
- Stride<Int<1>, Int<0>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
84
-
85
- // Don't want to support nullptr by default
86
- template <typename T, bool EnableNullPtr = false>
87
- using RowLoad = cutlass::epilogue::fusion::Sm90RowBroadcast<
88
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, T,
89
- Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<T>, EnableNullPtr>;
90
-
91
- // This utility function constructs the arguments for the load descriptors
92
- // from a tensor. It can handle both row and column, as well as row/column or
93
- // scalar cases.
94
- template <typename Descriptor, typename T>
95
- static auto args_from_tensor(torch::Tensor const& tensor) {
96
- using Arguments = typename Descriptor::Arguments;
97
- auto* data_ptr = static_cast<T*>(tensor.data_ptr());
98
- if constexpr (std::is_same_v<Descriptor, ColOrScalarLoad<T>> ||
99
- std::is_same_v<Descriptor, RowOrScalarLoad<T>>) {
100
- return Arguments{data_ptr, tensor.numel() != 1};
101
- } else {
102
- static_assert(!std::is_same_v<Descriptor, ColLoad<T, true>> &&
103
- !std::is_same_v<Descriptor, RowLoad<T, true>>);
104
- return Arguments{data_ptr};
105
- }
106
- }
107
-
108
- // This overload handles the case where there might not be a tensor, in which
109
- // case a nullptr is passed and a constant (0) is used.
110
- template <typename Descriptor, typename T>
111
- static auto args_from_tensor(c10::optional<torch::Tensor> const& tensor) {
112
- using Arguments = typename Descriptor::Arguments;
113
- auto* data_ptr = tensor ? static_cast<T*>(tensor->data_ptr()) : nullptr;
114
- static_assert(std::is_same_v<Descriptor, ColLoad<T, true>> ||
115
- std::is_same_v<Descriptor, RowLoad<T, true>>);
116
- return Arguments{data_ptr};
117
- }
118
-};
119
-
120
-/*
121
- This epilogue function defines a quantized GEMM operation similar to
122
- torch.scaled_mm_.
123
-
124
- A and B may be both either int8 or fp8_e4m3. A can be
125
- quantized per-tensor or per-row. B can be quantized per-tensor or per-column.
126
- Any combination of per-tensor and per-row or column is supported.
127
- A and B must have symmetric quantization (zero point == 0).
128
-
129
- So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
130
- scales are applied elementwise with numpy-style broadcasting.
131
-
132
- ScaleA and ScaleB define the epilogue functions that apply the scales for
133
- the A and B operands respectively. These scales may be either per-tensor or
134
- per row or column.
135
-*/
136
-template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
137
-struct ScaledEpilogue
138
- : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
139
- private:
140
- using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
141
- using Accum = typename SUPER::Accum;
142
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
143
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
144
-
145
- using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
146
- cutlass::multiplies, float, float,
147
- cutlass::FloatRoundStyle::round_to_nearest>;
148
-
149
- using EVTCompute0 =
150
- cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
151
-
152
- using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
153
- cutlass::multiplies, ElementD, float,
154
- cutlass::FloatRoundStyle::round_to_nearest>;
155
-
156
- public:
157
- using EVTCompute =
158
- cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
159
- using ArgumentType = typename EVTCompute::Arguments;
160
-
161
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
162
- torch::Tensor const& b_scales) {
163
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
164
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
165
-
166
- typename EVTCompute0::Arguments evt0_args{b_args};
167
- return ArgumentType{a_args, evt0_args};
168
- }
169
-};
170
-
171
-/*
172
- * This epilogue performs the same operation as ScaledEpilogue, but adds a bias.
173
- * This bias can also be used in the per-tensor azp case, where the activation
174
- * zero point (azp) is used to compute an azp correction term,
175
- * which is folded into the bias.
176
- *
177
- * The bias tensor must be per-output channel.
178
- * ScaleA and ScaleB can be per-tensor or per-token/per-channel.
179
- */
180
-template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
181
-struct ScaledEpilogueBias
182
- : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
183
- private:
184
- using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
185
- using Accum = typename SUPER::Accum;
186
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
187
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
188
- using Bias = typename SUPER::template RowLoad<ElementD>;
189
-
190
- using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
191
- cutlass::multiplies, float, float,
192
- cutlass::FloatRoundStyle::round_to_nearest>;
193
-
194
- using EVTCompute0 =
195
- cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
196
-
197
- using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
198
- cutlass::multiply_add, ElementD, float,
199
- cutlass::FloatRoundStyle::round_to_nearest>;
200
-
201
- public:
202
- using EVTCompute =
203
- cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
204
-
205
- using ArgumentType = typename EVTCompute::Arguments;
206
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
207
- torch::Tensor const& b_scales,
208
- torch::Tensor const& bias) {
209
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
210
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
211
- auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
212
-
213
- typename EVTCompute0::Arguments evt0_args{b_args};
214
- return ArgumentType{a_args, evt0_args, bias_args};
215
- }
216
-};
217
-
218
-/*
219
- * This epilogue directly supports per-tensor azp in int32 form.
220
- * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
221
- * term, which should already be multiplied with the scalar azp.
222
- * The azp_adj term is a 1D tensor of shape (1,n), computed as azp * J @ B.
223
- *
224
- * This epilogue also supports bias, which remains per-channel.
225
- */
226
-template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
227
-struct ScaledEpilogueBiasAzp
228
- : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
229
- private:
230
- using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
231
- using Accum = typename SUPER::Accum;
232
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
233
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
234
- using Bias = typename SUPER::template RowLoad<ElementD, true>;
235
-
236
- // This is the full AZP term, azp * J @ B, shape (1,n)
237
- using AzpWithAdj = typename SUPER::template RowLoad<int32_t>;
238
-
239
- // Compute float(accum - azp_adj), both operands are int32_t
240
- using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
241
- cutlass::minus, float, int32_t,
242
- cutlass::FloatRoundStyle::round_to_nearest>;
243
-
244
- using EVTComputeAzp =
245
- cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Accum, AzpWithAdj>;
246
-
247
- using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
248
- cutlass::multiplies, float, float,
249
- cutlass::FloatRoundStyle::round_to_nearest>;
250
-
251
- using EVTComputeScaleB =
252
- cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAzp>;
253
-
254
- using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
255
- cutlass::multiply_add, ElementD, float,
256
- cutlass::FloatRoundStyle::round_to_nearest>;
257
-
258
- public:
259
- using EVTCompute =
260
- cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
261
- EVTComputeScaleB, Bias>;
262
- using ArgumentType = typename EVTCompute::Arguments;
263
-
264
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
265
- torch::Tensor const& b_scales,
266
- torch::Tensor const& azp_adj,
267
- c10::optional<torch::Tensor> const& bias) {
268
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
269
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
270
- auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
271
- auto azp_adj_args =
272
- SUPER::template args_from_tensor<AzpWithAdj, int32_t>(azp_adj);
273
-
274
- typename EVTComputeAzp::Arguments evt_azp_args{{}, azp_adj_args};
275
- typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_azp_args};
276
- return ArgumentType{a_args, evt_scale_b_args, bias_args};
277
- }
278
-};
279
-
280
-/*
281
- * This epilogue supports per-token azp by computing and applying
282
- * the correction term using a rank-1 update. If the term were materialized,
283
- * it would require O(m*n) space, and this way it only requires O(m+n) space.
284
- * The azp term is a 1D tensor of shape (m,1), and represents the unscaled zero
285
- * point for each row of A.
286
- * The azp_adj term is a 1D tensor of shape (1,n), computed as J @ B.
287
- *
288
- * This epilogue also supports bias, which remains per-channel.
289
- */
290
-template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
291
-struct ScaledEpilogueBiasAzpToken
292
- : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
293
- private:
294
- using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
295
- using Accum = typename SUPER::Accum;
296
- using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
297
- using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
298
- using Bias = typename SUPER::template RowLoad<ElementD, true>;
299
-
300
- // Per-token azp term, shape (m,1)
301
- using Azp = typename SUPER::template ColLoad<int32_t>;
302
-
303
- // This is the AZP adjustment term, J @ B, shape (1,n)
304
- using AzpAdj = typename SUPER::template RowLoad<int32_t>;
305
-
306
- // Compute azp * azp_adj
307
- using ComputeAzp = cutlass::epilogue::fusion::Sm90Compute<
308
- cutlass::multiplies, int32_t, int32_t,
309
- cutlass::FloatRoundStyle::round_to_nearest>;
310
-
311
- using EVTComputeAzp =
312
- cutlass::epilogue::fusion::Sm90EVT<ComputeAzp, Azp, AzpAdj>;
313
-
314
- // Compute float(accum - azp*azp_adj), all operands are int32_t
315
- using ComputeAcc = cutlass::epilogue::fusion::Sm90Compute<
316
- cutlass::minus, float, int32_t,
317
- cutlass::FloatRoundStyle::round_to_nearest>;
318
-
319
- using EVTComputeAcc =
320
- cutlass::epilogue::fusion::Sm90EVT<ComputeAcc, Accum, EVTComputeAzp>;
321
-
322
- using ComputeScaleB = cutlass::epilogue::fusion::Sm90Compute<
323
- cutlass::multiplies, float, float,
324
- cutlass::FloatRoundStyle::round_to_nearest>;
325
-
326
- using EVTComputeScaleB =
327
- cutlass::epilogue::fusion::Sm90EVT<ComputeScaleB, ScaleB, EVTComputeAcc>;
328
-
329
- using ComputeScaleBiasA = cutlass::epilogue::fusion::Sm90Compute<
330
- cutlass::multiply_add, ElementD, float,
331
- cutlass::FloatRoundStyle::round_to_nearest>;
332
-
333
- public:
334
- using EVTCompute =
335
- cutlass::epilogue::fusion::Sm90EVT<ComputeScaleBiasA, ScaleA,
336
- EVTComputeScaleB, Bias>;
337
- using ArgumentType = typename EVTCompute::Arguments;
338
-
339
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
340
- torch::Tensor const& b_scales,
341
- torch::Tensor const& azp_adj,
342
- torch::Tensor const& azp,
343
- c10::optional<torch::Tensor> const& bias) {
344
- auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
345
- auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
346
- auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
347
- auto azp_args = SUPER::template args_from_tensor<Azp, int32_t>(azp);
348
- auto azp_adj_args =
349
- SUPER::template args_from_tensor<AzpAdj, int32_t>(azp_adj);
350
-
351
- typename EVTComputeAzp::Arguments evt_azp_args{azp_args, azp_adj_args};
352
- typename EVTComputeAcc::Arguments evt_acc_args{{}, evt_azp_args};
353
- typename EVTComputeScaleB::Arguments evt_scale_b_args{b_args, evt_acc_args};
354
- return ArgumentType{a_args, evt_scale_b_args, bias_args};
355
- }
356
-};
357
-
60
template <typename ElementAB_, typename ElementD_,
61
template <typename, typename, typename> typename Epilogue_,
62
typename TileShape, typename ClusterShape, typename KernelSchedule,
@@ -721,11 +423,11 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
423
if (bias) {
424
TORCH_CHECK(bias->dtype() == c.dtype(),
425
"currently bias dtype must match output dtype ", c.dtype());
724
- return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBias>(
426
+ return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
427
c, a, b, a_scales, b_scales, *bias);
428
} else {
727
- return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogue>(c, a, b, a_scales,
728
- b_scales);
429
+ return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogue>(
430
+ c, a, b, a_scales, b_scales);
431
}
432
}
433
@@ -740,10 +442,10 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
442
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
443
444
if (azp) {
743
- return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzpToken>(
445
+ return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzpToken>(
446
out, a, b, a_scales, b_scales, azp_adj, *azp, bias);
447
} else {
746
- return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBiasAzp>(
448
+ return cutlass_scaled_mm_sm90_epilogue<c3x::ScaledEpilogueBiasAzp>(
449
out, a, b, a_scales, b_scales, azp_adj, bias);
450
}
451
}
csrc/quantization/machete/generate.py
+429
-303
@@ -3,8 +3,10 @@ import math
3
import os
4
import shutil
5
from collections.abc import Iterable
6
-from dataclasses import dataclass
7
-from typing import List, Optional, Tuple, Union
6
+from copy import deepcopy
7
+from dataclasses import dataclass, fields
8
+from functools import reduce
9
+from typing import Dict, List, Optional, Tuple, Union
10
11
import jinja2
12
# yapf conflicts with isort for this block
@@ -14,7 +16,10 @@ from vllm_cutlass_library_extension import (DataType, EpilogueScheduleTag,
16
MixedInputKernelScheduleType,
17
TileSchedulerTag,
18
TileSchedulerType, VLLMDataType,
17
- VLLMDataTypeNames, VLLMDataTypeTag,
19
+ VLLMDataTypeNames,
20
+ VLLMDataTypeSize, VLLMDataTypeTag,
21
+ VLLMDataTypeTorchDataTypeTag,
22
+ VLLMDataTypeVLLMScalarTypeTag,
23
VLLMKernelScheduleTag)
24
25
# yapf: enable
@@ -27,49 +32,125 @@ DISPATCH_TEMPLATE = """
32
#include "../machete_mm_launcher.cuh"
33
34
namespace machete {
30
-using GemmDispatcher_ = GemmDispatcher<
31
- {{DataTypeTag[type_config.element_a]}}, // ElementA
32
- {{DataTypeTag[type_config.element_b]}}, // ElementB
33
- {{DataTypeTag[type_config.element_d]}}, // ElementD
34
- {{DataTypeTag[type_config.accumulator]}}, // Accumulator
35
- {{DataTypeTag[type_config.element_b_scale]}}, // Scales
36
- {{DataTypeTag[type_config.element_b_zeropoint]}}>; // Zeropoints
37
-
38
-{% for s in schedules %}extern torch::Tensor
39
-impl_{{type_name}}_sch_{{ gen_sch_name(s) }}(PyTorchArguments args);
40
-{% endfor %}
41
-template <>
42
-torch::Tensor GemmDispatcher_::dispatch(PyTorchArguments args) {
35
+
36
+{% for impl_config in impl_configs %}
37
+{% set type_sig = gen_type_sig(impl_config.types) -%}
38
+{% for s in impl_config.schedules %}
39
+extern torch::Tensor impl_{{type_sig}}_sch_{{gen_sch_sig(s)}}(MMArgs);
40
+{%- endfor %}
41
+
42
+torch::Tensor mm_dispatch_{{type_sig}}(MMArgs args) {
43
[[maybe_unused]] auto M = args.A.size(0);
44
[[maybe_unused]] auto N = args.B.size(1);
45
[[maybe_unused]] auto K = args.A.size(1);
46
47
- if (!args.schedule) {
48
- {%- for cond, s in heuristic %}
47
+ if (!args.maybe_schedule) {
48
+ {%- for cond, s in impl_config.heuristic %}
49
{%if cond is not none%}if ({{cond}})
50
{%- else %}else
51
{%- endif %}
52
- return impl_{{ type_name }}_sch_{{ gen_sch_name(s) }}(args);{% endfor %}
52
+ return impl_{{type_sig}}_sch_{{ gen_sch_sig(s) }}(args);{% endfor %}
53
}
54
55
- {% for s in schedules %}
56
- if (*args.schedule == "{{ gen_sch_name(s) }}") {
57
- return impl_{{ type_name }}_sch_{{ gen_sch_name(s) }}(args);
58
- }
59
- {% endfor %}
55
+ {%- for s in impl_config.schedules %}
56
+ if (*args.maybe_schedule == "{{ gen_sch_sig(s) }}")
57
+ return impl_{{type_sig}}_sch_{{ gen_sch_sig(s) }}(args);
58
+ {%- endfor %}
59
TORCH_CHECK_NOT_IMPLEMENTED(false, "machete_gemm(..) is not implemented for "
61
- "schedule = ", *args.schedule);
60
+ "schedule = ", *args.maybe_schedule);
61
}
62
+{%- endfor %}
63
+
64
64
-template <>
65
-std::vector<std::string> GemmDispatcher_::supported_schedules() {
66
- return {
67
- {% for s in schedules -%}
68
- "{{ gen_sch_name(s) }}"{{ ",
69
- " if not loop.last }}{%- endfor %}
70
- };
65
+static inline std::optional<at::ScalarType> maybe_scalartype(
66
+ c10::optional<at::Tensor> const& t) {
67
+ if (!t) {
68
+ return std::nullopt;
69
+ } else {
70
+ return t->scalar_type();
71
+ };
72
+}
73
+
74
+torch::Tensor mm_dispatch(MMArgs args) {
75
+ auto out_type = args.maybe_out_type.value_or(args.A.scalar_type());
76
+ auto a_type = args.A.scalar_type();
77
+ auto maybe_g_scales_type = maybe_scalartype(args.maybe_group_scales);
78
+ auto maybe_g_zeros_type = maybe_scalartype(args.maybe_group_zeros);
79
+ auto maybe_ch_scales_type = maybe_scalartype(args.maybe_channel_scales);
80
+ auto maybe_tok_scales_type = maybe_scalartype(args.maybe_token_scales);
81
+
82
+ {% for impl_config in impl_configs %}
83
+ {% set t = impl_config.types -%}
84
+ {% set type_sig = gen_type_sig(t) -%}
85
+ if (args.b_type == {{VLLMScalarTypeTag[t.b]}}
86
+ && a_type == {{TorchTypeTag[t.a]}}
87
+ && out_type == {{TorchTypeTag[t.out]}}
88
+ && {%if t.b_group_scale != void -%}
89
+ maybe_g_scales_type == {{TorchTypeTag[t.b_group_scale]}}
90
+ {%- else %}!maybe_g_scales_type{%endif%}
91
+ && {%if t.b_group_zeropoint != void -%}
92
+ maybe_g_zeros_type == {{TorchTypeTag[t.b_group_zeropoint]}}
93
+ {%- else %}!maybe_g_zeros_type{%endif%}
94
+ && {%if t.b_channel_scale != void -%}
95
+ maybe_ch_scales_type == {{TorchTypeTag[t.b_channel_scale]}}
96
+ {%- else %}!maybe_ch_scales_type{%endif%}
97
+ && {%if t.a_token_scale != void -%}
98
+ maybe_tok_scales_type == {{TorchTypeTag[t.a_token_scale]}}
99
+ {%- else %}!maybe_tok_scales_type{%endif%}
100
+ ) {
101
+ return mm_dispatch_{{type_sig}}(args);
102
+ }
103
+ {%- endfor %}
104
+
105
+ TORCH_CHECK_NOT_IMPLEMENTED(
106
+ false, "machete_mm(..) is not implemented for "
107
+ "a_type=", args.A.scalar_type(),
108
+ ", b_type=", args.b_type.str(),
109
+ ", out_type=", out_type,
110
+ ", with_group_scale_type=", maybe_g_scales_type
111
+ ? toString(*maybe_g_scales_type) : "None",
112
+ ", with_group_zeropoint_type=", maybe_g_zeros_type
113
+ ? toString(*maybe_g_zeros_type) : "None",
114
+ ", with_channel_scale_type=", maybe_ch_scales_type
115
+ ? toString(*maybe_ch_scales_type) : "None",
116
+ ", with_token_scale_type=", maybe_tok_scales_type
117
+ ? toString(*maybe_tok_scales_type) : "None",
118
+ "; implemented types are: \\n",
119
+ {%- for impl_config in impl_configs %}
120
+ {% set t = impl_config.types -%}
121
+ "\\t{{gen_type_option_name(t)}}\\n",
122
+ {%- endfor %}
123
+ "");
124
}
125
126
+std::vector<std::string> supported_schedules_dispatch(
127
+ SupportedSchedulesArgs args) {
128
+ auto out_type = args.maybe_out_type.value_or(args.a_type);
129
+
130
+ {% for impl_config in impl_configs %}
131
+ {% set t = impl_config.types -%}
132
+ {% set schs = impl_config.schedules -%}
133
+ if (args.b_type == {{VLLMScalarTypeTag[t.b]}}
134
+ && args.a_type == {{TorchTypeTag[t.a]}}
135
+ && out_type == {{TorchTypeTag[t.out]}}
136
+ && {%if t.b_group_scale != void -%}
137
+ args.maybe_group_scales_type == {{TorchTypeTag[t.b_group_scale]}}
138
+ {%- else %}!args.maybe_group_scales_type{%endif%}
139
+ && {%if t.b_group_zeropoint != void-%}
140
+ args.maybe_group_zeros_type == {{TorchTypeTag[t.b_group_zeropoint]}}
141
+ {%- else %}!args.maybe_group_zeros_type{%endif%}
142
+ ) {
143
+ return {
144
+ {%- for s in impl_config.schedules %}
145
+ "{{gen_sch_sig(s)}}"{% if not loop.last %},{% endif %}
146
+ {%- endfor %}
147
+ };
148
+ }
149
+ {%- endfor %}
150
+
151
+ return {};
152
+};
153
+
154
}; // namespace machete
155
"""
156
@@ -77,20 +158,10 @@ IMPL_TEMPLATE = """
158
#include "../machete_mm_launcher.cuh"
159
160
namespace machete {
80
-template <typename Config, bool with_C, bool with_scales, bool with_zeropoints>
81
-using Kernel = MacheteKernelTemplate<
82
- {{DataTypeTag[type_config.element_a]}}, // ElementA
83
- {{DataTypeTag[type_config.element_b]}}, // ElementB
84
- {{DataTypeTag[type_config.element_d]}}, // ElementD
85
- {{DataTypeTag[type_config.accumulator]}}, // Accumulator
86
- {{DataTypeTag[type_config.element_b_scale]}}, // Scales
87
- {{DataTypeTag[type_config.element_b_zeropoint]}}, // Zeropoints
88
- cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
89
- Config, with_C, with_scales, with_zeropoints>;
90
-
91
-{% for sch in schedules %}
92
-{% set schedule_name = gen_sch_name(sch) -%}
93
-struct sch_{{schedule_name}} {
161
+
162
+{% for sch in unique_schedules(impl_configs) %}
163
+{% set sch_sig = gen_sch_sig(sch) -%}
164
+struct sch_{{sch_sig}} {
165
using TileShapeNM = Shape<{{
166
to_cute_constant(sch.tile_shape_mn)|join(', ')}}>;
167
using ClusterShape = Shape<{{
@@ -101,27 +172,34 @@ struct sch_{{schedule_name}} {
172
using TileScheduler = {{TileSchedulerTag[sch.tile_scheduler]}};
173
using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
174
};
104
-
175
+{% endfor %}
176
+
177
+{% for impl_config in impl_configs %}
178
+{% set t = impl_config.types -%}
179
+{% set schs = impl_config.schedules -%}
180
+{% set type_sig = gen_type_sig(t) -%}
181
+
182
+template<typename Sch>
183
+using Kernel_{{type_sig}} = MacheteKernelTemplate<
184
+ {{DataTypeTag[t.a]}}, // ElementA
185
+ {{DataTypeTag[t.b]}}, // ElementB
186
+ {{DataTypeTag[t.out]}}, // ElementD
187
+ {{DataTypeTag[t.accumulator]}}, // Accumulator
188
+ {{DataTypeTag[t.b_group_scale]}}, // GroupScaleT
189
+ {{DataTypeTag[t.b_group_zeropoint]}}, // GroupZeroT
190
+ {{DataTypeTag[t.b_channel_scale]}}, // ChannelScaleT
191
+ {{DataTypeTag[t.a_token_scale]}}, // TokenScaleT
192
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
193
+ Sch>;
194
+
195
+{% for sch in schs %}
196
+{% set sch_sig = gen_sch_sig(sch) -%}
197
torch::Tensor
106
-impl_{{type_name}}_sch_{{schedule_name}}(PyTorchArguments args) {
107
- bool with_C = args.C.has_value(), with_scales = args.scales.has_value(),
108
- with_zeropoints = args.zeros.has_value();
109
-
110
- {% for s in specializations %}
111
- if (with_C == {{s.with_C|lower}}
112
- && with_zeropoints == {{s.with_zeropoints|lower}}
113
- && with_scales == {{s.with_scales|lower}}) {
114
- return run_impl<Kernel<sch_{{schedule_name}}, {{s.with_C|lower}},
115
- {{s.with_scales|lower}}, {{s.with_zeropoints|lower}}>>(args);
116
- }{% endfor %}
117
-
118
- TORCH_CHECK_NOT_IMPLEMENTED(
119
- false, "for the sake of compile times and binary size machete_mm(..) is "
120
- " not implemented for with_C=", with_C, ", with_scales=", with_scales,
121
- ", with_zeropoints=", with_zeropoints,
122
- " (for {{type_name}}_sch_{{schedule_name}})");
198
+impl_{{type_sig}}_sch_{{sch_sig}}(MMArgs args) {
199
+ return run_impl<Kernel_{{type_sig}}<sch_{{sch_sig}}>>(args);
200
}
124
-{% endfor %}
201
+{%- endfor %}
202
+{%- endfor %}
203
204
}; // namespace machete
205
"""
@@ -130,26 +208,34 @@ PREPACK_TEMPLATE = """
208
#include "../machete_prepack_launcher.cuh"
209
210
namespace machete {
133
-using PrepackBDispatcher_ = PrepackBDispatcher<
134
- {{DataTypeTag[type_config.element_a]}}, // ElementA
135
- {{DataTypeTag[type_config.element_b]}}, // ElementB
136
- {{DataTypeTag[type_config.element_d]}}, // ElementD
137
- {{DataTypeTag[type_config.accumulator]}}, // Accumulator
138
- {{DataTypeTag[type_config.element_b_scale]}}, // Scales
139
- {{DataTypeTag[type_config.element_b_zeropoint]}}>; // Zeropoints
140
-
141
-using PrepackedLayoutB = PrepackedLayoutBTemplate<
142
- {{DataTypeTag[type_config.element_a]}}, // ElementA
143
- {{DataTypeTag[type_config.element_b]}}, // ElementB
144
- {{DataTypeTag[type_config.element_d]}}, // ElementD
145
- {{DataTypeTag[type_config.accumulator]}}, // Accumulator
146
- cutlass::layout::ColumnMajor,
147
- cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>;
148
-
149
-template <>
150
-torch::Tensor PrepackBDispatcher_::dispatch(torch::Tensor B) {
151
- return prepack_impl<PrepackedLayoutB>(B);
211
+
212
+torch::Tensor prepack_B_dispatch(PrepackBArgs args) {
213
+ auto convert_type = args.maybe_group_scales_type.value_or(args.a_type);
214
+ {%- for t in types %}
215
+ {% set b_type = unsigned_type_with_bitwidth(t.b_num_bits) %}
216
+ if (args.a_type == {{TorchTypeTag[t.a]}}
217
+ && args.b_type.size_bits() == {{t.b_num_bits}}
218
+ && convert_type == {{TorchTypeTag[t.convert]}}) {
219
+ return prepack_impl<
220
+ PrepackedLayoutBTemplate<
221
+ {{DataTypeTag[t.a]}}, // ElementA
222
+ {{DataTypeTag[b_type]}}, // ElementB
223
+ {{DataTypeTag[t.convert]}}, // ElementConvert
224
+ {{DataTypeTag[t.accumulator]}}, // Accumulator
225
+ cutlass::layout::ColumnMajor,
226
+ cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>
227
+ >(args.B);
228
+ }
229
+ {%- endfor %}
230
+
231
+ TORCH_CHECK_NOT_IMPLEMENTED(false,
232
+ "prepack_B_dispatch(..) is not implemented for "
233
+ "atype = ", args.a_type,
234
+ ", b_type = ", args.b_type.str(),
235
+ ", with_group_scales_type= ", args.maybe_group_scales_type ?
236
+ toString(*args.maybe_group_scales_type) : "None");
237
}
238
+
239
}; // namespace machete
240
"""
241
@@ -166,32 +252,34 @@ class ScheduleConfig:
252
tile_scheduler: TileSchedulerType
253
254
169
-@dataclass
255
+@dataclass(frozen=True)
256
class TypeConfig:
171
- element_a: DataType
172
- element_b: Union[DataType, VLLMDataType]
173
- element_b_scale: DataType
174
- element_b_zeropoint: DataType
175
- element_d: DataType
257
+ a: DataType
258
+ b: Union[DataType, VLLMDataType]
259
+ b_group_scale: DataType
260
+ b_group_zeropoint: DataType
261
+ b_channel_scale: DataType
262
+ a_token_scale: DataType
263
+ out: DataType
264
accumulator: DataType
265
266
179
-@dataclass
180
-class Specialization:
181
- with_C: bool
182
- with_zeropoints: bool
183
- with_scales: bool
267
+@dataclass(frozen=True)
268
+class PrepackTypeConfig:
269
+ a: DataType
270
+ b_num_bits: int
271
+ convert: DataType
272
+ accumulator: DataType
273
274
275
@dataclass
276
class ImplConfig:
188
- type_config: TypeConfig
189
- schedule_configs: List[ScheduleConfig]
190
- specializations: List[Specialization]
277
+ types: TypeConfig
278
+ schedules: List[ScheduleConfig]
279
heuristic: List[Tuple[Optional[str], ScheduleConfig]]
280
281
194
-def generate_schedule_name(schedule_config: ScheduleConfig) -> str:
282
+def generate_sch_sig(schedule_config: ScheduleConfig) -> str:
283
tile_shape = (
284
f"{schedule_config.tile_shape_mn[0]}x{schedule_config.tile_shape_mn[1]}"
285
)
@@ -209,40 +297,34 @@ def generate_schedule_name(schedule_config: ScheduleConfig) -> str:
297
f"_{epilogue_schedule}_{tile_scheduler}")
298
299
212
-# mostly unique shorter schedule_name
213
-def generate_terse_schedule_name(schedule_config: ScheduleConfig) -> str:
300
+# mostly unique shorter sch_sig
301
+def generate_terse_sch_sig(schedule_config: ScheduleConfig) -> str:
302
kernel_terse_names_replace = {
303
"KernelTmaWarpSpecializedCooperativeMixedInput_": "TmaMI_",
304
"TmaWarpSpecializedCooperative_": "TmaCoop_",
305
"StreamKScheduler": "streamK",
306
}
307
220
- schedule_name = generate_schedule_name(schedule_config)
308
+ sch_sig = generate_sch_sig(schedule_config)
309
for orig, terse in kernel_terse_names_replace.items():
222
- schedule_name = schedule_name.replace(orig, terse)
223
- return schedule_name
310
+ sch_sig = sch_sig.replace(orig, terse)
311
+ return sch_sig
312
313
314
# unique type_name
227
-def generate_type_signature(kernel_type_config: TypeConfig):
228
- element_a = VLLMDataTypeNames[kernel_type_config.element_a]
229
- element_b = VLLMDataTypeNames[kernel_type_config.element_b]
230
- element_d = VLLMDataTypeNames[kernel_type_config.element_d]
231
- accumulator = VLLMDataTypeNames[kernel_type_config.accumulator]
232
- element_scale = VLLMDataTypeNames[kernel_type_config.element_b_scale]
233
- element_zeropoint = VLLMDataTypeNames[
234
- kernel_type_config.element_b_zeropoint]
235
-
236
- return (f"{element_a}{element_b}{element_d}"
237
- f"{accumulator}{element_scale}{element_zeropoint}")
238
-
315
+def generate_type_signature(kernel_types: TypeConfig):
316
+ return str("".join([
317
+ VLLMDataTypeNames[getattr(kernel_types, field.name)]
318
+ for field in fields(TypeConfig)
319
+ ]))
320
240
-# non-unique shorter type_name
241
-def generate_terse_type_signature(kernel_type_config: TypeConfig):
242
- element_a = VLLMDataTypeNames[kernel_type_config.element_a]
243
- element_b = VLLMDataTypeNames[kernel_type_config.element_b]
321
245
- return f"{element_a}{element_b}"
322
+def generate_type_option_name(kernel_types: TypeConfig):
323
+ return ", ".join([
324
+ f"{field.name.replace('b_', 'with_')+'_type'}=" +
325
+ VLLMDataTypeNames[getattr(kernel_types, field.name)]
326
+ for field in fields(TypeConfig)
327
+ ])
328
329
330
def is_power_of_two(n):
@@ -263,13 +345,36 @@ def to_cute_constant(value: List[int]):
345
return _to_cute_constant(value)
346
347
348
+def unique_schedules(impl_configs: List[ImplConfig]):
349
+ return list(
350
+ set(sch for impl_config in impl_configs
351
+ for sch in impl_config.schedules))
352
+
353
+
354
+def unsigned_type_with_bitwidth(num_bits):
355
+ return {
356
+ 4: DataType.u4,
357
+ 8: DataType.u8,
358
+ 16: DataType.u16,
359
+ 32: DataType.u32,
360
+ 64: DataType.u64,
361
+ }[num_bits]
362
+
363
+
364
template_globals = {
365
+ "void": DataType.void,
366
"DataTypeTag": VLLMDataTypeTag,
367
+ "VLLMScalarTypeTag": VLLMDataTypeVLLMScalarTypeTag,
368
+ "TorchTypeTag": VLLMDataTypeTorchDataTypeTag,
369
"KernelScheduleTag": VLLMKernelScheduleTag,
370
"EpilogueScheduleTag": EpilogueScheduleTag,
371
"TileSchedulerTag": TileSchedulerTag,
372
"to_cute_constant": to_cute_constant,
272
- "gen_sch_name": generate_terse_schedule_name,
373
+ "gen_sch_sig": generate_terse_sch_sig,
374
+ "gen_type_sig": generate_type_signature,
375
+ "unique_schedules": unique_schedules,
376
+ "unsigned_type_with_bitwidth": unsigned_type_with_bitwidth,
377
+ "gen_type_option_name": generate_type_option_name
378
}
379
380
@@ -284,42 +389,82 @@ mm_impl_template = create_template(IMPL_TEMPLATE)
389
prepack_dispatch_template = create_template(PREPACK_TEMPLATE)
390
391
287
-def create_sources(impl_config: ImplConfig, num_impl_files=1):
392
+def create_sources(impl_configs: List[ImplConfig], num_impl_files=8):
393
sources = []
394
290
- type_name = generate_type_signature(impl_config.type_config)
291
- terse_type_name = generate_terse_type_signature(impl_config.type_config)
292
-
395
sources.append((
294
- f"machete_mm_{terse_type_name}",
295
- mm_dispatch_template.render(type_name=type_name,
296
- type_config=impl_config.type_config,
297
- schedules=impl_config.schedule_configs,
298
- heuristic=impl_config.heuristic),
396
+ "machete_mm_dispatch",
397
+ mm_dispatch_template.render(impl_configs=impl_configs),
398
))
399
400
+ prepack_types = []
401
+ for impl_config in impl_configs:
402
+ convert_type = impl_config.types.a \
403
+ if impl_config.types.b_group_scale == DataType.void \
404
+ else impl_config.types.b_group_scale
405
+ prepack_types.append(
406
+ PrepackTypeConfig(
407
+ a=impl_config.types.a,
408
+ b_num_bits=VLLMDataTypeSize[impl_config.types.b],
409
+ convert=convert_type,
410
+ accumulator=impl_config.types.accumulator,
411
+ ))
412
+
413
+ def prepacked_type_key(prepack_type: PrepackTypeConfig):
414
+ # For now we we can just use the first accumulator type seen since
415
+ # the tensor core shapes/layouts don't vary based on accumulator
416
+ # type so we can generate less code this way
417
+ return (prepack_type.a, prepack_type.b_num_bits, prepack_type.convert)
418
+
419
+ unique_prepack_types = []
420
+ prepack_types_seen = set()
421
+ for prepack_type in prepack_types:
422
+ key = prepacked_type_key(prepack_type)
423
+ if key not in prepack_types_seen:
424
+ unique_prepack_types.append(prepack_type)
425
+ prepack_types_seen.add(key)
426
+
427
sources.append((
302
- f"machete_prepack_{terse_type_name}",
303
- prepack_dispatch_template.render(
304
- type_name=type_name,
305
- type_config=impl_config.type_config,
306
- ),
428
+ "machete_prepack",
429
+ prepack_dispatch_template.render(types=unique_prepack_types, ),
430
))
431
309
- num_schedules = len(impl_config.schedule_configs)
310
- schedules_per_file = math.ceil(num_schedules / num_impl_files)
311
- for part, i in enumerate(range(0, num_schedules, schedules_per_file)):
312
- file_schedules = impl_config.schedule_configs[i:i + schedules_per_file]
432
+ # Split up impls across files
433
+ num_impls = reduce(lambda x, y: x + len(y.schedules), impl_configs, 0)
434
+ num_impls_per_file = math.ceil(num_impls / num_impl_files)
435
+
436
+ files_impls: List[List[ImplConfig]] = [[]]
437
+
438
+ curr_num_impls_assigned = 0
439
+ curr_impl_in_file = 0
440
+ curr_impl_configs = deepcopy(list(reversed(impl_configs)))
441
+
442
+ while curr_num_impls_assigned < num_impls:
443
+ room_left_in_file = num_impls_per_file - curr_impl_in_file
444
+ if room_left_in_file == 0:
445
+ files_impls.append([])
446
+ room_left_in_file = num_impls_per_file
447
+ curr_impl_in_file = 0
448
+
449
+ curr_ic = curr_impl_configs[-1]
450
+ if len(curr_ic.schedules) >= room_left_in_file:
451
+ # Break apart the current impl config
452
+ tmp_ic = deepcopy(curr_ic)
453
+ tmp_ic.schedules = curr_ic.schedules[:room_left_in_file]
454
+ curr_ic.schedules = curr_ic.schedules[room_left_in_file:]
455
+ files_impls[-1].append(tmp_ic)
456
+ else:
457
+ files_impls[-1].append(curr_ic)
458
+ curr_impl_configs.pop()
459
+ curr_num_impls_assigned += len(files_impls[-1][-1].schedules)
460
+ curr_impl_in_file += len(files_impls[-1][-1].schedules)
461
462
+ for part, file_impls in enumerate(files_impls):
463
sources.append((
315
- f"machete_mm_{terse_type_name}_impl_part{part}",
316
- mm_impl_template.render(
317
- type_name=type_name,
318
- type_config=impl_config.type_config,
319
- schedules=file_schedules,
320
- specializations=impl_config.specializations,
321
- ),
464
+ f"machete_mm_impl_part{part+1}",
465
+ mm_impl_template.render(impl_configs=file_impls),
466
))
467
+
468
return sources
469
470
@@ -328,187 +473,169 @@ def generate():
473
# about how this works
474
SCRIPT_DIR = os.path.dirname(__file__)
475
331
- schedule_common_params = dict(
476
+ sch_common_params = dict(
477
kernel_schedule=TmaMI,
478
epilogue_schedule=TmaCoop,
479
tile_scheduler=TileSchedulerType.StreamK,
480
)
481
337
- # For now we use the same heuristic for all types
338
- # Heuristic is currently tuned for H100s
339
- default_heuristic = [
482
+ # Stored as "condition": ((tile_shape_mn), (cluster_shape_mnk))
483
+ default_tile_heuristic_config = {
484
#### M = 257+
341
- (
342
- "M > 256 && K <= 16384 && N <= 4096",
343
- ScheduleConfig(
344
- tile_shape_mn=(128, 128),
345
- cluster_shape_mnk=(2, 1, 1),
346
- **schedule_common_params # type: ignore
347
- )),
348
- (
349
- "M > 256",
350
- ScheduleConfig(
351
- tile_shape_mn=(128, 256),
352
- cluster_shape_mnk=(2, 1, 1),
353
- **schedule_common_params # type: ignore
354
- )),
485
+ "M > 256 && K <= 16384 && N <= 4096": ((128, 128), (2, 1, 1)),
486
+ "M > 256": ((128, 256), (2, 1, 1)),
487
#### M = 129-256
356
- (
357
- "M > 128 && K <= 4096 && N <= 4096",
358
- ScheduleConfig(
359
- tile_shape_mn=(128, 64),
360
- cluster_shape_mnk=(2, 1, 1),
361
- **schedule_common_params # type: ignore
362
- )),
363
- (
364
- "M > 128 && K <= 8192 && N <= 8192",
365
- ScheduleConfig(
366
- tile_shape_mn=(128, 128),
367
- cluster_shape_mnk=(2, 1, 1),
368
- **schedule_common_params # type: ignore
369
- )),
370
- (
371
- "M > 128",
372
- ScheduleConfig(
373
- tile_shape_mn=(128, 256),
374
- cluster_shape_mnk=(2, 1, 1),
375
- **schedule_common_params # type: ignore
376
- )),
488
+ "M > 128 && K <= 4096 && N <= 4096": ((128, 64), (2, 1, 1)),
489
+ "M > 128 && K <= 8192 && N <= 8192": ((128, 128), (2, 1, 1)),
490
+ "M > 128": ((128, 256), (2, 1, 1)),
491
#### M = 65-128
378
- (
379
- "M > 64 && K <= 4069 && N <= 4069",
380
- ScheduleConfig(
381
- tile_shape_mn=(128, 32),
382
- cluster_shape_mnk=(2, 1, 1),
383
- **schedule_common_params # type: ignore
384
- )),
385
- (
386
- "M > 64 && K <= 4069 && N <= 8192",
387
- ScheduleConfig(
388
- tile_shape_mn=(128, 64),
389
- cluster_shape_mnk=(2, 1, 1),
390
- **schedule_common_params # type: ignore
391
- )),
392
- (
393
- "M > 64 && K >= 8192 && N >= 12288",
394
- ScheduleConfig(
395
- tile_shape_mn=(256, 128),
396
- cluster_shape_mnk=(2, 1, 1),
397
- **schedule_common_params # type: ignore
398
- )),
399
- (
400
- "M > 64",
401
- ScheduleConfig(
402
- tile_shape_mn=(128, 128),
403
- cluster_shape_mnk=(2, 1, 1),
404
- **schedule_common_params # type: ignore
405
- )),
492
+ "M > 64 && K <= 4069 && N <= 4069": ((128, 32), (2, 1, 1)),
493
+ "M > 64 && K <= 4069 && N <= 8192": ((128, 64), (2, 1, 1)),
494
+ "M > 64 && K >= 8192 && N >= 12288": ((256, 128), (2, 1, 1)),
495
+ "M > 64": ((128, 128), (2, 1, 1)),
496
#### M = 33-64
407
- (
408
- "M > 32 && K <= 6144 && N <= 6144",
409
- ScheduleConfig(
410
- tile_shape_mn=(128, 16),
411
- cluster_shape_mnk=(1, 1, 1),
412
- **schedule_common_params # type: ignore
413
- )),
414
- (
415
- "M > 32 && K >= 16384 && N >= 12288",
416
- ScheduleConfig(
417
- tile_shape_mn=(256, 64),
418
- cluster_shape_mnk=(2, 1, 1),
419
- **schedule_common_params # type: ignore
420
- )),
421
- (
422
- "M > 32",
423
- ScheduleConfig(
424
- tile_shape_mn=(128, 64),
425
- cluster_shape_mnk=(2, 1, 1),
426
- **schedule_common_params # type: ignore
427
- )),
497
+ "M > 32 && K <= 6144 && N <= 6144": ((128, 16), (1, 1, 1)),
498
+ "M > 32 && K >= 16384 && N >= 12288": ((256, 64), (2, 1, 1)),
499
+ "M > 32": ((128, 64), (2, 1, 1)),
500
#### M = 17-32
429
- (
430
- "M > 16 && K <= 12288 && N <= 8192",
431
- ScheduleConfig(
432
- tile_shape_mn=(128, 32),
433
- cluster_shape_mnk=(2, 1, 1),
434
- **schedule_common_params # type: ignore
435
- )),
436
- (
437
- "M > 16",
438
- ScheduleConfig(
439
- tile_shape_mn=(256, 32),
440
- cluster_shape_mnk=(2, 1, 1),
441
- **schedule_common_params # type: ignore
442
- )),
501
+ "M > 16 && K <= 12288 && N <= 8192": ((128, 32), (2, 1, 1)),
502
+ "M > 16": ((256, 32), (2, 1, 1)),
503
#### M = 1-16
444
- (
445
- "N >= 26624",
446
- ScheduleConfig(
447
- tile_shape_mn=(256, 16),
448
- cluster_shape_mnk=(1, 1, 1),
449
- **schedule_common_params # type: ignore
450
- )),
451
- (
452
- None,
453
- ScheduleConfig(
454
- tile_shape_mn=(128, 16),
455
- cluster_shape_mnk=(1, 1, 1),
456
- **schedule_common_params # type: ignore
457
- )),
504
+ "N >= 26624": ((256, 16), (1, 1, 1)),
505
+ None: ((128, 16), (1, 1, 1)),
506
+ }
507
+
508
+ # For now we use the same heuristic for all types
509
+ # Heuristic is currently tuned for H100s
510
+ default_heuristic = [
511
+ (cond, ScheduleConfig(*tile_config,
512
+ **sch_common_params)) # type: ignore
513
+ for cond, tile_config in default_tile_heuristic_config.items()
514
]
515
460
- # Do not use schedules = list(set(...)) because we need to make sure
461
- # the output list is deterministic; otherwise the generated kernel file
462
- # will be non-deterministic and causes ccache miss.
463
- schedules = []
464
- for _, schedule_config in default_heuristic:
465
- if schedule_config not in schedules:
466
- schedules.append(schedule_config)
516
+ def get_unique_schedules(heuristic: Dict[str, ScheduleConfig]):
517
+ # Do not use schedules = list(set(...)) because we need to make sure
518
+ # the output list is deterministic; otherwise the generated kernel file
519
+ # will be non-deterministic and causes ccache miss.
520
+ schedules = []
521
+ for _, schedule_config in heuristic:
522
+ if schedule_config not in schedules:
523
+ schedules.append(schedule_config)
524
+ return schedules
525
526
impl_configs = []
527
528
GPTQ_kernel_type_configs = list(
529
TypeConfig(
472
- element_a=element_a,
473
- element_b=element_b,
474
- element_b_scale=element_a,
475
- element_b_zeropoint=element_a,
476
- element_d=element_a,
530
+ a=a,
531
+ b=b,
532
+ b_group_scale=a,
533
+ b_group_zeropoint=DataType.void,
534
+ b_channel_scale=DataType.void,
535
+ a_token_scale=DataType.void,
536
+ out=a,
537
accumulator=DataType.f32,
478
- ) for element_b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
479
- for element_a in (DataType.f16, DataType.bf16))
480
-
481
- GPTQ_kernel_specializations = [
482
- Specialization(with_C=False, with_zeropoints=False, with_scales=True)
483
- ]
538
+ ) for b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
539
+ for a in (DataType.f16, DataType.bf16))
540
541
impl_configs += [
486
- ImplConfig(x[0], x[1], x[2], x[3])
487
- for x in zip(GPTQ_kernel_type_configs, itertools.repeat(schedules),
488
- itertools.repeat(GPTQ_kernel_specializations),
542
+ ImplConfig(x[0], x[1], x[2])
543
+ for x in zip(GPTQ_kernel_type_configs,
544
+ itertools.repeat(get_unique_schedules(default_heuristic)),
545
itertools.repeat(default_heuristic))
546
]
547
548
AWQ_kernel_type_configs = list(
549
TypeConfig(
494
- element_a=element_a,
495
- element_b=element_b,
496
- element_b_scale=element_a,
497
- element_b_zeropoint=element_a,
498
- element_d=element_a,
550
+ a=a,
551
+ b=b,
552
+ b_group_scale=a,
553
+ b_group_zeropoint=a,
554
+ b_channel_scale=DataType.void,
555
+ a_token_scale=DataType.void,
556
+ out=a,
557
accumulator=DataType.f32,
500
- ) for element_b in (DataType.u4, DataType.u8)
501
- for element_a in (DataType.f16, DataType.bf16))
558
+ ) for b in (DataType.u4, DataType.u8)
559
+ for a in (DataType.f16, DataType.bf16))
560
+
561
+ impl_configs += [
562
+ ImplConfig(x[0], x[1], x[2])
563
+ for x in zip(AWQ_kernel_type_configs,
564
+ itertools.repeat(get_unique_schedules(default_heuristic)),
565
+ itertools.repeat(default_heuristic))
566
+ ]
567
503
- AWQ_kernel_specializations = [
504
- Specialization(with_C=False, with_zeropoints=True, with_scales=True)
568
+ # Stored as "condition": ((tile_shape_mn), (cluster_shape_mnk))
569
+ # TODO (LucasWilkinson): Further tuning required
570
+ qqq_tile_heuristic_config = {
571
+ #### M = 257+
572
+ # ((128, 256), (2, 1, 1)) Broken for QQQ types
573
+ # TODO (LucasWilkinson): Investigate further
574
+ # "M > 256 && K <= 16384 && N <= 4096": ((128, 128), (2, 1, 1)),
575
+ # "M > 256": ((128, 256), (2, 1, 1)),
576
+ "M > 256": ((128, 128), (2, 1, 1)),
577
+ #### M = 129-256
578
+ "M > 128 && K <= 4096 && N <= 4096": ((128, 64), (2, 1, 1)),
579
+ "M > 128 && K <= 8192 && N <= 8192": ((128, 128), (2, 1, 1)),
580
+ # ((128, 256), (2, 1, 1)) Broken for QQQ types
581
+ # TODO (LucasWilkinson): Investigate further
582
+ # "M > 128": ((128, 256), (2, 1, 1)),
583
+ "M > 128": ((128, 128), (2, 1, 1)),
584
+ #### M = 65-128
585
+ "M > 64 && K <= 4069 && N <= 4069": ((128, 32), (2, 1, 1)),
586
+ "M > 64 && K <= 4069 && N <= 8192": ((128, 64), (2, 1, 1)),
587
+ "M > 64 && K >= 8192 && N >= 12288": ((256, 128), (2, 1, 1)),
588
+ "M > 64": ((128, 128), (2, 1, 1)),
589
+ #### M = 33-64
590
+ "M > 32 && K <= 6144 && N <= 6144": ((128, 16), (1, 1, 1)),
591
+ # Broken for QQQ types
592
+ # TODO (LucasWilkinson): Investigate further
593
+ #"M > 32 && K >= 16384 && N >= 12288": ((256, 64), (2, 1, 1)),
594
+ "M > 32": ((128, 64), (2, 1, 1)),
595
+ #### M = 17-32
596
+ "M > 16 && K <= 12288 && N <= 8192": ((128, 32), (2, 1, 1)),
597
+ "M > 16": ((256, 32), (2, 1, 1)),
598
+ #### M = 1-16
599
+ "N >= 26624": ((256, 16), (1, 1, 1)),
600
+ None: ((128, 16), (1, 1, 1)),
601
+ }
602
+
603
+ # For now we use the same heuristic for all types
604
+ # Heuristic is currently tuned for H100s
605
+ qqq_heuristic = [
606
+ (cond, ScheduleConfig(*tile_config,
607
+ **sch_common_params)) # type: ignore
608
+ for cond, tile_config in qqq_tile_heuristic_config.items()
609
+ ]
610
+
611
+ QQQ_kernel_types = [
612
+ *(TypeConfig(
613
+ a=DataType.s8,
614
+ b=VLLMDataType.u4b8,
615
+ b_group_scale=b_group_scale,
616
+ b_group_zeropoint=DataType.void,
617
+ b_channel_scale=DataType.f32,
618
+ a_token_scale=DataType.f32,
619
+ out=DataType.f16,
620
+ accumulator=DataType.s32,
621
+ ) for b_group_scale in (DataType.f16, DataType.void)),
622
+ *(TypeConfig(
623
+ a=DataType.e4m3,
624
+ b=VLLMDataType.u4b8,
625
+ b_group_scale=b_group_scale,
626
+ b_group_zeropoint=DataType.void,
627
+ b_channel_scale=DataType.f32,
628
+ a_token_scale=DataType.f32,
629
+ out=DataType.f16,
630
+ accumulator=DataType.f32,
631
+ ) for b_group_scale in (DataType.f16, DataType.void)),
632
]
633
634
impl_configs += [
508
- ImplConfig(x[0], x[1], x[2], x[3])
509
- for x in zip(AWQ_kernel_type_configs, itertools.repeat(schedules),
510
- itertools.repeat(AWQ_kernel_specializations),
511
- itertools.repeat(default_heuristic))
635
+ ImplConfig(x[0], x[1], x[2])
636
+ for x in zip(QQQ_kernel_types,
637
+ itertools.repeat(get_unique_schedules(qqq_heuristic)),
638
+ itertools.repeat(qqq_heuristic))
639
]
640
641
output_dir = os.path.join(SCRIPT_DIR, "generated")
@@ -521,12 +648,11 @@ def generate():
648
os.makedirs(output_dir)
649
650
# Render each group of configurations into separate files
524
- for impl_config in impl_configs:
525
- for filename, code in create_sources(impl_config):
526
- filepath = os.path.join(output_dir, f"{filename}.cu")
527
- with open(filepath, "w") as output_file:
528
- output_file.write(code)
529
- print(f"Rendered template to {filepath}")
651
+ for filename, code in create_sources(impl_configs):
652
+ filepath = os.path.join(output_dir, f"{filename}.cu")
653
+ with open(filepath, "w") as output_file:
654
+ output_file.write(code)
655
+ print(f"Rendered template to {filepath}")
656
657
658
if __name__ == "__main__":
csrc/quantization/machete/machete_mainloop.cuh
+11
-14
@@ -171,6 +171,10 @@ struct MacheteCollectiveMma {
171
make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
172
Int<DispatchPolicy::Stages>{})));
173
174
+ using SmemLayoutACopy = decltype(GmemLayoutA::TVbNbKL_to_offset_copy(
175
+ make_shape(size<0>(TileShape_MNK{}), size<2>(TileShape_MNK{}),
176
+ Int<DispatchPolicy::Stages>{})));
177
+
178
using SmemLayoutAtomARowMajor =
179
decltype(rs_smem_selector<GmmaMajorA, ElementA,
180
decltype(cute::get<0>(TileShape_MNK{})),
@@ -288,14 +292,7 @@ struct MacheteCollectiveMma {
292
static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomScale{})) == 0,
293
"SmemLayoutAtomScale must evenly divide tile k shape.");
294
291
- // Tile along modes in a way that maximizes the TMA box size.
292
- using SmemLayoutACopy = decltype(tile_to_shape(
293
- SmemLayoutAtomARowMajor{},
294
- make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}),
295
- Int<DispatchPolicy::Stages>{}),
296
- conditional_t<::cutlass::gemm::detail::is_major<0, StrideA>(),
297
- Step<_2, _1, _3>, Step<_1, _2, _3>>{}));
298
-
295
+ // Tile along modes in a way that maximizes the TMA box size
296
using SmemLayoutB = decltype(tile_to_shape(
297
SmemLayoutAtomB{},
298
make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}),
@@ -428,12 +425,12 @@ struct MacheteCollectiveMma {
425
// clang-format on
426
427
// ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
431
- using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset(
428
+ using PrepackedStrideA = decltype(stride(GmemLayoutA::TVbNbKL_to_offset_copy(
429
make_shape(int32_t(0), int32_t(0), int32_t(0)))));
430
431
using ATensor = decltype(make_tensor(
432
get_logical_ptr(static_cast<InternalElementA const*>(nullptr)),
436
- shape(GmemLayoutA::TVbNbKL_to_offset(
433
+ shape(GmemLayoutA::TVbNbKL_to_offset_copy(
434
make_shape(int32_t(0), int32_t(0), int32_t(0)))),
435
PrepackedStrideA{}));
436
@@ -450,8 +447,8 @@ struct MacheteCollectiveMma {
447
448
static constexpr auto make_tma_copy_A(ATensor tensor_a = ATensor{}) {
449
return make_tma_copy<TmaElementA>(
453
- GmemTiledCopyA{}, tensor_a, SmemLayoutA{}(_, _, cute::Int<0>{}),
454
- shape(SmemLayoutA{}(_, _, cute::Int<0>{})),
450
+ GmemTiledCopyA{}, tensor_a, SmemLayoutACopy{}(_, _, cute::Int<0>{}),
451
+ shape(SmemLayoutACopy{}(_, _, cute::Int<0>{})),
452
size<1>(ClusterShape{})); // mcast along N mode for this M load, if any
453
}
454
@@ -584,7 +581,7 @@ struct MacheteCollectiveMma {
581
typename Params::TMA_Scale tma_load_scale;
582
typename Params::TMA_Zero tma_load_zero;
583
587
- auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
584
+ auto layout = GmemLayoutA::TVbNbKL_to_offset_copy(make_shape(M, K, L));
585
tma_load_a = make_tma_copy_A(
586
make_logical_tensor(ptr_A, shape(layout), stride(layout)));
587
@@ -722,7 +719,7 @@ struct MacheteCollectiveMma {
719
// (TILE_V,TILE_B,m,k,l)
720
auto make_gA_mkl = [&]() {
721
// ((athrid, val), (BlocksM, BlockK), L) -> (storage_idx)
725
- auto layout = GmemLayoutA::TVbNbKL_to_offset(make_shape(M, K, L));
722
+ auto layout = GmemLayoutA::TVbNbKL_to_offset_copy(make_shape(M, K, L));
723
Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(shape(layout));
724
return local_tile(mA_mkl,
725
make_shape(size<0>(layout), PPBlocksPerTile_MK{}),
csrc/quantization/machete/machete_mm_kernel.cuh
+141
-65
@@ -21,6 +21,8 @@
21
22
#include "cutlass_extensions/cute_utils.cuh"
23
#include "cutlass_extensions/vllm_numeric_conversion.cuh"
24
+#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
25
+#include "cutlass_extensions/torch_utils.hpp"
26
#include "machete_collective_builder.cuh"
27
#include "machete_prepacked_layout.cuh"
28
#include "machete_interleaving_utils.cuh"
@@ -37,27 +39,42 @@ using namespace cute;
39
// W is quantized, in this situation or right-hand operand is quantized so
40
// we compute the transpose to move it to the left-hand side.
41
template <typename ElementA_, typename ElementB_, typename ElementD_,
40
- typename AccumulatorT, typename ScaleT, typename ZeroT,
41
- class KernelSchedule, typename ScheduleConfig, bool with_C,
42
- bool with_scales, bool with_zeropoints>
42
+ typename AccumulatorT, typename GroupScaleT, typename GroupZeroT,
43
+ typename ChannelScaleT, typename TokenScaleT, class KernelSchedule,
44
+ typename ScheduleConfig>
45
struct MacheteKernelTemplate {
46
+ static constexpr bool with_C = false; // not ever used
47
+ static constexpr bool with_group_scales = !std::is_same_v<GroupScaleT, void>;
48
+ static constexpr bool with_group_zeropoints =
49
+ !std::is_same_v<GroupZeroT, void>;
50
+ static constexpr bool with_channel_scales =
51
+ !std::is_same_v<ChannelScaleT, void>;
52
+ static constexpr bool with_token_scales = !std::is_same_v<TokenScaleT, void>;
53
+
54
using MmaType = ElementA_;
55
using ElementA = ElementA_;
56
using ElementB = ElementB_;
57
using ElementD = ElementD_;
58
using ElementC = cute::conditional_t<with_C, ElementD, void>;
49
- using ElementZ = ZeroT;
50
- using ElementS = ScaleT;
51
-
52
- using ElementAccumulator =
53
- AccumulatorT; // Element type for internal accumulation
59
+ using ElementAccumulator = AccumulatorT;
60
using ElementCompute = AccumulatorT; // For Epilogue
61
+ // Use dummy values when we don't have scales or zeropoints
62
+ using ElementZGroup =
63
+ cute::conditional_t<with_group_zeropoints, GroupZeroT, MmaType>;
64
+ using ElementSGroup =
65
+ cute::conditional_t<with_group_scales, GroupScaleT, MmaType>;
66
+ using ElementConvertGroup =
67
+ cute::conditional_t<with_group_scales, GroupScaleT, MmaType>;
68
+ using ElementSChannel =
69
+ cute::conditional_t<with_channel_scales, ChannelScaleT, AccumulatorT>;
70
+ using ElementSToken =
71
+ cute::conditional_t<with_token_scales, TokenScaleT, AccumulatorT>;
72
73
using BTypeTuple = cute::conditional_t<
57
- with_scales,
58
- cute::conditional_t<with_zeropoints,
59
- cute::tuple<ElementB, ElementS, ElementZ>,
60
- cute::tuple<ElementB, ElementS>>,
74
+ with_group_scales,
75
+ cute::conditional_t<with_group_zeropoints,
76
+ cute::tuple<ElementB, ElementSGroup, ElementZGroup>,
77
+ cute::tuple<ElementB, ElementSGroup>>,
78
ElementB>;
79
80
using LayoutA = cutlass::layout::RowMajor;
@@ -71,8 +88,8 @@ struct MacheteKernelTemplate {
88
using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
89
using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
90
using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
74
- using StrideS = cutlass::detail::TagToStrideA_t<LayoutScale>;
75
- using StrideZ = StrideS;
91
+ using StrideSGroup = cutlass::detail::TagToStrideA_t<LayoutScale>;
92
+ using StrideZGroup = StrideSGroup;
93
94
using LayoutA_Transpose =
95
typename cutlass::layout::LayoutTranspose<LayoutA>::type;
@@ -85,8 +102,8 @@ struct MacheteKernelTemplate {
102
using OperatorClass = cutlass::arch::OpClassTensorOp;
103
104
using PrepackedLayoutB =
88
- PrepackedLayoutBTemplate<ElementA_, ElementB_, ElementD_, AccumulatorT,
89
- LayoutA_Transpose, KernelSchedule>;
105
+ PrepackedLayoutBTemplate<ElementA_, ElementB_, ElementConvertGroup,
106
+ AccumulatorT, LayoutA_Transpose, KernelSchedule>;
107
108
static int constexpr TileShapeK =
109
128 * 8 / cutlass::sizeof_bits<MmaType>::value;
@@ -103,12 +120,42 @@ struct MacheteKernelTemplate {
120
using EpilogueTileType = typename ScheduleConfig::EpilogueTileType;
121
using TileScheduler = typename ScheduleConfig::TileScheduler;
122
123
+ static_assert(
124
+ (!with_channel_scales && !with_token_scales) ||
125
+ ((with_channel_scales && with_token_scales) &&
126
+ std::is_same_v<ElementSChannel, ElementSToken>),
127
+ "Currently token and channel scales (if present) must be the same type");
128
+
129
+ using EpilogueDescriptor =
130
+ cutlass::epilogue::collective::detail::EpilogueDescriptor<
131
+ TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
132
+ ElementD, EpilogueSchedule>;
133
+
134
+ // Currently only supports float scales
135
+ using ChTokScalesEpilogue =
136
+ typename vllm::c3x::ScaledEpilogue<ElementAccumulator, ElementD,
137
+ EpilogueDescriptor>;
138
+ static_assert((with_channel_scales || with_token_scales) ||
139
+ (std::is_same_v<ElementSChannel, float> &&
140
+ std::is_same_v<ElementSToken, float>),
141
+ "Currently token and channel scales (if present) must be float "
142
+ "(and if one is present the other must be too)");
143
+
144
+ using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT<
145
+ cutlass::epilogue::fusion::Sm90AccFetch>;
146
+
147
+ using EVTCompute =
148
+ std::conditional_t<with_channel_scales || with_token_scales,
149
+ typename ChTokScalesEpilogue::EVTCompute,
150
+ StoreEpilogueCompute>;
151
+
152
+ // EVTCompute
153
using CollectiveEpilogue =
154
typename cutlass::epilogue::collective::CollectiveBuilder<
155
ArchTag, OperatorClass, TileShape, ClusterShape, EpilogueTileType,
109
- ElementAccumulator, ElementAccumulator, ElementC, LayoutC_Transpose,
110
- AlignmentC, ElementD, LayoutD_Transpose, AlignmentD,
111
- EpilogueSchedule>::CollectiveOp;
156
+ ElementAccumulator, ElementSChannel, ElementC, LayoutC_Transpose,
157
+ AlignmentC, ElementD, LayoutD_Transpose, AlignmentD, EpilogueSchedule,
158
+ EVTCompute>::CollectiveOp;
159
160
using CollectiveMainloop =
161
typename cutlass::gemm::collective::VLLMCollectiveBuilder<
@@ -131,26 +178,44 @@ struct MacheteKernelTemplate {
178
using MainloopArguments = typename GemmKernel::MainloopArguments;
179
using EpilogueArguments = typename GemmKernel::EpilogueArguments;
180
134
- template <typename ShapeA, typename ShapeC, typename ShapeD, typename ShapeS,
135
- typename ShapeZ>
181
static Arguments create_arguments(
182
cudaStream_t stream,
138
- ElementA const* A_ptr, // A is an MxK matrix
139
- Layout<ShapeA, StrideA> const& layout_A,
140
- ElementB const* B_ptr, // B is an KxN prepacked matrix
141
- ElementD* D_ptr, // D is an MxN matrix
142
- Layout<ShapeD, StrideD> const& layout_D,
143
- ElementC const* C_ptr, // C is an MxN matrix
144
- std::optional<Layout<ShapeC, StrideC>> const& layout_C,
145
- ElementS const* S_ptr, // S is an scale_KxN matrix
146
- std::optional<Layout<ShapeS, StrideS>> const& layout_S,
147
- ElementZ const* Z_ptr, // Z is an scale_KxN matrix
148
- std::optional<Layout<ShapeZ, StrideZ>> const& layout_Z,
149
- ElementCompute alpha, ElementCompute beta,
150
- std::optional<int> maybe_group_size) {
151
- static_assert(!with_zeropoints || with_scales);
152
-
153
- int M = size<0>(layout_A), N = size<1>(layout_D), K = size<1>(layout_A);
183
+ torch::Tensor const& A, // MxK matrix
184
+ torch::Tensor const& B, // KxN prepacked matrix
185
+ torch::Tensor& D, // MxN matrix
186
+ c10::optional<torch::Tensor> const& maybe_g_scales, // scale_KxN matrix
187
+ c10::optional<torch::Tensor> const& maybe_g_zeros, // scale_KxN matrix
188
+ c10::optional<int64_t> maybe_group_size,
189
+ c10::optional<torch::Tensor> const& maybe_ch_scales, // len N vector
190
+ c10::optional<torch::Tensor> const& maybe_tok_scales) // len M vector
191
+ {
192
+ static_assert(!with_group_zeropoints || with_group_scales);
193
+
194
+ int M = A.size(0), N = B.size(1), K = A.size(1);
195
+ TORCH_CHECK(D.size(0) == M && D.size(1) == N);
196
+
197
+ auto layout_A = make_cute_layout<StrideA>(A, "A");
198
+ auto layout_D = make_cute_layout<StrideD>(D, "D");
199
+ auto layout_S_group =
200
+ maybe_make_cute_layout<StrideSGroup>(maybe_g_scales, "group_scales");
201
+ auto layout_Z_group =
202
+ maybe_make_cute_layout<StrideZGroup>(maybe_g_zeros, "group_zeros");
203
+ int64_t numel_S_channel = maybe_ch_scales ? maybe_ch_scales->numel() : 0;
204
+ int64_t numel_S_token = maybe_tok_scales ? maybe_tok_scales->numel() : 0;
205
+
206
+ auto unwrap = [](auto const& t) {
207
+ return t ? t->const_data_ptr() : nullptr;
208
+ };
209
+ auto A_ptr = static_cast<ElementA const*>(A.const_data_ptr());
210
+ auto B_ptr = static_cast<ElementB const*>(B.const_data_ptr());
211
+ auto D_ptr = static_cast<ElementD*>(D.mutable_data_ptr());
212
+ auto S_group_ptr =
213
+ static_cast<ElementSGroup const*>(unwrap(maybe_g_scales));
214
+ auto Z_group_ptr = static_cast<ElementZGroup const*>(unwrap(maybe_g_zeros));
215
+ auto S_channel_ptr =
216
+ static_cast<ElementSChannel const*>(unwrap(maybe_ch_scales));
217
+ auto S_token_ptr =
218
+ static_cast<ElementSToken const*>(unwrap(maybe_tok_scales));
219
220
int const group_size =
221
maybe_group_size == -1 ? K : maybe_group_size.value_or(K);
@@ -159,26 +224,28 @@ struct MacheteKernelTemplate {
224
TORCH_CHECK(size<0>(layout_A) == M && size<1>(layout_A) == K);
225
TORCH_CHECK(size<0>(layout_D) == M && size<1>(layout_D) == N);
226
162
- if constexpr (with_C) {
163
- TORCH_CHECK(C_ptr && layout_C);
227
+ if constexpr (with_group_scales) {
228
+ TORCH_CHECK(S_group_ptr && layout_S_group);
229
+ TORCH_CHECK((size<0>(*layout_S_group) == scale_k &&
230
+ size<1>(*layout_S_group) == N));
231
} else {
165
- TORCH_CHECK(!C_ptr, "C not supported");
232
+ TORCH_CHECK(!S_group_ptr, "Scales not supported");
233
}
234
168
- if constexpr (with_scales) {
169
- TORCH_CHECK(S_ptr && layout_S);
170
- TORCH_CHECK((size<0>(*layout_S) == scale_k && size<1>(*layout_S) == N));
235
+ if constexpr (with_group_zeropoints) {
236
+ TORCH_CHECK(Z_group_ptr && layout_Z_group);
237
+ TORCH_CHECK((size<0>(*layout_Z_group) == scale_k &&
238
+ size<1>(*layout_Z_group) == N));
239
+ TORCH_CHECK(layout_S_group && *layout_Z_group == *layout_S_group,
240
+ "Scales and zeros must have the same layout");
241
} else {
172
- TORCH_CHECK(!S_ptr, "Scales not supported");
242
+ TORCH_CHECK(!Z_group_ptr, "Zeropoints not supported");
243
}
244
175
- if constexpr (with_zeropoints) {
176
- TORCH_CHECK(Z_ptr && layout_Z);
177
- TORCH_CHECK((size<0>(*layout_Z) == scale_k && size<1>(*layout_Z) == N));
178
- TORCH_CHECK(layout_S && *layout_Z == *layout_S,
179
- "Scales and zeros must have the same layout");
180
- } else {
181
- TORCH_CHECK(!Z_ptr, "Zeropoints not supported");
245
+ if constexpr (with_channel_scales || with_token_scales) {
246
+ TORCH_CHECK(
247
+ (maybe_ch_scales->numel() == N || maybe_ch_scales->numel() == 1) &&
248
+ (maybe_tok_scales->numel() == M || maybe_tok_scales->numel() == 1));
249
}
250
251
// Transpose A and D
@@ -186,24 +253,33 @@ struct MacheteKernelTemplate {
253
// for B (which is At)
254
auto stride_At = layout_A.stride();
255
auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
189
- auto stride_Ct = stride_Dt;
190
- if (layout_C) {
191
- stride_Ct = permute_layout<1, 0, 2>(*layout_C).stride();
192
- }
256
257
MainloopArguments mainloop_arguments{};
195
- EpilogueArguments epilogue_arguments{
196
- {alpha, beta}, C_ptr, stride_Ct, D_ptr, stride_Dt};
258
+ // {Accum, C, C_layout, D, D}
259
+ EpilogueArguments epilogue_arguments{};
260
+
261
+ if constexpr (with_channel_scales || with_token_scales) {
262
+ epilogue_arguments =
263
+ EpilogueArguments{ChTokScalesEpilogue::prepare_args(
264
+ *maybe_ch_scales, *maybe_tok_scales),
265
+ nullptr,
266
+ {},
267
+ D_ptr,
268
+ stride_Dt};
269
+ } else {
270
+ epilogue_arguments = EpilogueArguments{{}, nullptr, {}, D_ptr, stride_Dt};
271
+ }
272
198
- if constexpr (with_scales && with_zeropoints) {
199
- auto stride_S = permute_layout<1, 0, 2>(*layout_S).stride();
200
- mainloop_arguments =
201
- MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At,
202
- S_ptr, stride_S, group_size, Z_ptr};
203
- } else if constexpr (with_scales) {
204
- auto stride_S = permute_layout<1, 0, 2>(*layout_S).stride();
273
+ if constexpr (with_group_scales && with_group_zeropoints) {
274
+ auto stride_S_group = permute_layout<1, 0, 2>(*layout_S_group).stride();
275
mainloop_arguments = MainloopArguments{
206
- B_ptr, _StrideB{}, A_ptr, stride_At, S_ptr, stride_S, group_size};
276
+ B_ptr, _StrideB{}, A_ptr, stride_At,
277
+ S_group_ptr, stride_S_group, group_size, Z_group_ptr};
278
+ } else if constexpr (with_group_scales) {
279
+ auto stride_S_group = permute_layout<1, 0, 2>(*layout_S_group).stride();
280
+ mainloop_arguments =
281
+ MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At,
282
+ S_group_ptr, stride_S_group, group_size};
283
} else {
284
mainloop_arguments =
285
MainloopArguments{B_ptr, _StrideB{}, A_ptr, stride_At};
csrc/quantization/machete/machete_mm_launcher.cuh
+35
-55
@@ -5,73 +5,61 @@
5
6
#include "machete_mm_kernel.cuh"
7
#include "cutlass_extensions/torch_utils.hpp"
8
+#include "core/scalar_type.hpp"
9
10
namespace machete {
11
11
-struct PyTorchArguments {
12
+struct MMArgs {
13
torch::Tensor const& A;
14
torch::Tensor const& B;
14
- c10::optional<torch::Tensor> const& scales;
15
- c10::optional<torch::Tensor> const& zeros;
16
- c10::optional<int64_t> group_size;
17
- c10::optional<torch::Tensor> const& C;
18
- c10::optional<double> alpha;
19
- c10::optional<double> beta;
20
- c10::optional<std::string> schedule;
15
+ vllm::ScalarType const& b_type;
16
+ c10::optional<at::ScalarType> const& maybe_out_type;
17
+ c10::optional<torch::Tensor> const& maybe_group_scales;
18
+ c10::optional<torch::Tensor> const& maybe_group_zeros;
19
+ c10::optional<int64_t> maybe_group_size;
20
+ c10::optional<torch::Tensor> const& maybe_channel_scales;
21
+ c10::optional<torch::Tensor> const& maybe_token_scales;
22
+ c10::optional<std::string> maybe_schedule;
23
};
24
25
+struct SupportedSchedulesArgs {
26
+ at::ScalarType a_type;
27
+ vllm::ScalarType b_type;
28
+ c10::optional<at::ScalarType> maybe_group_scales_type;
29
+ c10::optional<at::ScalarType> maybe_group_zeros_type;
30
+ c10::optional<at::ScalarType> maybe_channel_scales_type;
31
+ c10::optional<at::ScalarType> maybe_token_scales_type;
32
+ c10::optional<at::ScalarType> maybe_out_type;
33
+};
34
+
35
+torch::Tensor mm_dispatch(MMArgs args);
36
+
37
+std::vector<std::string> supported_schedules_dispatch(
38
+ SupportedSchedulesArgs args);
39
+
40
template <typename MacheteKernel>
24
-torch::Tensor run_impl(PyTorchArguments args) {
41
+torch::Tensor run_impl(MMArgs args) {
42
const at::cuda::OptionalCUDAGuard device_guard(device_of(args.A));
43
44
auto device = args.A.device();
45
auto stream = at::cuda::getCurrentCUDAStream(device.index());
46
30
- using EleA = typename MacheteKernel::ElementA;
31
- using EleB = typename MacheteKernel::ElementB;
32
- using EleC = typename MacheteKernel::ElementC;
33
- using EleD = typename MacheteKernel::ElementD;
34
- using EleScale = typename MacheteKernel::ElementS;
35
- using EleZero = typename MacheteKernel::ElementZ;
36
-
37
- using StrideA = typename MacheteKernel::StrideA;
38
- using StrideC = typename MacheteKernel::StrideC;
39
- using StrideD = typename MacheteKernel::StrideD;
40
- using StrideS = typename MacheteKernel::StrideS;
41
- using StrideZ = typename MacheteKernel::StrideZ;
42
-
47
int M = args.A.size(0);
48
int N = args.B.size(1);
49
int K = args.A.size(1);
50
51
// Allocate output
48
- torch::Tensor D =
49
- torch::empty({M, N}, torch::TensorOptions()
50
- .dtype(equivalent_scalar_type_v<EleD>)
51
- .device(device));
52
-
53
- auto const &A = args.A, &B = args.B;
54
- auto const &C = args.C, &scales = args.scales, &zeros = args.zeros;
55
-
56
- auto layout_A = make_cute_layout<StrideA>(A, "A");
57
- auto layout_D = make_cute_layout<StrideD>(D, "D");
58
- auto layout_C = maybe_make_cute_layout<StrideC>(C, "C");
59
- auto layout_S = maybe_make_cute_layout<StrideS>(scales, "scales");
60
- auto layout_Z = maybe_make_cute_layout<StrideZ>(zeros, "zeros");
61
-
62
- auto A_ptr = static_cast<EleA const*>(A.const_data_ptr());
63
- auto B_ptr = static_cast<EleB const*>(B.const_data_ptr());
64
- auto D_ptr = static_cast<EleD*>(D.mutable_data_ptr());
65
- auto C_ptr = static_cast<EleC const*>(C ? C->const_data_ptr() : nullptr);
66
- auto S_ptr =
67
- static_cast<EleScale const*>(scales ? scales->const_data_ptr() : nullptr);
68
- auto Z_ptr =
69
- static_cast<EleZero const*>(zeros ? zeros->const_data_ptr() : nullptr);
52
+ torch::Tensor D = torch::empty(
53
+ {M, N},
54
+ torch::TensorOptions()
55
+ .dtype(equivalent_scalar_type_v<typename MacheteKernel::ElementD>)
56
+ .device(device));
57
58
auto arguments = MacheteKernel::create_arguments(
72
- stream, A_ptr, layout_A, B_ptr, D_ptr, layout_D, C_ptr, layout_C, S_ptr,
73
- layout_S, Z_ptr, layout_Z, args.alpha.value_or(1), args.beta.value_or(0),
74
- args.group_size);
59
+ stream, //
60
+ args.A, args.B, D, args.maybe_group_scales, args.maybe_group_zeros,
61
+ args.maybe_group_size, args.maybe_channel_scales,
62
+ args.maybe_token_scales);
63
TORCH_CHECK(MacheteKernel::can_implement(arguments),
64
"Machete kernel cannot be run with these arguments");
65
@@ -84,12 +72,4 @@ torch::Tensor run_impl(PyTorchArguments args) {
72
return D;
73
};
74
87
-template <typename ElementA, typename ElementB, typename ElementD = ElementA,
88
- typename AccumulatorT = float, typename ScaleT = ElementA,
89
- typename ZeroT = ElementA>
90
-struct GemmDispatcher {
91
- static torch::Tensor dispatch(PyTorchArguments args);
92
- static std::vector<std::string> supported_schedules();
93
-};
94
-
75
}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepack_kernel.cuh
+39
-24
@@ -6,31 +6,49 @@
6
7
namespace machete {
8
9
-template <typename TileShapeNKL, typename ElementB, typename BInTensor,
10
- typename BTiledOutTensor>
11
-static __global__ void prepack_B_kernel(BInTensor B_in,
12
- BTiledOutTensor B_tiled_out) {
13
- auto tB_in = local_tile(B_in, TileShapeNKL{},
14
- make_coord(blockIdx.x, blockIdx.y, blockIdx.z));
15
- auto tB_out = B_tiled_out(make_coord(_, _),
16
- make_coord(blockIdx.x, blockIdx.y), blockIdx.z);
9
+template <int threads, typename PrepackedLayoutB, typename BInTensor,
10
+ typename ElementB>
11
+static __global__ void prepack_B_kernel(BInTensor B_in, ElementB* B_out_ptr) {
12
+ auto constexpr block_size =
13
+ Int<size(typename PrepackedLayoutB::PPBlockShape_NK{})>{};
14
+ auto constexpr eles_per_thread = Int<block_size / threads>{};
15
+ static_assert(block_size % threads == 0,
16
+ "block_size must be divisible by the number of threads");
17
18
- auto tiled_copy = make_tiled_copy(Copy_Atom<DefaultCopy, ElementB>{},
19
- Layout<Shape<_4, _32>, Stride<_32, _1>>{},
20
- Layout<Shape<_1, _2>>{});
18
+ // Which pre-packed are we responsible for
19
+ auto blk_coord = make_coord(blockIdx.x, blockIdx.y, blockIdx.z);
20
+ auto tB_in = local_tile(
21
+ B_in, append(typename PrepackedLayoutB::PPBlockShape_NK{}, _1{}),
22
+ blk_coord);
23
22
- auto thr_copy = tiled_copy.get_thread_slice(threadIdx.x);
24
+ // Find the start offset in the output for this pre-packed block
25
+ auto bNbKL_to_offset = PrepackedLayoutB::bNbKL_to_offset(shape(B_in));
26
24
- Tensor thr_tile_S = thr_copy.partition_S(tB_in);
25
- Tensor thr_tile_D = thr_copy.partition_D(tB_out);
27
+ // Tensor representing a 1:1 mapping to the output space in 1D
28
+ auto tB_out_linear =
29
+ make_tensor(get_logical_ptr(B_out_ptr) + bNbKL_to_offset(blk_coord),
30
+ make_layout(make_shape(block_size)));
31
+ // Mapping from output space (1D) to input space
32
+ auto tB_in_linear = make_tensor(
33
+ tB_in.data(),
34
+ tB_in.layout()
35
+ .compose(right_inverse(PrepackedLayoutB::ppblock_ilvd_NK_to_offset()))
36
+ .with_shape(make_shape(block_size)));
37
+
38
+ // Tile for this specific thread (could have used a TiledCopy but these work
39
+ // best with 2d layouts, this is a simple 1d layout so local_tile is enough,
40
+ // we are also not that concerned with performance for this kernel)
41
+ auto thr_tB_in_linear =
42
+ local_tile(tB_in_linear, make_shape(eles_per_thread), threadIdx.x);
43
+ auto thr_tB_out_linear =
44
+ local_tile(tB_out_linear, make_shape(eles_per_thread), threadIdx.x);
45
46
// Construct a register-backed Tensor with the same shape as each thread's
47
// partition
29
- auto fragment = make_tensor<ElementB>(shape(thr_tile_D));
48
+ auto fragment = make_tensor<ElementB>(shape(thr_tB_in_linear));
49
31
- // Copy from GMEM to RMEM and from RMEM to GMEM
32
- copy(tiled_copy, thr_tile_S, fragment);
33
- copy(Copy_Atom<DefaultCopy, uint8_t>{}, fragment, thr_tile_D);
50
+ copy(thr_tB_in_linear, fragment);
51
+ copy(Copy_Atom<DefaultCopy, uint8_t>{}, fragment, thr_tB_out_linear);
52
}
53
54
template <typename PrepackedLayoutB, typename InLayout>
@@ -44,18 +62,15 @@ static void prepack_B_template(
62
63
TORCH_CHECK(size<0>(B_layout) % size<0>(TileShapeNKL{}) == 0);
64
TORCH_CHECK(size<1>(B_layout) % size<1>(TileShapeNKL{}) == 0);
47
- TORCH_CHECK(size<2>(B_layout) % size<2>(TileShapeNKL{}) == 0);
65
66
auto N_tiles = size<0>(B_layout) / size<0>(TileShapeNKL{});
67
auto K_tiles = size<1>(B_layout) / size<1>(TileShapeNKL{});
51
- auto L_tiles = size<2>(B_layout) / size<2>(TileShapeNKL{});
68
+ auto L_tiles = size<2>(B_layout);
69
70
auto B_in = make_tensor(get_logical_ptr(B_in_ptr), B_layout);
54
- auto B_tiled_out =
55
- make_tensor(get_logical_ptr(B_out_ptr), ilvd_NKbNbKL_to_offset);
71
57
- prepack_B_kernel<TileShapeNKL, typename PrepackedLayoutB::ElementB>
58
- <<<dim3(N_tiles, K_tiles, L_tiles), 128, 0, stream>>>(B_in, B_tiled_out);
72
+ prepack_B_kernel<128, PrepackedLayoutB>
73
+ <<<dim3(N_tiles, K_tiles, L_tiles), 128, 0, stream>>>(B_in, B_out_ptr);
74
}
75
76
}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepack_launcher.cuh
+9
-6
@@ -2,9 +2,17 @@
2
3
#include "machete_prepack_kernel.cuh"
4
#include "cutlass_extensions/torch_utils.hpp"
5
+#include "core/scalar_type.hpp"
6
7
namespace machete {
8
9
+struct PrepackBArgs {
10
+ torch::Tensor const& B;
11
+ at::ScalarType a_type;
12
+ vllm::ScalarType b_type;
13
+ c10::optional<at::ScalarType> maybe_group_scales_type;
14
+};
15
+
16
template <typename PrepackedLayoutB>
17
torch::Tensor prepack_impl(torch::Tensor const B) {
18
const at::cuda::OptionalCUDAGuard device_guard(device_of(B));
@@ -61,11 +69,6 @@ torch::Tensor prepack_impl(torch::Tensor const B) {
69
return D;
70
};
71
64
-template <typename ElementA, typename ElementB, typename ElementD,
65
- typename AccumulatorT = float, typename ScaleT = cutlass::half_t,
66
- typename ZeroT = cutlass::half_t>
67
-struct PrepackBDispatcher {
68
- static torch::Tensor dispatch(torch::Tensor B);
69
-};
72
+torch::Tensor prepack_B_dispatch(PrepackBArgs args);
73
74
}; // namespace machete
\ No newline at end of file
csrc/quantization/machete/machete_prepacked_layout.cuh
+42
-12
@@ -41,7 +41,7 @@ struct IlvBlkLayoutAuto {};
41
// The contract here is that the `TiledMma` determined below matches the one
42
// ultimately used in the kernel. (this is also why the other element types are
43
// required along with the kernel schedule)
44
-template <typename ElementA_, typename ElementB_, typename ElementD_,
44
+template <typename ElementA_, typename ElementB_, typename ElementConvert_,
45
typename AccumulatorT, class LayoutB, class KernelSchedule,
46
typename IlvBlkLayout_ = IlvBlkLayoutAuto>
47
// clang-format on
@@ -49,20 +49,27 @@ struct PrepackedLayoutBTemplate {
49
using MmaType = ElementA_;
50
using ElementA = ElementA_;
51
using ElementB = ElementB_;
52
- using ElementD = ElementD_;
53
- using ElementAccumulator =
54
- AccumulatorT; // Element type for internal accumulation
52
+ using ElementAccumulator = AccumulatorT;
53
using ElementMma = MmaType;
54
57
- // Only use interleaved layouts for subbyte weights, prmt instructions makes
58
- // non-interleaved layouts for 8bit+ weights efficient enough we don't need
59
- // iterleaved layouts
55
+ // Interleave for 4bit bit types when we are not upconverting to fp8 or int8,
56
+ // in those cases case we use a LUT using prmt instructions to upconvert and
57
+ // is more efficient if the data is not interleaved For 8bit+ prmt
58
+ // instructions makes non-interleaved layouts efficient enough we don't need
59
+ // iterleaved layouts (and can reuse more of the existing cutlass converts)
60
+ static constexpr bool should_interleave =
61
+ sizeof_bits_v<ElementB> <= 4 &&
62
+ !std::is_same_v<ElementConvert_, cutlass::float_e4m3_t> &&
63
+ !std::is_same_v<ElementConvert_, int8_t>;
64
+
65
+ // Only use interleaved layouts for subbyte weights,
66
using IlvdBlkLayout = std::conditional_t<
67
std::is_same_v<IlvBlkLayout_, IlvBlkLayoutAuto>,
62
- std::conditional_t<sizeof_bits_v<ElementB> <= 4,
63
- decltype(get_interleaved_blk_layout<
64
- ElementB, sizeof_bits_v<ElementA>, 32>()),
65
- void>,
68
+ std::conditional_t<
69
+ should_interleave,
70
+ decltype(get_interleaved_blk_layout<
71
+ ElementB, sizeof_bits_v<ElementConvert_>, 32>()),
72
+ void>,
73
IlvBlkLayout_>;
74
75
// TODO (LucasWilkinson): compare the performance for other sizes
@@ -135,7 +142,8 @@ struct PrepackedLayoutBTemplate {
142
// then ((IlvBlk), FrgB) is {A, C, B, D, C, G, D, H}
143
auto frgV = get<1, 0>(layout_no_interleave);
144
auto ilvdBlk = IlvdBlkLayout{};
138
- static_assert(size(frgV) % 4 == 0, "FrgV must be divisible by 4");
145
+ static_assert(size(frgV) % size(ilvdBlk) == 0,
146
+ "FrgV must be divisible by size(ilvdBlk)");
147
auto ilvd_FrgV = make_layout(
148
make_shape(shape(ilvdBlk), Int<size(frgV) / size(ilvdBlk)>{}),
149
make_stride(stride(ilvdBlk), size(ilvdBlk)));
@@ -175,6 +183,15 @@ struct PrepackedLayoutBTemplate {
183
return group<1, 3>(result(_, repeat<rank<1>(result)>(_)));
184
}
185
186
+ // ((athrid_val), (BlocksN, BlocksK, L)) -> (N, K, L)
187
+ template <typename Shape_NKL>
188
+ CUTE_HOST_DEVICE static constexpr auto TVbNbKL_to_offset_copy(
189
+ Shape_NKL shape_mkl) {
190
+ auto layout = TVbNbKL_to_offset(shape_mkl);
191
+ return make_layout(coalesce(get<0>(layout)), get<1>(layout),
192
+ get<2>(layout));
193
+ }
194
+
195
// ((BlockN, BlockK), (BlocksN, BlocksK), L) -> (storage_idx)
196
template <typename Shape_NKL>
197
CUTE_HOST_DEVICE static constexpr auto ilvd_NKbNbKL_to_offset(
@@ -197,6 +214,19 @@ struct PrepackedLayoutBTemplate {
214
return group<1, 3>(result(_, repeat<rank<1>(result)>(_)));
215
}
216
217
+ // (BlocksN, BlocksK, L) -> (storage_idx)
218
+ template <typename Shape_NKL>
219
+ CUTE_HOST_DEVICE static constexpr auto bNbKL_to_offset(Shape_NKL shape_mkl) {
220
+ // (BlocksN, BlocksK, L)
221
+ auto blocks_shape =
222
+ cute::transform(shape_mkl, append(PPBlockShape_NK{}, _1{}),
223
+ [](auto x, auto y) { return x / y; });
224
+ auto stride = size(PPBlockShape_NK{});
225
+
226
+ // (BlocksN, BlocksK, L) -> (storage_idx)
227
+ return make_layout(blocks_shape, compact_col_major(blocks_shape, stride));
228
+ }
229
+
230
// ((athrid, val), (BlocksN, BlocksK, L)) -> (N, K, L)
231
template <class Shape_NKL>
232
CUTE_HOST_DEVICE static auto TVbNbK_to_NKL(Shape_NKL shape_mkl) {
csrc/quantization/machete/machete_pytorch.cu
+46
-74
@@ -8,89 +8,61 @@ namespace machete {
8
9
using namespace vllm;
10
11
-//
12
-// Utils (type dispatching)
13
-//
14
-
15
-template <typename Fn>
16
-static auto scalar_type_dispatch(ScalarType const& type, Fn fn) {
17
- if (type == vllm::kU4) {
18
- return fn(cutlass::uint4b_t{});
19
- } else if (type == vllm::kU8) {
20
- return fn(cutlass::uint8_t{});
21
- } else if (type == vllm::kU4B8) {
22
- return fn(cutlass::vllm_uint4b8_t{});
23
- } else if (type == vllm::kU8B128) {
24
- return fn(cutlass::vllm_uint8b128_t{});
25
- } else {
26
- TORCH_CHECK(false, "Unsupported type ", type.str());
27
- }
28
-}
29
-
30
-#define AT_DISPATCH_CASE_SUPPORTED_COMPUTE_TYPES(...) \
31
- AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__)
32
-
33
-#define AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(TYPE, NAME, ...) \
34
- AT_DISPATCH_SWITCH(TYPE, NAME, \
35
- AT_DISPATCH_CASE_SUPPORTED_COMPUTE_TYPES(__VA_ARGS__))
36
-
37
-//
38
-// Interface
39
-//
40
-
41
-std::vector<std::string> supported_schedules(ScalarTypeId const btype_id) {
42
-#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
43
- vllm::ScalarType b_type = ScalarType::from_id(btype_id);
44
- return scalar_type_dispatch(b_type, [&](auto BType) {
45
- return GemmDispatcher<half_t, decltype(BType)>::supported_schedules();
11
+std::vector<std::string> supported_schedules(
12
+ at::ScalarType a_type, int64_t b_type_id,
13
+ c10::optional<at::ScalarType> maybe_group_scales_type,
14
+ c10::optional<at::ScalarType> maybe_group_zeros_type,
15
+ c10::optional<at::ScalarType> maybe_channel_scales_type,
16
+ c10::optional<at::ScalarType> maybe_token_scales_type,
17
+ c10::optional<at::ScalarType> maybe_out_type) {
18
+ ScalarType const b_type = ScalarType::from_id(b_type_id);
19
+ return supported_schedules_dispatch({
20
+ .a_type = a_type,
21
+ .b_type = b_type,
22
+ .maybe_group_scales_type = maybe_group_scales_type,
23
+ .maybe_group_zeros_type = maybe_group_zeros_type,
24
+ .maybe_channel_scales_type = maybe_channel_scales_type,
25
+ .maybe_token_scales_type = maybe_token_scales_type,
26
+ .maybe_out_type = maybe_out_type,
27
});
47
-#else
48
- TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
49
-#endif
28
}
29
52
-torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
53
- ScalarTypeId const btype_id,
54
- c10::optional<torch::Tensor> const& scales,
55
- c10::optional<torch::Tensor> const& zeros,
56
- c10::optional<int64_t> group_size,
57
- c10::optional<torch::Tensor> const& C,
58
- c10::optional<double> alpha, c10::optional<double> beta,
59
- c10::optional<std::string> schedule) {
60
-#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
61
- ScalarType const btype = ScalarType::from_id(btype_id);
62
- auto args = PyTorchArguments{.A = A,
63
- .B = B,
64
- .scales = scales,
65
- .zeros = zeros,
66
- .group_size = group_size,
67
- .C = C,
68
- .alpha = alpha,
69
- .beta = beta,
70
- .schedule = schedule};
71
-
72
- return scalar_type_dispatch(btype, [&](auto BType) {
73
- return AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(
74
- A.scalar_type(), "machete_gemm", [&] {
75
- using ComputeType = equivalent_cutlass_type_t<scalar_t>;
76
- return GemmDispatcher<ComputeType, decltype(BType)>::dispatch(args);
77
- });
78
- });
79
-#else
80
- TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
81
-#endif
30
+torch::Tensor mm(torch::Tensor const& A, torch::Tensor const& B,
31
+ int64_t b_type_id,
32
+ c10::optional<at::ScalarType> const& maybe_out_type,
33
+ c10::optional<torch::Tensor> const& maybe_group_scales,
34
+ c10::optional<torch::Tensor> const& maybe_group_zeros,
35
+ c10::optional<int64_t> maybe_group_size,
36
+ c10::optional<torch::Tensor> const& maybe_channel_scales,
37
+ c10::optional<torch::Tensor> const& maybe_token_scales,
38
+ c10::optional<std::string> maybe_schedule) {
39
+ ScalarType const b_type = ScalarType::from_id(b_type_id);
40
+ return mm_dispatch({.A = A,
41
+ .B = B,
42
+ .b_type = b_type,
43
+ .maybe_out_type = maybe_out_type,
44
+ .maybe_group_scales = maybe_group_scales,
45
+ .maybe_group_zeros = maybe_group_zeros,
46
+ .maybe_group_size = maybe_group_size,
47
+ .maybe_channel_scales = maybe_channel_scales,
48
+ .maybe_token_scales = maybe_token_scales,
49
+ .maybe_schedule = maybe_schedule});
50
}
51
84
-torch::Tensor prepack_B(torch::Tensor const& B, ScalarTypeId const btype_id) {
85
- ScalarType const btype = ScalarType::from_id(btype_id);
86
- return scalar_type_dispatch(btype, [&](auto BType) {
87
- return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
88
- });
52
+torch::Tensor prepack_B(
53
+ torch::Tensor const& B, at::ScalarType const& a_type, int64_t b_type_id,
54
+ c10::optional<at::ScalarType> const& maybe_group_scales_type) {
55
+ ScalarType const b_type = ScalarType::from_id(b_type_id);
56
+ return prepack_B_dispatch(
57
+ {.B = B,
58
+ .a_type = a_type,
59
+ .b_type = b_type,
60
+ .maybe_group_scales_type = maybe_group_scales_type});
61
}
62
63
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
64
m.impl("machete_prepack_B", &prepack_B);
93
- m.impl("machete_gemm", &gemm);
65
+ m.impl("machete_mm", &mm);
66
}
67
68
// use CatchAll since supported_schedules has no tensor arguments
csrc/torch_bindings.cpp
+29
-6
@@ -203,13 +203,36 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
203
// conditionally compiled so impl in source file
204
205
// Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
206
- ops.def("machete_supported_schedules(int btype) -> str[]");
206
ops.def(
208
- "machete_gemm(Tensor A, Tensor B, int btype, "
209
- " Tensor? scales, Tensor? zeros, int? group_size, "
210
- " Tensor? C, float? alpha, float? beta, str? schedule)"
211
- "-> Tensor");
212
- ops.def("machete_prepack_B(Tensor B, int btype) -> Tensor");
207
+ "machete_supported_schedules("
208
+ " ScalarType a_type,"
209
+ " int b_type,"
210
+ " ScalarType? maybe_group_scales_type,"
211
+ " ScalarType? maybe_group_zeros_type,"
212
+ " ScalarType? maybe_channel_scales_type,"
213
+ " ScalarType? maybe_token_scales_type,"
214
+ " ScalarType? maybe_out_type"
215
+ ") -> str[]");
216
+ ops.def(
217
+ "machete_mm("
218
+ " Tensor A,"
219
+ " Tensor B,"
220
+ " int b_type,"
221
+ " ScalarType? out_type,"
222
+ " Tensor? group_scales,"
223
+ " Tensor? group_zeros,"
224
+ " int? group_size,"
225
+ " Tensor? channel_scales,"
226
+ " Tensor? token_scales,"
227
+ " str? schedule"
228
+ ") -> Tensor");
229
+ ops.def(
230
+ "machete_prepack_B("
231
+ " Tensor B,"
232
+ " ScalarType a_type,"
233
+ " int b_type,"
234
+ " ScalarType? group_scales_type"
235
+ ") -> Tensor");
236
// conditionally compiled so impl registration is in source file
237
238
ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
tests/kernels/test_machete_gemm.py
deleted
-284
@@ -1,284 +0,0 @@
1
-"""Tests for the machete kernel.
2
-
3
-Run `pytest tests/kernels/test_machete_gemm.py`.
4
-"""
5
-
6
-import math
7
-from typing import Optional, Tuple
8
-
9
-import pytest
10
-import torch
11
-
12
-from tests.kernels.utils import opcheck
13
-from vllm import _custom_ops as ops
14
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
15
- pack_rows, quantize_weights)
16
-from vllm.platforms import current_platform
17
-from vllm.scalar_type import ScalarType, scalar_types
18
-
19
-CUDA_DEVICES = [
20
- f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
21
-]
22
-
23
-MNK_SHAPES = [
24
- (1, 128, 128),
25
- (1, 512, 1024),
26
- (1, 4096, 4096),
27
- (1, 8192, 28672),
28
- (13, 8192, 4096),
29
- (26, 4096, 8192),
30
- (64, 4096, 4096),
31
- (64, 8192, 28672),
32
- (257, 128, 4096),
33
- (257, 4224, 4160),
34
- (257, 4096, 4096),
35
- (1024, 4096, 8192),
36
- (1024, 8192, 4096),
37
-]
38
-
39
-ACT_TYPES = [torch.float16, torch.bfloat16]
40
-WTYPE_ZEROPOINTS = [
41
- # GPTQ style
42
- (scalar_types.uint4b8, False),
43
- (scalar_types.uint8b128, False),
44
- # AWQ style
45
- (scalar_types.uint4, True),
46
- (scalar_types.uint8, True),
47
-]
48
-
49
-# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
50
-# unit tests to a common utility function. Currently the use of
51
-# `is_quant_method_supported` conflates kernels with quantization methods
52
-# an assumption which is breaking down as quantizations methods can have
53
-# have kernels and some kernels support multiple quantization methods.
54
-IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
55
-
56
-
57
-def rand_data(shape, dtype=torch.float16):
58
- return 10 * (torch.rand(shape, dtype=dtype, device="cuda") - 0.3)
59
-
60
-
61
-def maybe_convert_zeropoints(zps: Optional[torch.Tensor], s: torch.Tensor):
62
- return zps if zps is None else -1 * s * (zps.to(s.dtype))
63
-
64
-
65
-def machete_quantize_and_pack(w: torch.Tensor,
66
- wtype: ScalarType,
67
- group_size: int,
68
- zero_points: bool = False):
69
- assert wtype.is_integer(), "TODO: support floating point weights"
70
-
71
- w_ref, w_q, w_s, w_zp = quantize_weights(
72
- w,
73
- wtype,
74
- group_size,
75
- zero_points=zero_points,
76
- # to match how the kernel applies zps
77
- ref_zero_points_after_scales=True)
78
-
79
- w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
80
- w_q = w_q.t().contiguous().t() # convert to col major
81
- w_q_machete = ops.machete_prepack_B(w_q, wtype)
82
-
83
- opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype.id))
84
-
85
- return w_ref, w_q_machete, w_s, w_zp
86
-
87
-
88
-def machete_gemm_test_helper(a: torch.Tensor, b: torch.Tensor,
89
- wtype: ScalarType, group_size: int,
90
- zero_points: bool):
91
- w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
92
- b, wtype, group_size, zero_points)
93
-
94
- output_ref = torch.matmul(a, w_ref)
95
-
96
- output = ops.machete_gemm(
97
- a=a,
98
- b_q=w_q_packed,
99
- b_type=wtype,
100
- b_scales=w_s,
101
- b_zeros=maybe_convert_zeropoints(w_zp, w_s),
102
- b_group_size=group_size,
103
- )
104
-
105
- # Relax atol as our reduction dim becomes larger (more rounding error)
106
- # Relax atol when we have zeropoints since the way machete applies
107
- # zeropoints (after scales) causes noise around 0
108
- atol = 1 if zero_points else min(5e-2 * math.sqrt(a.shape[1]), 1)
109
- torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
110
-
111
-
112
-@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
113
- reason="Machete is not supported on this GPU type.")
114
-@pytest.mark.parametrize("shape",
115
- MNK_SHAPES,
116
- ids=lambda x: "x".join(str(v) for v in x))
117
-@pytest.mark.parametrize("atype", ACT_TYPES, ids=lambda x: str(x))
118
-@pytest.mark.parametrize("wtype_zeropoints", WTYPE_ZEROPOINTS)
119
-@pytest.mark.parametrize("group_size", [128, None])
120
-def test_machete_all_schedules(shape, atype: torch.dtype,
121
- wtype_zeropoints: Tuple[ScalarType, bool],
122
- group_size: Optional[int]):
123
- m, n, k = shape
124
- wtype, zero_points = wtype_zeropoints
125
-
126
- if group_size is not None and k % group_size != 0:
127
- return
128
-
129
- print(f"MNK = {m} {n} {k}")
130
-
131
- # Normalize group_size
132
- if group_size is None:
133
- group_size = k
134
- assert group_size <= k
135
-
136
- a = rand_data((m, k), atype)
137
- w = rand_data((k, n), atype)
138
-
139
- w_ref, w_q_machete, w_s, w_zp = machete_quantize_and_pack(
140
- w, wtype, group_size, zero_points)
141
-
142
- output_ref = torch.matmul(a, w_ref)
143
-
144
- for schedule in ops.machete_supported_schedules(wtype):
145
- print(f"Testing schedule {schedule}")
146
- output = ops.machete_gemm(
147
- a,
148
- b_q=w_q_machete,
149
- b_type=wtype,
150
- b_scales=w_s,
151
- b_zeros=maybe_convert_zeropoints(w_zp, w_s),
152
- b_group_size=group_size,
153
- schedule=schedule,
154
- )
155
-
156
- opcheck(
157
- torch.ops._C.machete_gemm,
158
- (a, w_q_machete, wtype.id, w_s, maybe_convert_zeropoints(
159
- w_zp, w_s), group_size, None, None, None, schedule))
160
-
161
- # Relax atol as our reduction dim becomes larger (more rounding error)
162
- # Relax atol when we have zeropoints since the way machete applies
163
- # zeropoints (after scales) causes noise around 0
164
- atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
165
- torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol),\
166
- f"Schedule failed {schedule}"
167
-
168
-
169
-@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
170
- reason="Machete is not supported on this GPU type.")
171
-@pytest.mark.parametrize("shape",
172
- MNK_SHAPES,
173
- ids=lambda x: "x".join(str(v) for v in x))
174
-@pytest.mark.parametrize("atype", ACT_TYPES, ids=lambda x: str(x))
175
-@pytest.mark.parametrize("wtype_zeropoints", WTYPE_ZEROPOINTS)
176
-@pytest.mark.parametrize("group_size", [128, None])
177
-def test_machete_heuristic(shape, atype: torch.dtype,
178
- wtype_zeropoints: Tuple[ScalarType, bool],
179
- group_size: Optional[int]):
180
- m, n, k = shape
181
- wtype, zero_points = wtype_zeropoints
182
-
183
- if group_size is not None and k % group_size != 0:
184
- return
185
-
186
- # Normalize group_size
187
- if group_size is None:
188
- group_size = k
189
- assert group_size <= k
190
-
191
- a = rand_data((m, k), atype)
192
- b = rand_data((k, n), atype)
193
-
194
- machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
195
-
196
-
197
-# Test working on other devices
198
-@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
199
- reason="Machete is not supported on this GPU type.")
200
-@pytest.mark.parametrize("device", CUDA_DEVICES)
201
-def test_machete_devices(device: str):
202
- m, n, k = 512, 4096, 4096
203
- wtype = scalar_types.uint4b8
204
- group_size = 128
205
- zero_points = False
206
-
207
- print(f"MNK = {m} {n} {k}, device = {device}")
208
-
209
- a = rand_data((m, k), torch.float16).to(device)
210
- b = rand_data((k, n), torch.float16).to(device)
211
-
212
- machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
213
-
214
-
215
-# Test working with a subset of A and B
216
-@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
217
- reason="Machete is not supported on this GPU type.")
218
-def test_machete_subset():
219
- big_m, big_n, big_k = 1024, 1024, 1024
220
- m, n, k = 512, 512, 512
221
- wtype = scalar_types.uint4b8
222
- group_size = 128
223
- zero_points = False
224
-
225
- whole_a = rand_data((big_m, big_k), torch.float16)
226
- whole_b = rand_data((big_k, big_n), torch.float16)
227
-
228
- a = whole_a[0:m, 0:k]
229
- b = whole_b[0:k, 0:n]
230
-
231
- machete_gemm_test_helper(a, b, wtype, group_size, zero_points)
232
-
233
-
234
-# Test to make sure cuda graphs work
235
-class MacheteLayer(torch.nn.Module):
236
-
237
- def __init__(self, **kwargs):
238
- super().__init__()
239
- self.kwargs = kwargs
240
-
241
- def forward(self, a):
242
- return ops.machete_gemm(**self.kwargs)
243
-
244
-
245
-@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
246
- reason="Machete is not supported on this GPU type.")
247
-def test_machete_cuda_graph():
248
- m, n, k = 512, 4096, 4096
249
-
250
- a = rand_data((m, k), torch.float16)
251
- b = rand_data((k, n), torch.float16)
252
- wtype = scalar_types.uint4b8
253
- group_size = 128
254
- zero_points = False
255
-
256
- w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
257
- b, wtype, group_size, zero_points)
258
-
259
- # Construct a trivial model with a single layer that calls a machete kernel
260
- model = MacheteLayer(
261
- a=a,
262
- b_q=w_q_packed,
263
- b_type=wtype,
264
- b_scales=w_s,
265
- b_zeros=maybe_convert_zeropoints(w_zp, w_s),
266
- b_group_size=group_size,
267
- )
268
-
269
- output_ref = torch.matmul(a, w_ref)
270
-
271
- # Run the model with a cuda graph
272
- stream = torch.cuda.Stream()
273
- with torch.cuda.stream(stream):
274
- g = torch.cuda.CUDAGraph()
275
- with torch.cuda.graph(g):
276
- output = model(a)
277
- output.zero_()
278
- g.replay()
279
-
280
- # Relax atol as our reduction dim becomes larger (more rounding error)
281
- # Relax atol when we have zeropoints since the way machete applies
282
- # zeropoints (after scales) causes noise around 0
283
- atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
284
- torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
tests/kernels/test_machete_mm.py
new
+406
@@ -0,0 +1,406 @@
1
+"""Tests for the machete kernel.
2
+
3
+Run `pytest tests/kernels/test_machete_mm.py`.
4
+"""
5
+
6
+import math
7
+from dataclasses import dataclass, fields
8
+from typing import List, Optional, Tuple
9
+
10
+import pytest
11
+import torch
12
+
13
+from tests.kernels.utils import opcheck
14
+from vllm import _custom_ops as ops
15
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
+ pack_rows, quantize_weights)
17
+from vllm.platforms import current_platform
18
+from vllm.scalar_type import ScalarType, scalar_types
19
+
20
+CUDA_DEVICES = [
21
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
22
+]
23
+
24
+# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
25
+# unit tests to a common utility function. Currently the use of
26
+# `is_quant_method_supported` conflates kernels with quantization methods
27
+# an assumption which is breaking down as quantizations methods can have
28
+# have kernels and some kernels support multiple quantization methods.
29
+IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
30
+
31
+MNK_SHAPES = [
32
+ (1, 128, 128),
33
+ (1, 512, 1024),
34
+ (1, 4096, 4096),
35
+ (1, 8192, 28672),
36
+ (13, 8192, 4096),
37
+ (26, 4096, 8192),
38
+ (64, 4096, 4096),
39
+ (64, 8192, 28672),
40
+ (257, 128, 4096),
41
+ (257, 4224, 4160),
42
+ (257, 4096, 4096),
43
+ (1024, 4096, 8192),
44
+ (1024, 8192, 4096),
45
+]
46
+
47
+GROUP_SIZES_TO_TEST: List[Optional[int]] = [128, -1]
48
+
49
+
50
+@dataclass
51
+class TypeConfig:
52
+ act_type: torch.dtype
53
+ weight_type: ScalarType
54
+ output_type: Optional[torch.dtype]
55
+ group_scale_type: Optional[torch.dtype]
56
+ group_zero_type: Optional[torch.dtype]
57
+ channel_scale_type: Optional[torch.dtype]
58
+ token_scale_type: Optional[torch.dtype]
59
+
60
+
61
+@dataclass
62
+class Tensors:
63
+ w_ref: torch.Tensor
64
+ a_ref: torch.Tensor
65
+ a: torch.Tensor
66
+ w_q: torch.Tensor
67
+ w_g_s: Optional[torch.Tensor]
68
+ w_g_zp: Optional[torch.Tensor]
69
+ w_ch_s: Optional[torch.Tensor]
70
+ w_tok_s: Optional[torch.Tensor]
71
+
72
+
73
+# (Act Type, Weight Type, Output Type, Scale Type, ZeroPoints,
74
+# Ch Scales Type, Tok Scales Type)
75
+# NOTE: None "Scale Type" means the act type is floating point
76
+# None "Output Type" means the output type is the same as the act type
77
+TestTypeTuple = Tuple[List[torch.dtype], ScalarType, Optional[torch.dtype],
78
+ Optional[torch.dtype], bool]
79
+TEST_TYPES = [
80
+ # GPTQ style
81
+ *(TypeConfig(act_type=a_type,
82
+ weight_type=w_type,
83
+ output_type=None,
84
+ group_scale_type=a_type,
85
+ group_zero_type=None,
86
+ channel_scale_type=None,
87
+ token_scale_type=None)
88
+ for w_type in [scalar_types.uint4b8, scalar_types.uint8b128]
89
+ for a_type in [torch.float16, torch.bfloat16]),
90
+ # AWQ style
91
+ *(TypeConfig(act_type=a_type,
92
+ weight_type=w_type,
93
+ output_type=None,
94
+ group_scale_type=a_type,
95
+ group_zero_type=a_type,
96
+ channel_scale_type=None,
97
+ token_scale_type=None)
98
+ for w_type in [scalar_types.uint4, scalar_types.uint8]
99
+ for a_type in [torch.float16, torch.bfloat16]),
100
+ # QQQ style
101
+ *(TypeConfig(act_type=torch.int8,
102
+ weight_type=scalar_types.uint4b8,
103
+ output_type=torch.float16,
104
+ group_scale_type=group_scale_type,
105
+ group_zero_type=None,
106
+ channel_scale_type=torch.float,
107
+ token_scale_type=torch.float)
108
+ for group_scale_type in [None, torch.float16]),
109
+ *(TypeConfig(act_type=torch.float8_e4m3fn,
110
+ weight_type=scalar_types.uint4b8,
111
+ output_type=torch.float16,
112
+ group_scale_type=group_scale_type,
113
+ group_zero_type=None,
114
+ channel_scale_type=torch.float,
115
+ token_scale_type=torch.float)
116
+ for group_scale_type in [None, torch.float16]),
117
+]
118
+
119
+# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
120
+# unit tests to a common utility function. Currently the use of
121
+# `is_quant_method_supported` conflates kernels with quantization methods
122
+# an assumption which is breaking down as quantizations methods can have
123
+# have kernels and some kernels support multiple quantization methods.
124
+IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
125
+
126
+
127
+def rand_data(shape, dtype=torch.float16, scale=1, offset=0):
128
+ if dtype.is_floating_point:
129
+ return (scale * torch.rand(shape, device="cuda") - offset).to(dtype)
130
+ else:
131
+ return torch.randint(-8, 7, shape, dtype=dtype, device="cuda")
132
+
133
+
134
+def maybe_convert_zeropoints(zps: Optional[torch.Tensor], s: torch.Tensor):
135
+ return zps if zps is None else -1 * s * (zps.to(s.dtype))
136
+
137
+
138
+def group_size_valid(shape: Tuple[int, int, int],
139
+ group_size: Optional[int]) -> bool:
140
+ return group_size is None or group_size == -1 or group_size % shape[2] == 0
141
+
142
+
143
+def machete_quantize_and_pack(atype: torch.dtype,
144
+ w: torch.Tensor,
145
+ wtype: ScalarType,
146
+ stype: Optional[torch.dtype],
147
+ group_size: Optional[int],
148
+ zero_points: bool = False):
149
+ assert wtype.is_integer(), "TODO: support floating point weights"
150
+
151
+ w_ref, w_q, w_s, w_zp = quantize_weights(
152
+ w,
153
+ wtype,
154
+ group_size=group_size,
155
+ zero_points=zero_points,
156
+ # to match how the kernel applies zps
157
+ ref_zero_points_after_scales=True)
158
+
159
+ w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
160
+ w_q = w_q.t().contiguous().t() # convert to col major
161
+
162
+ w_q_machete = ops.machete_prepack_B(w_q, atype, wtype, stype)
163
+ opcheck(torch.ops._C.machete_prepack_B, (w_q, atype, wtype.id, stype))
164
+
165
+ return w_ref, w_q_machete, w_s, w_zp
166
+
167
+
168
+def create_test_tensors(shape: Tuple[int, int, int],
169
+ types: TypeConfig,
170
+ group_size: Optional[int],
171
+ subset_stride_factor: Optional[int] = None) -> Tensors:
172
+ m, n, k = shape
173
+ factor = subset_stride_factor or 1
174
+
175
+ print("create_test_tensors, shape:", shape, "types:", types, "group_size:",
176
+ group_size)
177
+
178
+ a = rand_data((m * factor, k * factor), types.act_type, scale=3, offset=2)
179
+ w = rand_data((k * factor, n * factor), types.act_type, scale=3, offset=1)
180
+
181
+ if factor > 1:
182
+ a = a[0:m, 0:k]
183
+ w = w[0:k, 0:n]
184
+
185
+ if types.group_scale_type is not None:
186
+ w = w.to(types.group_scale_type)
187
+ if w.dtype.itemsize == 1:
188
+ w = w.to(torch.float16)
189
+
190
+ w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
191
+ a.dtype, w, types.weight_type, types.group_scale_type, group_size,
192
+ types.group_zero_type is not None)
193
+
194
+ if not a.dtype.is_floating_point:
195
+ aiinfo = torch.iinfo(a.dtype)
196
+ w_ref = w_ref.round().clamp(aiinfo.min, aiinfo.max)
197
+
198
+ a_ref = a.to(torch.float32)
199
+ w_ref = w_ref.to(torch.float32)
200
+
201
+ w_ch_s = None if types.channel_scale_type is None else\
202
+ rand_data((n,), types.channel_scale_type)
203
+ w_tok_s = None if types.token_scale_type is None else\
204
+ rand_data((m,), types.token_scale_type)
205
+
206
+ return Tensors(w_ref=w_ref,
207
+ a_ref=a_ref,
208
+ a=a,
209
+ w_q=w_q_packed,
210
+ w_g_s=w_s,
211
+ w_g_zp=maybe_convert_zeropoints(w_zp, w_s),
212
+ w_ch_s=w_ch_s,
213
+ w_tok_s=w_tok_s)
214
+
215
+
216
+# None stype means scales use the same dtype as a
217
+def machete_mm_test_helper(types: TypeConfig,
218
+ tensors: Tensors,
219
+ group_size: Optional[int] = None,
220
+ schedule: Optional[str] = None):
221
+ output_ref = torch.matmul(tensors.a_ref, tensors.w_ref)
222
+ output_ref_type = output_ref.dtype
223
+
224
+ if tensors.w_ch_s is not None:
225
+ output_ref = (output_ref.to(tensors.w_ch_s.dtype) *
226
+ tensors.w_ch_s.unsqueeze(0)).to(output_ref_type)
227
+ if tensors.w_tok_s is not None:
228
+ output_ref = (output_ref.to(tensors.w_tok_s.dtype) *
229
+ tensors.w_tok_s.unsqueeze(1)).to(output_ref_type)
230
+
231
+ output = ops.machete_mm(
232
+ a=tensors.a,
233
+ b_q=tensors.w_q,
234
+ b_type=types.weight_type,
235
+ b_group_scales=tensors.w_g_s,
236
+ b_group_zeros=tensors.w_g_zp,
237
+ b_group_size=group_size,
238
+ b_channel_scales=tensors.w_ch_s,
239
+ a_token_scales=tensors.w_tok_s,
240
+ out_type=types.output_type,
241
+ schedule=schedule,
242
+ )
243
+
244
+ print(output)
245
+ print(output_ref)
246
+
247
+ # Relax atol as our reduction dim becomes larger (more rounding error)
248
+ # Relax atol when we have zeropoints since the way machete applies
249
+ # zeropoints (after scales) causes noise around 0
250
+ atol = 1 if tensors.w_g_zp is not None\
251
+ else min(5e-2 * math.sqrt(tensors.a.shape[1]), 1)
252
+ rtol = 1e-1 if tensors.a.element_size() >= 2 else 2e-1
253
+ torch.testing.assert_close(output,
254
+ output_ref.to(output.dtype),
255
+ rtol=rtol,
256
+ atol=atol)
257
+
258
+
259
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
260
+ reason="Machete is not supported on this GPU type.")
261
+@pytest.mark.parametrize("shape",
262
+ MNK_SHAPES,
263
+ ids=lambda x: "x".join(str(v) for v in x))
264
+@pytest.mark.parametrize("types", TEST_TYPES)
265
+def test_machete_all_schedules(shape, types: TypeConfig):
266
+
267
+ group_sizes: List[Optional[int]] = []
268
+ if types.group_scale_type is None:
269
+ group_sizes = [None]
270
+ else:
271
+ group_sizes = GROUP_SIZES_TO_TEST
272
+
273
+ for group_size in group_sizes:
274
+ if not group_size_valid(shape, group_size):
275
+ continue
276
+
277
+ tensors = create_test_tensors(shape, types, group_size)
278
+ print(f"MNK = {shape}")
279
+ for schedule in ops.machete_supported_schedules(
280
+ types.act_type,
281
+ types.weight_type,
282
+ group_scales_type=types.group_scale_type,
283
+ group_zeros_type=types.group_scale_type,
284
+ out_type=types.output_type):
285
+ print(f"Testing schedule {schedule}")
286
+ machete_mm_test_helper(types, tensors, group_size, schedule)
287
+
288
+
289
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
290
+ reason="Machete is not supported on this GPU type.")
291
+@pytest.mark.parametrize("shape",
292
+ MNK_SHAPES,
293
+ ids=lambda x: "x".join(str(v) for v in x))
294
+@pytest.mark.parametrize("types", TEST_TYPES)
295
+def test_machete_heuristic(shape, types: TypeConfig):
296
+ group_sizes: List[Optional[int]] = []
297
+ if types.group_scale_type is None:
298
+ group_sizes = [None]
299
+ else:
300
+ group_sizes = GROUP_SIZES_TO_TEST
301
+
302
+ for group_size in group_sizes:
303
+ if not group_size_valid(shape, group_size):
304
+ continue
305
+
306
+ tensors = create_test_tensors(shape, types, group_size)
307
+ machete_mm_test_helper(types, tensors, group_size)
308
+
309
+
310
+# Test working on other devices
311
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
312
+ reason="Machete is not supported on this GPU type.")
313
+@pytest.mark.parametrize("device", CUDA_DEVICES)
314
+def test_machete_devices(device: str):
315
+ group_size = 128
316
+
317
+ type_config = TypeConfig(act_type=torch.float16,
318
+ weight_type=scalar_types.uint4b8,
319
+ output_type=None,
320
+ group_scale_type=torch.float16,
321
+ group_zero_type=None,
322
+ channel_scale_type=None,
323
+ token_scale_type=None)
324
+
325
+ tensors = create_test_tensors((512, 4096, 4096), type_config, group_size)
326
+
327
+ for field in fields(Tensors):
328
+ tensor = getattr(tensors, field.name)
329
+ if isinstance(tensor, torch.Tensor):
330
+ setattr(tensors, field.name, tensor.to(device))
331
+
332
+ machete_mm_test_helper(type_config, tensors, group_size)
333
+
334
+
335
+# Test working with a subset of A and B
336
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
337
+ reason="Machete is not supported on this GPU type.")
338
+def test_machete_subset():
339
+ group_size = 128
340
+
341
+ type_config = TypeConfig(act_type=torch.float16,
342
+ weight_type=scalar_types.uint4b8,
343
+ output_type=None,
344
+ group_scale_type=torch.float16,
345
+ group_zero_type=None,
346
+ channel_scale_type=None,
347
+ token_scale_type=None)
348
+
349
+ tensors = create_test_tensors((512, 4096, 4096),
350
+ type_config,
351
+ group_size,
352
+ subset_stride_factor=2)
353
+ machete_mm_test_helper(type_config, tensors, group_size)
354
+
355
+
356
+# Test to make sure cuda graphs work
357
+class MacheteLayer(torch.nn.Module):
358
+
359
+ def __init__(self, **kwargs):
360
+ super().__init__()
361
+ self.kwargs = kwargs
362
+
363
+ def forward(self, a):
364
+ return ops.machete_mm(a=a, **self.kwargs)
365
+
366
+
367
+@pytest.mark.skipif(not IS_SUPPORTED_BY_GPU,
368
+ reason="Machete is not supported on this GPU type.")
369
+def test_machete_cuda_graph():
370
+ m, n, k = 512, 4096, 4096
371
+
372
+ a = rand_data((m, k), torch.float16)
373
+ b = rand_data((k, n), torch.float16)
374
+ wtype = scalar_types.uint4b8
375
+ stype = torch.float16
376
+ group_size = 128
377
+ zero_points = False
378
+
379
+ w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
380
+ a.dtype, b, wtype, stype, group_size, zero_points)
381
+
382
+ # Construct a trivial model with a single layer that calls a machete kernel
383
+ model = MacheteLayer(
384
+ b_q=w_q_packed,
385
+ b_type=wtype,
386
+ b_group_scales=w_s,
387
+ b_group_zeros=maybe_convert_zeropoints(w_zp, w_s),
388
+ b_group_size=group_size,
389
+ )
390
+
391
+ output_ref = torch.matmul(a, w_ref)
392
+
393
+ # Run the model with a cuda graph
394
+ stream = torch.cuda.Stream()
395
+ with torch.cuda.stream(stream):
396
+ g = torch.cuda.CUDAGraph()
397
+ with torch.cuda.graph(g):
398
+ output = model(a)
399
+ output.zero_()
400
+ g.replay()
401
+
402
+ # Relax atol as our reduction dim becomes larger (more rounding error)
403
+ # Relax atol when we have zeropoints since the way machete applies
404
+ # zeropoints (after scales) causes noise around 0
405
+ atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
406
+ torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
vllm/_custom_ops.py
+44
-31
@@ -444,18 +444,18 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
444
size_k: torch.SymInt) -> torch.Tensor:
445
return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
446
447
- @register_fake("_C::machete_gemm")
448
- def machete_gemm_fake(
447
+ @register_fake("_C::machete_mm")
448
+ def machete_mm_fake(
449
a: torch.Tensor,
450
- # Should be the tensor returned by machete_prepack_B
450
+ # b_q Should be the tensor returned by machete_prepack_B
451
b_q: torch.Tensor,
452
b_type: ScalarType,
453
- b_scales: Optional[torch.Tensor] = None,
454
- b_zeros: Optional[torch.Tensor] = None,
453
+ out_type: Optional[torch.dtype] = None,
454
+ b_group_scales: Optional[torch.Tensor] = None,
455
+ b_group_zeros: Optional[torch.Tensor] = None,
456
b_group_size: Optional[int] = None,
456
- c: Optional[torch.Tensor] = None,
457
- alpha: Optional[float] = None,
458
- beta: Optional[float] = None,
457
+ b_channel_scales: Optional[torch.Tensor] = None,
458
+ a_token_scales: Optional[torch.Tensor] = None,
459
schedule: Optional[str] = None,
460
) -> torch.Tensor:
461
m = a.size(0)
@@ -463,8 +463,9 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
463
return torch.empty((m, n), device=a.device, dtype=a.dtype)
464
465
@register_fake("_C::machete_prepack_B")
466
- def machete_prepack_B_fake(b_q_weight: torch.Tensor,
467
- b_type: ScalarType) -> torch.Tensor:
466
+ def machete_prepack_B_fake(
467
+ b_q_weight: torch.Tensor, a_type: torch.dtype, b_type: ScalarType,
468
+ group_scales_type: Optional[torch.dtype]) -> torch.Tensor:
469
return torch.empty_like(b_q_weight,
470
memory_format=torch.contiguous_format)
471
@@ -617,29 +618,41 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
618
619
620
# machete
620
-def machete_supported_schedules(b_type: ScalarType) -> List[str]:
621
- return torch.ops._C.machete_supported_schedules(b_type.id)
622
-
623
-
624
-def machete_gemm(
625
- a: torch.Tensor,
626
- b_q: torch.Tensor, # Should be the tensor returned by machete_prepack_B
627
- b_type: ScalarType,
628
- b_scales: Optional[torch.Tensor] = None,
629
- b_zeros: Optional[torch.Tensor] = None,
630
- b_group_size: Optional[int] = None,
631
- c: Optional[torch.Tensor] = None,
632
- alpha: Optional[float] = None,
633
- beta: Optional[float] = None,
634
- schedule: Optional[str] = None,
635
-) -> torch.Tensor:
636
- return torch.ops._C.machete_gemm(a, b_q, b_type.id, b_scales, b_zeros,
637
- b_group_size, c, alpha, beta, schedule)
621
+def machete_supported_schedules(
622
+ a_type: torch.dtype,
623
+ b_type: ScalarType,
624
+ group_scales_type: Optional[torch.dtype],
625
+ group_zeros_type: Optional[torch.dtype] = None,
626
+ channel_scales_type: Optional[torch.dtype] = None,
627
+ token_scales_type: Optional[torch.dtype] = None,
628
+ out_type: Optional[torch.dtype] = None) -> List[str]:
629
+ return torch.ops._C.machete_supported_schedules(
630
+ a_type, b_type.id, group_scales_type, group_zeros_type,
631
+ channel_scales_type, token_scales_type, out_type)
632
633
640
-def machete_prepack_B(b_q_weight: torch.Tensor,
641
- b_type: ScalarType) -> torch.Tensor:
642
- return torch.ops._C.machete_prepack_B(b_q_weight, b_type.id)
634
+def machete_mm(
635
+ a: torch.Tensor,
636
+ # b_q Should be the tensor returned by machete_prepack_B
637
+ b_q: torch.Tensor,
638
+ b_type: ScalarType,
639
+ out_type: Optional[torch.dtype] = None,
640
+ b_group_scales: Optional[torch.Tensor] = None,
641
+ b_group_zeros: Optional[torch.Tensor] = None,
642
+ b_group_size: Optional[int] = None,
643
+ b_channel_scales: Optional[torch.Tensor] = None,
644
+ a_token_scales: Optional[torch.Tensor] = None,
645
+ schedule: Optional[str] = None) -> torch.Tensor:
646
+ return torch.ops._C.machete_mm(a, b_q, b_type.id, out_type, b_group_scales,
647
+ b_group_zeros, b_group_size,
648
+ b_channel_scales, a_token_scales, schedule)
649
+
650
+
651
+def machete_prepack_B(
652
+ b_q_weight: torch.Tensor, a_type: torch.dtype, b_type: ScalarType,
653
+ group_scales_type: Optional[torch.dtype]) -> torch.Tensor:
654
+ return torch.ops._C.machete_prepack_B(b_q_weight, a_type, b_type.id,
655
+ group_scales_type)
656
657
658
if hasattr(torch.ops._C, "permute_cols"):
vllm/model_executor/layers/quantization/kernels/machete.py
+9
-7
@@ -79,7 +79,9 @@ class MacheteLinearKernel(MPLinearKernel):
79
c.weight_type,
80
packed_dim=0)
81
x.data = ops.machete_prepack_B(x.data.t().contiguous().t(),
82
- self.config.weight_type)
82
+ a_type=c.act_type,
83
+ b_type=c.weight_type,
84
+ group_scales_type=c.act_type)
85
return x
86
87
def transform_w_s(x):
@@ -105,12 +107,12 @@ class MacheteLinearKernel(MPLinearKernel):
107
if c.has_g_idx:
108
x_2d = self.act_perm(x_2d)
109
108
- output = ops.machete_gemm(a=x_2d,
109
- b_q=w_q,
110
- b_type=c.weight_type,
111
- b_zeros=None,
112
- b_scales=w_s,
113
- b_group_size=c.group_size)
110
+ output = ops.machete_mm(a=x_2d,
111
+ b_q=w_q,
112
+ b_type=c.weight_type,
113
+ b_group_zeros=None,
114
+ b_group_scales=w_s,
115
+ b_group_size=c.group_size)
116
117
if bias is not None:
118
output.add_(bias) # In-place add
vllm/model_executor/layers/quantization/utils/quant_utils.py
+24
-21
@@ -126,11 +126,14 @@ def permute_rows(q_w: torch.Tensor,
126
127
def quantize_weights(w: torch.Tensor,
128
quant_type: ScalarType,
129
- group_size: int,
129
+ group_size: Optional[int],
130
zero_points: bool = False,
131
ref_zero_points_after_scales: bool = False):
132
assert quant_type.is_integer(), \
133
"Floating point quantization may work but has not been tested"
134
+ assert not zero_points or group_size is not None, \
135
+ "to have group zero points, group_size must be provided "\
136
+ "(-1 group_size is channelwise)"
137
138
orig_device = w.device
139
orig_type = w.dtype
@@ -140,10 +143,9 @@ def quantize_weights(w: torch.Tensor,
143
144
if group_size == -1:
145
group_size = size_k
143
- assert group_size <= size_k
146
147
# Reshape to [groupsize, -1]
146
- if group_size < size_k:
148
+ if group_size is not None and group_size < size_k:
149
w = w.reshape((-1, group_size, size_n))
150
w = w.permute(1, 0, 2)
151
w = w.reshape((group_size, -1))
@@ -155,18 +157,20 @@ def quantize_weights(w: torch.Tensor,
157
max_q_val = quant_type.max()
158
min_q_val = quant_type.min()
159
158
- if zero_points:
159
- assert not quant_type.is_signed() and quant_type.max() > 0
160
- w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
161
- maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
162
- .clamp(min_q_val, max_q_val).int()
163
- else:
164
- # If the bias is such that there are no possible negative/positive
165
- # values, set the max value to inf to avoid divide by 0
166
- w_s = torch.max(
167
- abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
168
- abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
169
- maybe_w_zp = None
160
+ w_s = torch.Tensor([1.0]).to(w.device) # unscaled case
161
+ maybe_w_zp = None
162
+ if group_size is not None:
163
+ if zero_points:
164
+ assert not quant_type.is_signed() and quant_type.max() > 0
165
+ w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
166
+ maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
167
+ .clamp(min_q_val, max_q_val).int()
168
+ else:
169
+ # If the bias is such that there are no possible negative/positive
170
+ # values, set the max value to inf to avoid divide by 0
171
+ w_s = torch.max(
172
+ abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
173
+ abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
174
175
# Quantize
176
w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0)
@@ -176,7 +180,7 @@ def quantize_weights(w: torch.Tensor,
180
# For some kernels (namely Machete) the zero-points are applied after the
181
# scales are applied, for this case computing the reference in similar way
182
# allows us to use tighter error tolerances in our unit tests.
179
- if ref_zero_points_after_scales and zero_points:
183
+ if ref_zero_points_after_scales and maybe_w_zp is not None:
184
w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s
185
else:
186
w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
@@ -185,7 +189,7 @@ def quantize_weights(w: torch.Tensor,
189
w_q += quant_type.bias
190
191
# Restore original shapes
188
- if group_size < size_k:
192
+ if group_size is not None and group_size < size_k:
193
194
def reshape_w(w):
195
w = w.reshape((group_size, -1, size_n))
@@ -195,17 +199,16 @@ def quantize_weights(w: torch.Tensor,
199
200
w_q = reshape_w(w_q)
201
w_ref = reshape_w(w_ref)
202
+ w_s = w_s.reshape((-1, size_n)).contiguous()
203
199
- w_s = w_s.reshape((-1, size_n)).contiguous()
200
-
201
- if zero_points:
204
+ if maybe_w_zp is not None:
205
maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous()
206
maybe_w_zp = maybe_w_zp.to(device=orig_device)
207
208
return (
209
w_ref.to(device=orig_device),
210
w_q.to(device=orig_device),
208
- w_s.to(device=orig_device),
211
+ w_s if group_size is not None else None,
212
maybe_w_zp,
213
)
214