@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f081c3ce4b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Update Cutlass fp8 configs (#5144)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>

Varun Sundar Rabindranath committed Jun 1, 2024 at 14:16 UTC f081c3ce4b020fb094e33575d178345c477ab0c6
4 files changed +480 -15
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py new
+352
@@ -0,0 +1,352 @@
1 +import argparse
2 +import copy
3 +import itertools
4 +import pickle as pkl
5 +import time
6 +from typing import Callable, Iterable, List, Tuple
7 +
8 +import torch
9 +import torch.utils.benchmark as TBenchmark
10 +from torch.utils.benchmark import Measurement as TMeasurement
11 +from weight_shapes import WEIGHT_SHAPES
12 +
13 +from vllm import _custom_ops as ops
14 +
15 +DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())[1:]
16 +DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
17 +DEFAULT_TP_SIZES = [1]
18 +
19 +# helpers
20 +
21 +
22 +def to_fp8(tensor: torch.tensor) -> torch.tensor:
23 + finfo = torch.finfo(torch.float8_e4m3fn)
24 + return torch.round(tensor.clamp(
25 + min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
26 +
27 +
28 +def to_int8(tensor: torch.tensor) -> torch.tensor:
29 + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
30 +
31 +
32 +def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
33 + k: int) -> Tuple[torch.tensor, torch.tensor]:
34 +
35 + a = torch.randn((m, k), device='cuda') * 5
36 + b = torch.randn((n, k), device='cuda').t() * 5
37 +
38 + if dtype == torch.int8:
39 + return to_int8(a), to_int8(b)
40 + if dtype == torch.float8_e4m3fn:
41 + return to_fp8(a), to_fp8(b)
42 +
43 + raise ValueError("unsupported dtype")
44 +
45 +
46 +# impl
47 +
48 +
49 +def pytorch_i8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
50 + scale_b: torch.tensor,
51 + out_dtype: torch.dtype) -> torch.tensor:
52 + return torch.mm(a, b)
53 +
54 +
55 +def pytorch_fp8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
56 + scale_b: torch.tensor,
57 + out_dtype: torch.dtype) -> torch.tensor:
58 + return torch._scaled_mm(a,
59 + b,
60 + scale_a=scale_a,
61 + scale_b=scale_b,
62 + out_dtype=out_dtype)
63 +
64 +
65 +def pytorch_fp8_impl_fast_accum(a: torch.tensor, b: torch.tensor,
66 + scale_a: torch.tensor, scale_b: torch.tensor,
67 + out_dtype: torch.dtype) -> torch.tensor:
68 + return torch._scaled_mm(a,
69 + b,
70 + scale_a=scale_a,
71 + scale_b=scale_b,
72 + out_dtype=out_dtype,
73 + use_fast_accum=True)
74 +
75 +
76 +def cutlass_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
77 + scale_b: torch.tensor,
78 + out_dtype: torch.dtype) -> torch.tensor:
79 + return ops.cutlass_scaled_mm_dq(a,
80 + b,
81 + scale_a,
82 + scale_b,
83 + out_dtype=out_dtype)
84 +
85 +
86 +# bench
87 +def bench_fn(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
88 + scale_b: torch.tensor, out_dtype: torch.dtype, label: str,
89 + sub_label: str, fn: Callable, description: str) -> TMeasurement:
90 +
91 + min_run_time = 1
92 +
93 + globals = {
94 + "a": a,
95 + "b": b,
96 + "scale_a": scale_a,
97 + "scale_b": scale_b,
98 + "out_dtype": out_dtype,
99 + "fn": fn,
100 + }
101 + return TBenchmark.Timer(
102 + stmt="fn(a, b, scale_a, scale_b, out_dtype)",
103 + globals=globals,
104 + label=label,
105 + sub_label=sub_label,
106 + description=description,
107 + ).blocked_autorange(min_run_time=min_run_time)
108 +
109 +
110 +def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
111 + sub_label: str) -> Iterable[TMeasurement]:
112 + assert dtype == torch.int8
113 + a, b = make_rand_tensors(torch.int8, m, n, k)
114 + scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
115 + scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
116 +
117 + timers = []
118 + # pytorch impl
119 + timers.append(
120 + bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
121 + b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
122 + torch.bfloat16, label, sub_label, pytorch_i8_impl,
123 + "pytorch_bf16_bf16_bf16_matmul-no-scales"))
124 +
125 + # cutlass impl
126 + timers.append(
127 + bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
128 + torch.bfloat16, label, sub_label, cutlass_impl,
129 + "cutlass_i8_i8_bf16_scaled_mm"))
130 +
131 + return timers
132 +
133 +
134 +def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
135 + sub_label: str) -> Iterable[TMeasurement]:
136 + assert dtype == torch.float8_e4m3fn
137 + a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k)
138 + scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
139 + scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
140 +
141 + timers = []
142 +
143 + # pytorch impl: bf16 output, without fp8 fast accum
144 + timers.append(
145 + bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
146 + pytorch_fp8_impl, "pytorch_fp8_fp8_bf16_scaled_mm"))
147 +
148 + # pytorch impl: bf16 output, with fp8 fast accum
149 + timers.append(
150 + bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
151 + pytorch_fp8_impl_fast_accum,
152 + "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum"))
153 +
154 + # pytorch impl: fp16 output, without fp8 fast accum
155 + timers.append(
156 + bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
157 + pytorch_fp8_impl, "pytorch_fp8_fp8_fp16_scaled_mm"))
158 +
159 + # pytorch impl: fp16 output, with fp8 fast accum
160 + timers.append(
161 + bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
162 + pytorch_fp8_impl_fast_accum,
163 + "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum"))
164 +
165 + # cutlass impl: bf16 output
166 + timers.append(
167 + bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
168 + torch.bfloat16, label, sub_label, cutlass_impl,
169 + "cutlass_fp8_fp8_bf16_scaled_mm"))
170 + # cutlass impl: fp16 output
171 + timers.append(
172 + bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
173 + torch.float16, label, sub_label, cutlass_impl,
174 + "cutlass_fp8_fp8_fp16_scaled_mm"))
175 + return timers
176 +
177 +
178 +def bench(dtype: torch.dtype, m: int, k: int, n: int, label: str,
179 + sub_label: str) -> Iterable[TMeasurement]:
180 + if dtype == torch.int8:
181 + return bench_int8(dtype, m, k, n, label, sub_label)
182 + if dtype == torch.float8_e4m3fn:
183 + return bench_fp8(dtype, m, k, n, label, sub_label)
184 + raise ValueError("unsupported type")
185 +
186 +
187 +# runner
188 +def print_timers(timers: Iterable[TMeasurement]):
189 + compare = TBenchmark.Compare(timers)
190 + compare.print()
191 +
192 +
193 +def run(dtype: torch.dtype,
194 + MKNs: Iterable[Tuple[int, int, int]]) -> Iterable[TMeasurement]:
195 +
196 + results = []
197 + for m, k, n in MKNs:
198 + timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm",
199 + f"MKN=({m}x{k}x{n})")
200 + print_timers(timers)
201 + results.extend(timers)
202 +
203 + return results
204 +
205 +
206 +# output makers
207 +def make_output(data: Iterable[TMeasurement],
208 + MKNs: Iterable[Tuple[int, int, int]],
209 + base_description: str,
210 + timestamp=None):
211 +
212 + print(f"== All Results {base_description} ====")
213 + print_timers(data)
214 +
215 + # pickle all the results
216 + timestamp = int(time.time()) if timestamp is None else timestamp
217 + with open(f"{base_description}-{timestamp}.pkl", "wb") as f:
218 + pkl.dump(data, f)
219 +
220 +
221 +# argparse runners
222 +
223 +
224 +def run_square_bench(args):
225 + dim_sizes = list(
226 + range(args.dim_start, args.dim_end + 1, args.dim_increment))
227 + MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
228 + data = run(args.dtype, MKNs)
229 +
230 + make_output(data, MKNs, f"square_bench-{args.dtype}")
231 +
232 +
233 +def run_range_bench(args):
234 + dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
235 + n = len(dim_sizes)
236 + Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
237 + Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
238 + Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
239 + MKNs = list(zip(Ms, Ks, Ns))
240 + data = run(args.dtype, MKNs)
241 +
242 + make_output(data, MKNs, f"range_bench-{args.dtype}")
243 +
244 +
245 +def run_model_bench(args):
246 +
247 + print("Benchmarking models:")
248 + for i, model in enumerate(args.models):
249 + print(f"[{i}] {model}")
250 +
251 + def model_shapes(model_name: str, tp_size: int) -> List[Tuple[int, int]]:
252 + KNs = []
253 + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):
254 + KN[tp_split_dim] = KN[tp_split_dim] // tp_size
255 + KNs.append(KN)
256 + return KNs
257 +
258 + model_bench_data = []
259 + models_tps = list(itertools.product(args.models, args.tp_sizes))
260 + for model, tp_size in models_tps:
261 + Ms = args.batch_sizes
262 + KNs = model_shapes(model, tp_size)
263 + MKNs = []
264 + for m in Ms:
265 + for k, n in KNs:
266 + MKNs.append((m, k, n))
267 +
268 + data = run(args.dtype, MKNs)
269 + model_bench_data.append(data)
270 +
271 + # Print all results
272 + for data, model_tp in zip(model_bench_data, models_tps):
273 + model, tp_size = model_tp
274 + print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
275 + print_timers(data)
276 +
277 + timestamp = int(time.time())
278 +
279 + all_data = []
280 + for d in model_bench_data:
281 + all_data.extend(d)
282 + # pickle all data
283 + with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
284 + pkl.dump(all_data, f)
285 +
286 +
287 +if __name__ == '__main__':
288 +
289 + def to_torch_dtype(dt):
290 + if dt == "int8":
291 + return torch.int8
292 + if dt == "fp8":
293 + return torch.float8_e4m3fn
294 + raise ValueError("unsupported dtype")
295 +
296 + parser = argparse.ArgumentParser(
297 + description="""
298 +Benchmark Cutlass GEMM.
299 +
300 + To run square GEMMs:
301 + python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 square_bench --dim-start 128 --dim-end 512 --dim-increment 64
302 +
303 + To run constant N and K and sweep M:
304 + python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384
305 +
306 + To run dimensions from a model:
307 + python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1
308 +
309 + Output:
310 + - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs.
311 + """, # noqa: E501
312 + formatter_class=argparse.RawTextHelpFormatter)
313 +
314 + parser.add_argument("--dtype",
315 + type=to_torch_dtype,
316 + required=True,
317 + help="Available options are ['int8', 'fp8']")
318 + subparsers = parser.add_subparsers(dest="cmd")
319 +
320 + square_parser = subparsers.add_parser("square_bench")
321 + square_parser.add_argument("--dim-start", type=int, required=True)
322 + square_parser.add_argument("--dim-end", type=int, required=True)
323 + square_parser.add_argument("--dim-increment", type=int, required=True)
324 + square_parser.set_defaults(func=run_square_bench)
325 +
326 + range_parser = subparsers.add_parser("range_bench")
327 + range_parser.add_argument("--dim-start", type=int, required=True)
328 + range_parser.add_argument("--dim-end", type=int, required=True)
329 + range_parser.add_argument("--dim-increment", type=int, required=True)
330 + range_parser.add_argument("--m-constant", type=int, default=None)
331 + range_parser.add_argument("--n-constant", type=int, default=None)
332 + range_parser.add_argument("--k-constant", type=int, default=None)
333 + range_parser.set_defaults(func=run_range_bench)
334 +
335 + model_parser = subparsers.add_parser("model_bench")
336 + model_parser.add_argument("--models",
337 + nargs="+",
338 + type=str,
339 + default=DEFAULT_MODELS,
340 + choices=WEIGHT_SHAPES.keys())
341 + model_parser.add_argument("--tp-sizes",
342 + nargs="+",
343 + type=int,
344 + default=DEFAULT_TP_SIZES)
345 + model_parser.add_argument("--batch-sizes",
346 + nargs="+",
347 + type=int,
348 + default=DEFAULT_BATCH_SIZES)
349 + model_parser.set_defaults(func=run_model_bench)
350 +
351 + args = parser.parse_args()
352 + args.func(args)
benchmarks/cutlass_benchmarks/weight_shapes.py new
+37
@@ -0,0 +1,37 @@
1 +# Weight Shapes are in the format
2 +# ([K, N], TP_SPLIT_DIM)
3 +# Example:
4 +# A shape of ([14336, 4096], 0) indicates the following GEMM shape,
5 +# - TP1 : K = 14336, N = 4096
6 +# - TP2 : K = 7168, N = 4096
7 +# A shape of ([4096, 6144], 1) indicates the following GEMM shape,
8 +# - TP1 : K = 4096, N = 6144
9 +# - TP4 : K = 4096, N = 1536
10 +
11 +# TP1 shapes
12 +WEIGHT_SHAPES = {
13 + "mistralai/Mistral-7B-v0.1": [
14 + ([4096, 6144], 1),
15 + ([4096, 4096], 0),
16 + ([4096, 28672], 1),
17 + ([14336, 4096], 0),
18 + ],
19 + "meta-llama/Llama-2-7b-hf": [
20 + ([4096, 12288], 1),
21 + ([4096, 4096], 0),
22 + ([4096, 22016], 1),
23 + ([11008, 4096], 0),
24 + ],
25 + "meta-llama/Llama-2-13b-hf": [
26 + ([5120, 15360], 1),
27 + ([5120, 5120], 0),
28 + ([5120, 27648], 1),
29 + ([13824, 5120], 0),
30 + ],
31 + "meta-llama/Llama-2-70b-hf": [
32 + ([8192, 10240], 1),
33 + ([8192, 8192], 0),
34 + ([8192, 57344], 1),
35 + ([28672, 8192], 0),
36 + ],
37 +}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+90 -14
@@ -51,6 +51,11 @@ using namespace cute;
51
52 namespace {
53
54 +uint32_t next_pow_2(uint32_t const num) {
55 + if (num <= 1) return num;
56 + return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
57 +}
58 +
59 template <typename ElementAB_, typename ElementD_, typename TileShape,
60 typename ClusterShape, typename KernelSchedule,
61 typename EpilogueSchedule>
@@ -188,8 +193,89 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
193 cutlass::Status status = gemm_op.run(args, workspace.get(), stream);
194 CUTLASS_CHECK(status);
195 }
196 +
197 +template <typename InType, typename OutType, int32_t M>
198 +struct sm90_fp8_config {
199 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
200 + using KernelSchedule =
201 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
202 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
203 + using TileShape = Shape<_128, _128, _128>;
204 + using ClusterShape = Shape<_2, _1, _1>;
205 +
206 + using Cutlass3xGemm =
207 + cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
208 + EpilogueSchedule>;
209 +};
210 +
211 +template <typename InType, typename OutType>
212 +struct sm90_fp8_config<InType, OutType, 128> {
213 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
214 + using KernelSchedule =
215 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
216 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
217 + using TileShape = Shape<_64, _128, _128>;
218 + using ClusterShape = Shape<_2, _1, _1>;
219 +
220 + using Cutlass3xGemm =
221 + cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
222 + EpilogueSchedule>;
223 +};
224 +
225 +template <typename InType, typename OutType>
226 +struct sm90_fp8_config<InType, OutType, 64> {
227 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
228 + using KernelSchedule =
229 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
230 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
231 + using TileShape = Shape<_64, _64, _128>;
232 + using ClusterShape = Shape<_1, _8, _1>;
233 +
234 + using Cutlass3xGemm =
235 + cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
236 + EpilogueSchedule>;
237 +};
238 +
239 } // namespace
240
241 +template <typename InType, typename OutType>
242 +void cutlass_scaled_mm_dq_sm90_fp8_dispatch(torch::Tensor& out,
243 + torch::Tensor const& a,
244 + torch::Tensor const& b,
245 + torch::Tensor const& a_scales,
246 + torch::Tensor const& b_scales) {
247 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
248 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
249 + TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
250 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
251 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
252 +
253 + using Cutlass3xGemmDefault =
254 + typename sm90_fp8_config<InType, OutType, 0>::Cutlass3xGemm;
255 + using Cutlass3xGemmM64 =
256 + typename sm90_fp8_config<InType, OutType, 64>::Cutlass3xGemm;
257 + using Cutlass3xGemmM128 =
258 + typename sm90_fp8_config<InType, OutType, 128>::Cutlass3xGemm;
259 +
260 + uint32_t const m = a.size(0);
261 + uint32_t const mp2 =
262 + std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
263 +
264 + if (mp2 <= 64) {
265 + // m in [1, 64]
266 + return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmM64>(
267 + out, a, b, a_scales, b_scales);
268 + } else if (mp2 <= 128) {
269 + // m in (64, 128]
270 + return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmM128>(
271 + out, a, b, a_scales, b_scales);
272 + } else {
273 + // m in (128, inf)
274 + return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmDefault>(
275 + out, a, b, a_scales, b_scales);
276 + }
277 +}
278 +
279 void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
280 torch::Tensor const& b,
281 torch::Tensor const& a_scales,
@@ -223,24 +309,14 @@ void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
309 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
310 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
311
226 - using TileShape = Shape<_128, _128, _128>;
227 - using ClusterShape = Shape<_1, _2, _1>;
228 - using KernelSchedule =
229 - typename cutlass::gemm::KernelCpAsyncWarpSpecializedCooperative;
230 - using EpilogueSchedule =
231 - typename cutlass::epilogue::TmaWarpSpecializedCooperative;
232 -
312 if (out.dtype() == torch::kBFloat16) {
234 - return cutlass_scaled_mm_dq_dispatcher<
235 - cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::bfloat16_t, TileShape,
236 - ClusterShape, KernelSchedule, EpilogueSchedule>>(
313 + return cutlass_scaled_mm_dq_sm90_fp8_dispatch<cutlass::float_e4m3_t,
314 + cutlass::bfloat16_t>(
315 out, a, b, a_scales, b_scales);
316 } else {
317 TORCH_CHECK(out.dtype() == torch::kFloat16);
240 -
241 - return cutlass_scaled_mm_dq_dispatcher<
242 - cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::half_t, TileShape,
243 - ClusterShape, KernelSchedule, EpilogueSchedule>>(
318 + return cutlass_scaled_mm_dq_sm90_fp8_dispatch<cutlass::float_e4m3_t,
319 + cutlass::half_t>(
320 out, a, b, a_scales, b_scales);
321 }
322 }
tests/kernels/test_cutlass.py
+1 -1
@@ -82,7 +82,7 @@ def cutlass_int8_gemm_helper(m: int,
82 assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
83
84
85 -@pytest.mark.parametrize("m", [512, 222, 33, 1])
85 +@pytest.mark.parametrize("m", [512, 222, 100, 33, 1])
86 @pytest.mark.parametrize("n", [2048, 256, 1024])
87 @pytest.mark.parametrize("k", [128, 496, 1024])
88 @pytest.mark.parametrize("per_act_token", [True, False])