@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86e9c8df29
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin (#7701)

Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Divakar Verma <137818590+divakar-amd@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>

Lucas Wilkinson committed Sep 23, 2024 at 13:46 UTC 86e9c8df29a954a7a2fc46e9985fecc2a2e15ae8
27 files changed +1005 -246
CMakeLists.txt
+1
@@ -223,6 +223,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
223 "csrc/quantization/gguf/gguf_kernel.cu"
224 "csrc/quantization/fp8/fp8_marlin.cu"
225 "csrc/custom_all_reduce.cu"
226 + "csrc/permute_cols.cu"
227 "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
228 "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
229 "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
benchmarks/kernels/benchmark_machete.py
+61 -13
@@ -4,8 +4,10 @@ import itertools
4 import math
5 import pickle as pkl
6 import time
7 -from typing import Callable, Iterable, List, Tuple
7 +from itertools import product
8 +from typing import Callable, Iterable, List, Optional, Tuple
9
10 +import pandas as pd
11 import torch
12 import torch.utils.benchmark as TBenchmark
13 from torch.utils.benchmark import Measurement as TMeasurement
@@ -84,6 +86,10 @@ def loop_over_weights(
86 fn(a, w_ref, w_q, w_s)
87
88
89 +_SWEEP_SCHEDULES_RESULTS: Optional[pd.DataFrame] = None
90 +_SWEEP_SCHEDULES_RESULTS_CSV: Optional[str] = None
91 +
92 +
93 def bench(atype: torch.dtype,
94 wtype: ScalarType,
95 group_size: int,
@@ -94,6 +100,8 @@ def bench(atype: torch.dtype,
100 sub_label: str,
101 benchmark_marlinv1: bool = True,
102 sweep_schedules: bool = True) -> Iterable[TMeasurement]:
103 + global _SWEEP_SCHEDULES_RESULTS
104 +
105 a, weights = make_bench_tensors(atype, wtype, group_size, m, n, k)
106 sub_label += f", L={len(weights)}"
107
@@ -163,6 +171,11 @@ def bench(atype: torch.dtype,
171 best_schedule = None
172 schedules = ops.machete_supported_schedules(wtype)
173 for schedule in reversed(schedules):
174 + schedule_M = int(schedule.split("_")[0].split("x")[1])
175 +
176 + # Prune known bad schedules
177 + if schedule_M >= 2 * max(m, 16) or schedule_M < m // 4:
178 + continue
179
180 def run(a, _, w_q, w_s, schedule=schedule):
181 ops.machete_gemm(a,
@@ -175,6 +188,20 @@ def bench(atype: torch.dtype,
188 res = bench_fn(label, sub_label, "machete_best",
189 lambda: loop_over_weights(a, weights_machete, run))
190
191 + results_row = {
192 + "M": m,
193 + "K": k,
194 + "N": n,
195 + "group_size": group_size,
196 + "schedule": schedule,
197 + "median": res.median,
198 + }
199 + if _SWEEP_SCHEDULES_RESULTS is None:
200 + _SWEEP_SCHEDULES_RESULTS = pd.DataFrame(
201 + columns=results_row.keys())
202 + _SWEEP_SCHEDULES_RESULTS.\
203 + loc[len(_SWEEP_SCHEDULES_RESULTS)] = results_row
204 +
205 print(f" {res.median:5.5} ", schedule)
206 if not best or res.median < best.median:
207 best = res
@@ -235,18 +262,22 @@ def run_square_bench(args):
262 dim_sizes = list(
263 range(args.dim_start, args.dim_end + 1, args.dim_increment))
264 MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
265 +
266 data = run(args.dtype, args.sweep_schedules, MKNs)
267
268 make_output(data, MKNs, f"square_bench-{args.dtype}")
269
270
271 def run_range_bench(args):
244 - dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
245 - n = len(dim_sizes)
246 - Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
247 - Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
248 - Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
249 - MKNs = list(zip(Ms, Ks, Ns))
272 + m_start, k_start, n_start = [int(x) for x in args.dim_start.split(",")]
273 + m_end, k_end, n_end = [int(x) for x in args.dim_end.split(",")]
274 + m_increment, k_increment, n_increment = \
275 + [int(x) for x in args.dim_increment.split(",")]
276 + Ms = list(range(m_start, m_end + 1, m_increment))
277 + Ks = list(range(k_start, k_end + 1, k_increment))
278 + Ns = list(range(n_start, n_end + 1, n_increment))
279 + MKNs = list(product(Ms, Ks, Ns))
280 +
281 data = run(args.dtype, args.sweep_schedules, MKNs)
282
283 make_output(data, MKNs, f"range_bench-{args.dtype}")
@@ -333,6 +364,9 @@ Benchmark Machete GEMM.
364 action="store_true",
365 help="Run a sweep over all supported schedules",
366 )
367 + parser.add_argument("--sweep-csv-out",
368 + help="CSV to store sweep results",
369 + default="sch_sweep_results.csv")
370 subparsers = parser.add_subparsers(dest="cmd", required=True)
371
372 square_parser = subparsers.add_parser("square_bench")
@@ -342,12 +376,21 @@ Benchmark Machete GEMM.
376 square_parser.set_defaults(func=run_square_bench)
377
378 range_parser = subparsers.add_parser("range_bench")
345 - range_parser.add_argument("--dim-start", type=int, required=True)
346 - range_parser.add_argument("--dim-end", type=int, required=True)
347 - range_parser.add_argument("--dim-increment", type=int, required=True)
348 - range_parser.add_argument("--m-constant", type=int, default=None)
349 - range_parser.add_argument("--n-constant", type=int, default=None)
350 - range_parser.add_argument("--k-constant", type=int, default=None)
379 + range_parser.add_argument(
380 + "--dim-start",
381 + type=str,
382 + required=True,
383 + help="Start value for M,K,N as common separated list")
384 + range_parser.add_argument(
385 + "--dim-end",
386 + type=str,
387 + required=True,
388 + help="End value (inclusive) for M,K,N as common separated list")
389 + range_parser.add_argument(
390 + "--dim-increment",
391 + type=str,
392 + required=True,
393 + help="Increment value for M,K,N as common separated list")
394 range_parser.set_defaults(func=run_range_bench)
395
396 model_parser = subparsers.add_parser("model_bench")
@@ -369,4 +412,9 @@ Benchmark Machete GEMM.
412 model_parser.set_defaults(func=run_model_bench)
413
414 args = parser.parse_args()
415 +
416 + _SWEEP_SCHEDULES_RESULTS_CSV = args.sweep_csv_out
417 args.func(args)
418 +
419 + if _SWEEP_SCHEDULES_RESULTS is not None:
420 + _SWEEP_SCHEDULES_RESULTS.to_csv(_SWEEP_SCHEDULES_RESULTS_CSV)
benchmarks/kernels/requirements.txt new
+1
@@ -0,0 +1 @@
1 +pandas
\ No newline at end of file
csrc/cutlass_extensions/torch_utils.hpp
+7 -1
@@ -68,7 +68,13 @@ static inline auto make_cute_layout(torch::Tensor const& tensor,
68 name, ".stride(", idx, ") to be ", StrideEle::value);
69 return StrideEle{};
70 } else {
71 - return tensor.stride(idx);
71 + if (tensor.size(idx) == 1) {
72 + // use 0 stride for dim with size 1, this is easier for
73 + // cute/cutlass to optimize (helps the TMA code flatten dims)
74 + return StrideEle{0};
75 + } else {
76 + return tensor.stride(idx);
77 + }
78 }
79 } else {
80 // Extra strides are assumed to be 0 or 1
csrc/ops.h
+2
@@ -113,6 +113,8 @@ torch::Tensor prepack_B(torch::Tensor const& B,
113
114 }; // namespace machete
115
116 +torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
117 +
118 torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
119 torch::Tensor& b_meta,
120 torch::Tensor& b_scales,
csrc/permute_cols.cu new
+88
@@ -0,0 +1,88 @@
1 +#include <torch/all.h>
2 +
3 +#include <ATen/cuda/CUDAContext.h>
4 +#include <c10/cuda/CUDAGuard.h>
5 +
6 +#include <cuda_fp16.h>
7 +
8 +static constexpr int default_threads = 256;
9 +static constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
10 +
11 +// For a given "a" of size [M,K] performs a permutation of the K columns based
12 +// on the given "perm" indices.
13 +// Currently only supports 16bit types (since we permute half types)
14 +__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
15 + int const* __restrict__ perm_int_ptr,
16 + int4* __restrict__ out_int4_ptr, int size_m,
17 + int size_k, int block_rows) {
18 + int start_row = block_rows * blockIdx.x;
19 + int finish_row = start_row + block_rows;
20 + if (finish_row > size_m) {
21 + finish_row = size_m;
22 + }
23 + int cur_block_rows = std::max(finish_row - start_row, 0);
24 +
25 + int row_stride = size_k * sizeof(half) / 16;
26 +
27 + auto permute_row = [&](int row) {
28 + int iters = size_k / default_threads;
29 + int rest = size_k % default_threads;
30 +
31 + int offset = row * row_stride;
32 +
33 + half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
34 + half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
35 +
36 + int base_k = 0;
37 +
38 + for (int i = 0; i < iters; i++) {
39 + int cur_k = base_k + threadIdx.x;
40 + int src_pos = perm_int_ptr[cur_k];
41 +
42 + out_half[cur_k] = a_row_half[src_pos];
43 +
44 + base_k += default_threads;
45 + }
46 +
47 + if (rest) {
48 + if (threadIdx.x < rest) {
49 + int cur_k = base_k + threadIdx.x;
50 + int src_pos = perm_int_ptr[cur_k];
51 +
52 + out_half[cur_k] = a_row_half[src_pos];
53 + }
54 + }
55 + };
56 +
57 + for (int i = 0; i < cur_block_rows; i++) {
58 + int cur_row = start_row + i;
59 + if (cur_row < size_m) {
60 + permute_row(cur_row);
61 + }
62 + }
63 +}
64 +
65 +// More efficient version of A[..., perm]
66 +// taken from gptq_marlin.cu
67 +torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm) {
68 + const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
69 + auto dev = A.get_device();
70 + auto stream = at::cuda::getCurrentCUDAStream(dev);
71 +
72 + TORCH_CHECK(A.scalar_type() == at::kHalf || A.scalar_type() == at::kBFloat16,
73 + "Currently only 16bit types are supported");
74 + TORCH_CHECK(A.is_contiguous(), "A must be contiguous");
75 + TORCH_CHECK(A.size(-1) % 8 == 0,
76 + "A columns must be a multiple of 8 (128bits)");
77 + auto A_2d = A.view({-1, A.size(-1)});
78 +
79 + torch::Tensor D = torch::empty_like(A);
80 + int sms;
81 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
82 + int block_rows = div_ceil(A_2d.size(0), sms);
83 + permute_cols_kernel<<<sms, default_threads, 0, stream>>>(
84 + reinterpret_cast<int4 const*>(A_2d.const_data_ptr()),
85 + perm.const_data_ptr<int>(), reinterpret_cast<int4*>(D.mutable_data_ptr()),
86 + A_2d.size(0), A_2d.size(1), block_rows);
87 + return D;
88 +}
\ No newline at end of file
csrc/quantization/machete/generate.py
+127 -46
@@ -157,7 +157,7 @@ TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput
157 TmaCoop = EpilogueScheduleType.TmaWarpSpecializedCooperative
158
159
160 -@dataclass
160 +@dataclass(frozen=True)
161 class ScheduleConfig:
162 tile_shape_mn: Tuple[int, int]
163 cluster_shape_mnk: Tuple[int, int, int]
@@ -328,56 +328,137 @@ def generate():
328 # about how this works
329 SCRIPT_DIR = os.path.dirname(__file__)
330
331 - schedules = [
332 - ScheduleConfig(
333 - tile_shape_mn=tile_shape_mn,
334 - cluster_shape_mnk=cluster_shape_mnk,
335 - kernel_schedule=kernel_schedule,
336 - epilogue_schedule=epilogue_schedule,
337 - tile_scheduler=tile_scheduler,
338 - ) for tile_shape_mn, cluster_shape_mnk in (
339 - ((128, 16), (1, 1, 1)),
340 - ((128, 32), (1, 1, 1)),
341 - ((128, 64), (1, 1, 1)),
342 - ((128, 128), (1, 1, 1)),
343 - ) for kernel_schedule in (TmaMI, ) for epilogue_schedule in (TmaCoop, )
344 - for tile_scheduler in (TileSchedulerType.StreamK, )
345 - ]
331 + schedule_common_params = dict(
332 + kernel_schedule=TmaMI,
333 + epilogue_schedule=TmaCoop,
334 + tile_scheduler=TileSchedulerType.StreamK,
335 + )
336
337 # For now we use the same heuristic for all types
338 + # Heuristic is currently tuned for H100s
339 default_heuristic = [
349 - ("M > 64",
350 - ScheduleConfig(
351 - tile_shape_mn=(128, 128),
352 - cluster_shape_mnk=(1, 1, 1),
353 - kernel_schedule=TmaMI,
354 - epilogue_schedule=TmaCoop,
355 - tile_scheduler=TileSchedulerType.StreamK,
356 - )),
357 - ("M > 32",
358 - ScheduleConfig(
359 - tile_shape_mn=(128, 64),
360 - cluster_shape_mnk=(1, 1, 1),
361 - kernel_schedule=TmaMI,
362 - epilogue_schedule=TmaCoop,
363 - tile_scheduler=TileSchedulerType.StreamK,
364 - )),
365 - ("M > 16",
366 - ScheduleConfig(
367 - tile_shape_mn=(128, 32),
368 - cluster_shape_mnk=(1, 1, 1),
369 - kernel_schedule=TmaMI,
370 - epilogue_schedule=TmaCoop,
371 - tile_scheduler=TileSchedulerType.StreamK,
372 - )),
373 - (None,
374 - ScheduleConfig(tile_shape_mn=(128, 16),
375 - cluster_shape_mnk=(1, 1, 1),
376 - kernel_schedule=TmaMI,
377 - epilogue_schedule=TmaCoop,
378 - tile_scheduler=TileSchedulerType.StreamK))
340 + #### M = 257+
341 + (
342 + "M > 256 && K <= 16384 && N <= 4096",
343 + ScheduleConfig(
344 + tile_shape_mn=(128, 128),
345 + cluster_shape_mnk=(2, 1, 1),
346 + **schedule_common_params # type: ignore
347 + )),
348 + (
349 + "M > 256",
350 + ScheduleConfig(
351 + tile_shape_mn=(128, 256),
352 + cluster_shape_mnk=(2, 1, 1),
353 + **schedule_common_params # type: ignore
354 + )),
355 + #### M = 129-256
356 + (
357 + "M > 128 && K <= 4096 && N <= 4096",
358 + ScheduleConfig(
359 + tile_shape_mn=(128, 64),
360 + cluster_shape_mnk=(2, 1, 1),
361 + **schedule_common_params # type: ignore
362 + )),
363 + (
364 + "M > 128 && K <= 8192 && N <= 8192",
365 + ScheduleConfig(
366 + tile_shape_mn=(128, 128),
367 + cluster_shape_mnk=(2, 1, 1),
368 + **schedule_common_params # type: ignore
369 + )),
370 + (
371 + "M > 128",
372 + ScheduleConfig(
373 + tile_shape_mn=(128, 256),
374 + cluster_shape_mnk=(2, 1, 1),
375 + **schedule_common_params # type: ignore
376 + )),
377 + #### M = 65-128
378 + (
379 + "M > 64 && K <= 4069 && N <= 4069",
380 + ScheduleConfig(
381 + tile_shape_mn=(128, 32),
382 + cluster_shape_mnk=(2, 1, 1),
383 + **schedule_common_params # type: ignore
384 + )),
385 + (
386 + "M > 64 && K <= 4069 && N <= 8192",
387 + ScheduleConfig(
388 + tile_shape_mn=(128, 64),
389 + cluster_shape_mnk=(2, 1, 1),
390 + **schedule_common_params # type: ignore
391 + )),
392 + (
393 + "M > 64 && K >= 8192 && N >= 12288",
394 + ScheduleConfig(
395 + tile_shape_mn=(256, 128),
396 + cluster_shape_mnk=(2, 1, 1),
397 + **schedule_common_params # type: ignore
398 + )),
399 + (
400 + "M > 64",
401 + ScheduleConfig(
402 + tile_shape_mn=(128, 128),
403 + cluster_shape_mnk=(2, 1, 1),
404 + **schedule_common_params # type: ignore
405 + )),
406 + #### M = 33-64
407 + (
408 + "M > 32 && K <= 6144 && N <= 6144",
409 + ScheduleConfig(
410 + tile_shape_mn=(128, 16),
411 + cluster_shape_mnk=(1, 1, 1),
412 + **schedule_common_params # type: ignore
413 + )),
414 + (
415 + "M > 32 && K >= 16384 && N >= 12288",
416 + ScheduleConfig(
417 + tile_shape_mn=(256, 64),
418 + cluster_shape_mnk=(2, 1, 1),
419 + **schedule_common_params # type: ignore
420 + )),
421 + (
422 + "M > 32",
423 + ScheduleConfig(
424 + tile_shape_mn=(128, 64),
425 + cluster_shape_mnk=(2, 1, 1),
426 + **schedule_common_params # type: ignore
427 + )),
428 + #### M = 17-32
429 + (
430 + "M > 16 && K <= 12288 && N <= 8192",
431 + ScheduleConfig(
432 + tile_shape_mn=(128, 32),
433 + cluster_shape_mnk=(2, 1, 1),
434 + **schedule_common_params # type: ignore
435 + )),
436 + (
437 + "M > 16",
438 + ScheduleConfig(
439 + tile_shape_mn=(256, 32),
440 + cluster_shape_mnk=(2, 1, 1),
441 + **schedule_common_params # type: ignore
442 + )),
443 + #### M = 1-16
444 + (
445 + "N >= 26624",
446 + ScheduleConfig(
447 + tile_shape_mn=(256, 16),
448 + cluster_shape_mnk=(1, 1, 1),
449 + **schedule_common_params # type: ignore
450 + )),
451 + (
452 + None,
453 + ScheduleConfig(
454 + tile_shape_mn=(128, 16),
455 + cluster_shape_mnk=(1, 1, 1),
456 + **schedule_common_params # type: ignore
457 + )),
458 ]
459
460 + schedules = list(set([x[1] for x in default_heuristic]))
461 +
462 impl_configs = []
463
464 GPTQ_kernel_type_configs = list(
csrc/quantization/machete/machete_mm_kernel.cuh
+2 -1
@@ -152,7 +152,8 @@ struct MacheteKernelTemplate {
152
153 int M = size<0>(layout_A), N = size<1>(layout_D), K = size<1>(layout_A);
154
155 - int const group_size = maybe_group_size.value_or(K);
155 + int const group_size =
156 + maybe_group_size == -1 ? K : maybe_group_size.value_or(K);
157 int const scale_k = (K + group_size - 1) / group_size;
158
159 TORCH_CHECK(size<0>(layout_A) == M && size<1>(layout_A) == K);
csrc/quantization/machete/machete_mm_launcher.cuh
+1 -1
@@ -71,7 +71,7 @@ torch::Tensor run_impl(PyTorchArguments args) {
71 auto arguments = MacheteKernel::create_arguments(
72 stream, A_ptr, layout_A, B_ptr, D_ptr, layout_D, C_ptr, layout_C, S_ptr,
73 layout_S, Z_ptr, layout_Z, args.alpha.value_or(1), args.beta.value_or(0),
74 - args.group_size.value_or(K));
74 + args.group_size);
75 TORCH_CHECK(MacheteKernel::can_implement(arguments),
76 "Machete kernel cannot be run with these arguments");
77
csrc/quantization/machete/machete_prepack_launcher.cuh
+1 -1
@@ -53,7 +53,7 @@ torch::Tensor prepack_impl(torch::Tensor const B) {
53 // clang-format on
54
55 // Allocate output
56 - torch::Tensor D = torch::empty_like(B);
56 + torch::Tensor D = torch::empty_like(B, {}, at::MemoryFormat::Contiguous);
57
58 prepack_B<PrepackedLayoutB>(stream, B_ptr, layout_Bt,
59 static_cast<ElementB*>(D.mutable_data_ptr()));
csrc/torch_bindings.cpp
+3
@@ -192,6 +192,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
192 "-> Tensor");
193 ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
194
195 + ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
196 + ops.impl("permute_cols", torch::kCUDA, &permute_cols);
197 +
198 // gptq_marlin Optimized Quantized GEMM for GPTQ.
199 ops.def(
200 "gptq_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
tests/kernels/test_machete_gemm.py
+3
@@ -31,6 +31,8 @@ MNK_SHAPES = [
31 (257, 4224, 4160),
32 (257, 4096, 4096),
33 (64, 4096, 4096),
34 + (1024, 4096, 8192),
35 + (1024, 8192, 4096),
36 ]
37
38 ACT_TYPES = [torch.float16, torch.bfloat16]
@@ -139,6 +141,7 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
141 output_ref = torch.matmul(a, w_ref)
142
143 for schedule in ops.machete_supported_schedules(wtype):
144 + print(f"Testing schedule {schedule}")
145 output = ops.machete_gemm(
146 a,
147 b_q=w_q_machete,
tests/kernels/test_permute_cols.py new
+15
@@ -0,0 +1,15 @@
1 +import pytest
2 +import torch
3 +
4 +from tests.kernels.utils import opcheck
5 +from vllm._custom_ops import permute_cols
6 +
7 +
8 +@pytest.mark.parametrize('shape', [(1, 512), (544, 4096), (67, 8192)])
9 +@pytest.mark.parametrize('dtype', [torch.bfloat16, torch.float16])
10 +def test_permute_cols(shape, dtype):
11 + x = torch.randn(shape, dtype=dtype).cuda()
12 + perm = torch.randperm(x.shape[1]).to(torch.int).cuda()
13 + opcheck(torch.ops._C.permute_cols, (x, perm))
14 + y = permute_cols(x, perm)
15 + torch.testing.assert_close(y, x[:, perm])
\ No newline at end of file
vllm/_custom_ops.py
+18 -1
@@ -438,7 +438,8 @@ try:
438 @torch.library.register_fake("_C::machete_prepack_B")
439 def machete_prepack_B_fake(b_q_weight: torch.Tensor,
440 b_type: ScalarType) -> torch.Tensor:
441 - return torch.empty_like(b_q_weight)
441 + return torch.empty_like(b_q_weight,
442 + memory_format=torch.contiguous_format)
443
444 @torch.library.register_fake("_C::causal_conv1d_fwd")
445 def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
@@ -625,6 +626,22 @@ def machete_prepack_B(b_q_weight: torch.Tensor,
626 return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
627
628
629 +# TODO: has to be a better way to do this
630 +try:
631 + torch.ops._C.permute_cols # noqa B018
632 +
633 + @torch.library.register_fake("_C::permute_cols")
634 + def _permute_cols_fake(a: torch.Tensor,
635 + perm: torch.Tensor) -> torch.Tensor:
636 + return torch.empty_like(a)
637 +except Exception:
638 + pass
639 +
640 +
641 +def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:
642 + return torch.ops._C.permute_cols(a, perm)
643 +
644 +
645 # fp8
646 def scaled_fp8_quant(
647 input: torch.Tensor,
vllm/model_executor/layers/quantization/awq_marlin.py
+5 -4
@@ -7,10 +7,11 @@ from vllm.logger import init_logger
7 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
10 +from vllm.model_executor.layers.quantization.utils import replace_parameter
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
13 marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
13 - replace_tensor, verify_marlin_supported, verify_marlin_supports_shape)
14 + verify_marlin_supported, verify_marlin_supports_shape)
15 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
16 from vllm.model_executor.parameter import (GroupQuantScaleParameter,
17 PackedvLLMParameter)
@@ -231,7 +232,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
232 size_k=layer.input_size_per_partition,
233 size_n=layer.output_size_per_partition,
234 num_bits=self.quant_config.quant_type.size_bits)
234 - replace_tensor(layer, "qweight", marlin_qweight)
235 + replace_parameter(layer, "qweight", marlin_qweight)
236
237 # Permute scales from AWQ format to marlin format.
238 marlin_scales = marlin_permute_scales(
@@ -239,7 +240,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
240 size_k=layer.input_size_per_partition,
241 size_n=layer.output_size_per_partition,
242 group_size=self.quant_config.group_size)
242 - replace_tensor(layer, "scales", marlin_scales)
243 + replace_parameter(layer, "scales", marlin_scales)
244
245 # Permute zero-points from AWQ format to marlin format.
246 marlin_zp = awq_to_marlin_zero_points(
@@ -247,7 +248,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
248 size_k=layer.num_groups,
249 size_n=layer.output_size_per_partition,
250 num_bits=self.quant_config.quant_type.size_bits)
250 - replace_tensor(layer, "qzeros", marlin_zp)
251 + replace_parameter(layer, "qzeros", marlin_zp)
252
253 # Not-used
254 layer.g_idx = marlin_make_empty_g_idx(device)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+36 -78
@@ -1,17 +1,16 @@
1 -from typing import Callable, List, Optional
1 +from typing import Callable, List, Optional, Set
2
3 import torch
4
5 -from vllm import _custom_ops as ops
5 +from vllm.logger import init_logger
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
8 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 ActivationOrdering)
10 +from vllm.model_executor.layers.quantization.kernels import (
11 + MPLinearLayerConfig, choose_mp_linear_kernel)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11 - apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
12 - marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
13 - marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
14 - verify_marlin_supports_shape)
13 + marlin_repeat_scales_on_all_ranks)
14 from vllm.model_executor.parameter import (BasevLLMParameter,
15 ChannelQuantScaleParameter,
16 GroupQuantScaleParameter,
@@ -19,6 +18,8 @@ from vllm.model_executor.parameter import (BasevLLMParameter,
18 RowvLLMParameter)
19 from vllm.scalar_type import scalar_types
20
21 +logger = init_logger(__name__)
22 +
23 __all__ = ["CompressedTensorsWNA16"]
24 WNA16_SUPPORTED_TYPES_MAP = {
25 4: scalar_types.uint4b8,
@@ -28,6 +29,7 @@ WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
29
30
31 class CompressedTensorsWNA16(CompressedTensorsScheme):
32 + _kernel_backends_being_used: Set[str] = set()
33
34 def __init__(self,
35 strategy: str,
@@ -52,35 +54,43 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
54
55 self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
56
55 - # Verify supported on platform.
56 - verify_marlin_supported(quant_type=self.quant_type,
57 - group_size=self.group_size)
58 -
57 @classmethod
58 def get_min_capability(cls) -> int:
59 # ampere and up
60 return 80
61
64 - def create_weights(self, layer: torch.nn.Module, input_size: int,
65 - output_partition_sizes: List[int],
62 + def create_weights(self, layer: torch.nn.Module, output_size: int,
63 + input_size: int, output_partition_sizes: List[int],
64 input_size_per_partition: int,
65 params_dtype: torch.dtype, weight_loader: Callable,
66 **kwargs):
67
68 output_size_per_partition = sum(output_partition_sizes)
69
70 + mp_linear_kernel_config = MPLinearLayerConfig(
71 + full_weight_shape=(input_size, output_size),
72 + partition_weight_shape=\
73 + (input_size_per_partition, output_size_per_partition),
74 + weight_type=self.quant_type,
75 + act_type=params_dtype,
76 + group_size=self.group_size,
77 + zero_points=False,
78 + has_g_idx=self.has_g_idx
79 + )
80 +
81 + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
82 +
83 + if kernel_type.__name__ not in self._kernel_backends_being_used:
84 + logger.info("Using %s for CompressedTensorsWNA16",
85 + kernel_type.__name__)
86 + self._kernel_backends_being_used.add(kernel_type.__name__)
87 +
88 # If group_size is -1, we are in channelwise case.
89 group_size = self.group_size if self.group_size != -1 else input_size
90 row_parallel = (input_size != input_size_per_partition)
91 partition_scales = not marlin_repeat_scales_on_all_ranks(
92 self.has_g_idx, self.group_size, row_parallel)
93
78 - verify_marlin_supports_shape(
79 - output_size_per_partition=output_size_per_partition,
80 - input_size_per_partition=input_size_per_partition,
81 - input_size=input_size,
82 - group_size=group_size)
83 -
94 scales_and_zp_size = input_size // group_size
95
96 if partition_scales:
@@ -137,69 +147,17 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
147 weight_loader=weight_loader)
148 layer.register_parameter("weight_g_idx", weight_g_idx)
149
140 - layer.input_size_per_partition = input_size_per_partition
141 - layer.output_size_per_partition = output_size_per_partition
142 - layer.input_size = input_size
143 - layer.group_size = group_size
150 + self.kernel = kernel_type(mp_linear_kernel_config,
151 + w_q_param_name="weight_packed",
152 + w_s_param_name="weight_scale",
153 + w_zp_param_name=None,
154 + w_gidx_param_name="weight_g_idx")
155
156 # Checkpoints are serialized in compressed-tensors format, which is
146 - # different from marlin format. Handle repacking here.
157 + # different from the format the kernel may want. Handle repacking here.
158 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
148 - device = layer.weight_packed.device
149 -
150 - # Allocate marlin workspace.
151 - layer.workspace = marlin_make_workspace(
152 - layer.output_size_per_partition, device)
153 -
154 - # Handle sorting for activation reordering if needed.
155 - if self.has_g_idx:
156 - g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.weight_g_idx)
157 - layer.g_idx_sort_indices = g_idx_sort_indices
158 - replace_tensor(layer, "weight_g_idx", g_idx)
159 - else:
160 - layer.weight_g_idx = marlin_make_empty_g_idx(device)
161 - layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
162 -
163 - # No zero-point
164 - layer.weight_zp = marlin_make_empty_g_idx(device)
165 - # Update for kernel
166 - layer.weight_packed = torch.nn.Parameter(
167 - layer.weight_packed.t().contiguous(), requires_grad=False)
168 - layer.weight_scale = torch.nn.Parameter(
169 - layer.weight_scale.squeeze().t().contiguous(), requires_grad=False)
170 -
171 - # Repack weights from compressed-tensors format to marlin format.
172 - marlin_qweight = ops.gptq_marlin_repack(
173 - layer.weight_packed,
174 - perm=layer.g_idx_sort_indices,
175 - size_k=layer.input_size_per_partition,
176 - size_n=layer.output_size_per_partition,
177 - num_bits=self.quant_type.size_bits)
178 - replace_tensor(layer, "weight_packed", marlin_qweight)
179 -
180 - # Permute scales from compressed-tensors format to marlin format.
181 - # scale is required on all partitions if activation reordering
182 - marlin_scales = marlin_permute_scales(
183 - layer.weight_scale,
184 - size_k=(layer.input_size
185 - if self.has_g_idx else layer.input_size_per_partition),
186 - size_n=layer.output_size_per_partition,
187 - group_size=layer.group_size)
188 - replace_tensor(layer, "weight_scale", marlin_scales)
159 + self.kernel.process_weights_after_loading(layer)
160
161 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
162 bias: Optional[torch.Tensor]) -> torch.Tensor:
192 -
193 - return apply_gptq_marlin_linear(
194 - input=x,
195 - weight=layer.weight_packed,
196 - weight_scale=layer.weight_scale,
197 - weight_zp=layer.weight_zp,
198 - g_idx=layer.weight_g_idx,
199 - g_idx_sort_indices=layer.g_idx_sort_indices,
200 - workspace=layer.workspace,
201 - wtype=self.quant_type,
202 - output_size_per_partition=layer.output_size_per_partition,
203 - input_size_per_partition=layer.input_size_per_partition,
204 - is_k_full=True,
205 - bias=bias)
163 + return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/gptq_marlin.py
+44 -89
@@ -1,7 +1,6 @@
1 -from typing import Any, Callable, Dict, List, Optional, Union
1 +from typing import Any, Callable, Dict, List, Optional, Set, Union
2
3 import torch
4 -from torch.nn import Parameter
4
5 from vllm import _custom_ops as ops
6 from vllm.logger import init_logger
@@ -11,12 +10,12 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 set_weight_attrs)
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 +from vllm.model_executor.layers.quantization.kernels import (
14 + MPLinearLayerConfig, choose_mp_linear_kernel)
15 +from vllm.model_executor.layers.quantization.utils import replace_parameter
16 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 - apply_gptq_marlin_linear, check_marlin_supported, marlin_is_k_full,
16 - marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
17 - marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
18 - marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
19 - verify_marlin_supports_shape)
17 + check_marlin_supported, marlin_moe_permute_scales,
18 + marlin_repeat_scales_on_all_ranks, verify_marlin_supported)
19 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
21 GroupQuantScaleParameter,
@@ -159,6 +158,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
158 quant_config: The GPTQ Marlin quantization config.
159 """
160
161 + _kernel_backends_being_used: Set[str] = set()
162 +
163 def __init__(self, quant_config: GPTQMarlinConfig) -> None:
164 self.quant_config = quant_config
165
@@ -176,25 +177,34 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
177 params_dtype: torch.dtype,
178 **extra_weight_attrs,
179 ) -> None:
179 -
180 - del output_size
180 output_size_per_partition = sum(output_partition_sizes)
181 is_row_parallel = input_size != input_size_per_partition
182 weight_loader = extra_weight_attrs.get("weight_loader")
183
184 + mp_linear_kernel_config = MPLinearLayerConfig(
185 + full_weight_shape=(input_size, output_size),
186 + partition_weight_shape=\
187 + (input_size_per_partition, output_size_per_partition),
188 + weight_type=self.quant_config.quant_type,
189 + act_type=params_dtype,
190 + group_size=self.quant_config.group_size,
191 + zero_points=False,
192 + has_g_idx=self.quant_config.desc_act
193 + )
194 +
195 + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
196 +
197 + if kernel_type.__name__ not in self._kernel_backends_being_used:
198 + logger.info("Using %s for GPTQMarlinLinearMethod",
199 + kernel_type.__name__)
200 + self._kernel_backends_being_used.add(kernel_type.__name__)
201 +
202 # Normalize group_size
203 if self.quant_config.group_size != -1:
204 group_size = self.quant_config.group_size
205 else:
206 group_size = input_size
207
191 - verify_marlin_supports_shape(
192 - output_size_per_partition=output_size_per_partition,
193 - input_size_per_partition=input_size_per_partition,
194 - input_size=input_size,
195 - group_size=group_size,
196 - )
197 -
208 # Determine sharding
209 if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act,
210 self.quant_config.group_size,
@@ -275,57 +285,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
285 layer.register_parameter("g_idx", g_idx)
286 layer.register_parameter("scales", scales)
287 layer.register_parameter("qzeros", qzeros)
278 - layer.input_size_per_partition = input_size_per_partition
279 - layer.output_size_per_partition = output_size_per_partition
280 - layer.input_size = input_size
281 - layer.is_k_full = marlin_is_k_full(self.quant_config.desc_act,
282 - is_row_parallel)
283 -
284 - # Checkpoints are serialized in AutoGPTQ format, which is different from the
285 - # marlin format. This function is called after the weights are loaded.
286 - # Here, we handle the repacking, including the activation reordering case.
287 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
288 - device = layer.qweight.device
288
290 - # required by torch.compile
291 - layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
292 - layer.scales = Parameter(layer.scales.data, requires_grad=False)
289 + self.kernel = kernel_type(mp_linear_kernel_config,
290 + w_q_param_name="qweight",
291 + w_s_param_name="scales",
292 + w_zp_param_name="qzeros",
293 + w_gidx_param_name="g_idx")
294
294 - # Allocate marlin workspace
295 - layer.workspace = marlin_make_workspace(
296 - layer.output_size_per_partition, device)
297 -
298 - # Handle sorting for activation reordering if needed.
299 - if self.quant_config.desc_act:
300 - g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.g_idx)
301 - layer.g_idx_sort_indices = g_idx_sort_indices
302 - replace_tensor(layer, "g_idx", g_idx)
303 - else:
304 - layer.g_idx = marlin_make_empty_g_idx(device)
305 - layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
306 -
307 - # No zero-point
308 - layer.zp = marlin_make_empty_g_idx(device)
309 -
310 - # Repack weights from autogptq format to marlin format.
311 - marlin_qweight = ops.gptq_marlin_repack(
312 - layer.qweight,
313 - perm=layer.g_idx_sort_indices,
314 - size_k=layer.input_size_per_partition,
315 - size_n=layer.output_size_per_partition,
316 - num_bits=self.quant_config.quant_type.size_bits,
317 - )
318 - replace_tensor(layer, "qweight", marlin_qweight)
319 -
320 - # Permute scales from autogptq format to marlin format.
321 - marlin_scales = marlin_permute_scales(
322 - layer.scales,
323 - size_k=(layer.input_size if self.quant_config.desc_act else
324 - layer.input_size_per_partition),
325 - size_n=layer.output_size_per_partition,
326 - group_size=self.quant_config.group_size,
327 - )
328 - replace_tensor(layer, "scales", marlin_scales)
295 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
296 + self.kernel.process_weights_after_loading(layer)
297
298 def apply(
299 self,
@@ -333,20 +301,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
301 x: torch.Tensor,
302 bias: Optional[torch.Tensor] = None,
303 ) -> torch.Tensor:
336 - return apply_gptq_marlin_linear(
337 - input=x,
338 - weight=layer.qweight,
339 - weight_scale=layer.scales,
340 - weight_zp=layer.zp,
341 - g_idx=layer.g_idx,
342 - g_idx_sort_indices=layer.g_idx_sort_indices,
343 - workspace=layer.workspace,
344 - wtype=self.quant_config.quant_type,
345 - output_size_per_partition=layer.output_size_per_partition,
346 - input_size_per_partition=layer.input_size_per_partition,
347 - is_k_full=layer.is_k_full,
348 - bias=bias,
349 - )
304 + return self.kernel.apply_weights(layer, x, bias)
305
306
307 class GPTQMarlinMoEMethod(FusedMoEMethodBase):
@@ -506,12 +461,12 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
461 w13_g_idx_sort_indices[e]]
462 w2_sorted_g_idx[e] = layer.w2_g_idx[e][
463 w2_g_idx_sort_indices[e]]
509 - replace_tensor(layer, "w13_g_idx", w13_sorted_g_idx)
510 - replace_tensor(layer, "w2_g_idx", w2_sorted_g_idx)
511 - replace_tensor(layer, "w13_g_idx_sort_indices",
512 - w13_g_idx_sort_indices)
513 - replace_tensor(layer, "w2_g_idx_sort_indices",
514 - w2_g_idx_sort_indices)
464 + replace_parameter(layer, "w13_g_idx", w13_sorted_g_idx)
465 + replace_parameter(layer, "w2_g_idx", w2_sorted_g_idx)
466 + replace_parameter(layer, "w13_g_idx_sort_indices",
467 + w13_g_idx_sort_indices)
468 + replace_parameter(layer, "w2_g_idx_sort_indices",
469 + w2_g_idx_sort_indices)
470 else:
471 # Reset g_idx related tensors
472 num_experts = layer.w13_g_idx.shape[0]
@@ -544,7 +499,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
499 layer.w13_qweight.shape[2],
500 self.quant_config.quant_type.size_bits,
501 )
547 - replace_tensor(layer, "w13_qweight", marlin_w13_qweight)
502 + replace_parameter(layer, "w13_qweight", marlin_w13_qweight)
503 marlin_w2_qweight = ops.gptq_marlin_moe_repack(
504 layer.w2_qweight,
505 layer.w2_g_idx_sort_indices,
@@ -552,7 +507,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
507 layer.w2_qweight.shape[2],
508 self.quant_config.quant_type.size_bits,
509 )
555 - replace_tensor(layer, "w2_qweight", marlin_w2_qweight)
510 + replace_parameter(layer, "w2_qweight", marlin_w2_qweight)
511 # Repack scales
512 marlin_w13_scales = marlin_moe_permute_scales(
513 s=layer.w13_scales,
@@ -560,14 +515,14 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
515 size_n=layer.w13_scales.shape[2],
516 group_size=self.quant_config.group_size,
517 )
563 - replace_tensor(layer, "w13_scales", marlin_w13_scales)
518 + replace_parameter(layer, "w13_scales", marlin_w13_scales)
519 marlin_w2_scales = marlin_moe_permute_scales(
520 s=layer.w2_scales,
521 size_k=layer.w2_scales.shape[1] * self.quant_config.pack_factor,
522 size_n=layer.w2_scales.shape[2],
523 group_size=self.quant_config.group_size,
524 )
570 - replace_tensor(layer, "w2_scales", marlin_w2_scales)
525 + replace_parameter(layer, "w2_scales", marlin_w2_scales)
526
527 def apply(
528 self,
vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py new
+83
@@ -0,0 +1,83 @@
1 +from abc import ABC, abstractmethod
2 +from dataclasses import dataclass
3 +from typing import Callable, Optional, Tuple
4 +
5 +import torch
6 +
7 +from vllm.model_executor.layers.quantization.utils import replace_parameter
8 +from vllm.scalar_type import ScalarType
9 +
10 +
11 +@dataclass
12 +class MPLinearLayerConfig:
13 + full_weight_shape: Tuple[int, int] # [in, out]
14 + partition_weight_shape: Tuple[int, int]
15 + weight_type: ScalarType
16 + act_type: torch.dtype
17 + group_size: int
18 + zero_points: bool
19 + has_g_idx: bool
20 +
21 +
22 +class MPLinearKernel(ABC):
23 +
24 + @classmethod
25 + @abstractmethod
26 + def get_min_capability(cls) -> int:
27 + raise NotImplementedError
28 +
29 + @classmethod
30 + @abstractmethod
31 + def can_implement(cls,
32 + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
33 + raise NotImplementedError
34 +
35 + def __init__(self,
36 + c: MPLinearLayerConfig,
37 + w_q_param_name: str,
38 + w_s_param_name: str,
39 + w_zp_param_name: Optional[str] = None,
40 + w_gidx_param_name: Optional[str] = None) -> None:
41 + assert self.can_implement(c)
42 + self.config = c
43 + self.w_q_name = w_q_param_name
44 + self.w_s_name = w_s_param_name
45 + self.w_zp_name = w_zp_param_name
46 + self.w_gidx_name = w_gidx_param_name
47 +
48 + @abstractmethod
49 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
50 + raise NotImplementedError
51 +
52 + @abstractmethod
53 + def apply_weights(self,
54 + layer: torch.nn.Module,
55 + x: torch.Tensor,
56 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
57 + raise NotImplementedError
58 +
59 + def _transform_param(self, layer: torch.nn.Module, name: Optional[str],
60 + fn: Callable) -> None:
61 + if name is not None and getattr(layer, name, None) is not None:
62 +
63 + old_param = getattr(layer, name)
64 + new_param = fn(old_param)
65 + # replace the parameter with torch.nn.Parameter for TorchDynamo
66 + # compatibility
67 + replace_parameter(
68 + layer, name,
69 + torch.nn.Parameter(new_param.data, requires_grad=False))
70 +
71 + def _get_weight_params(
72 + self, layer: torch.nn.Module
73 + ) -> Tuple[torch.Tensor, # w_q
74 + torch.Tensor, # w_s
75 + Optional[torch.Tensor], # w_zp,
76 + Optional[torch.Tensor] # w_gidx
77 + ]:
78 + return (
79 + getattr(layer, self.w_q_name),
80 + getattr(layer, self.w_s_name),
81 + getattr(layer, self.w_zp_name or "", None),
82 + getattr(layer, self.w_gidx_name or "", None),
83 + )
vllm/model_executor/layers/quantization/kernels/__init__.py new
+72
@@ -0,0 +1,72 @@
1 +import os
2 +from typing import List, Optional, Type
3 +
4 +from vllm.model_executor.layers.quantization.kernels.machete import (
5 + MacheteLinearKernel)
6 +from vllm.model_executor.layers.quantization.kernels.marlin import (
7 + MarlinLinearKernel)
8 +from vllm.model_executor.layers.quantization.kernels.MPLinearKernel import (
9 + MPLinearKernel, MPLinearLayerConfig)
10 +from vllm.platforms import current_platform
11 +
12 +# in priority/performance order (when available)
13 +_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
14 + MacheteLinearKernel,
15 + MarlinLinearKernel,
16 +]
17 +
18 +
19 +def choose_mp_linear_kernel(
20 + config: MPLinearLayerConfig,
21 + compute_capability: Optional[int] = None) -> Type[MPLinearKernel]:
22 + """
23 + Choose an MPLinearKernel that can implement the given config for the given
24 + compute capability. Attempts to choose the best kernel in terms of
25 + performance.
26 +
27 + Args:
28 + config (MPLinearLayerConfig): Description of the linear layer to be
29 + implemented.
30 + compute_capability (Optional[int], optional): The compute capability of
31 + the target device, if None uses `current_platform` to get the compute
32 + capability. Defaults to None.
33 +
34 + Raises:
35 + ValueError: If no kernel can implement the given config.
36 +
37 + Returns:
38 + Type[MPLinearKernel]: Chosen kernel.
39 + """
40 + if compute_capability is None:
41 + if current_platform is None:
42 + raise ValueError("Cannot determine compute capability")
43 + _cc = current_platform.get_device_capability()
44 + compute_capability = _cc[0] * 10 + _cc[1]
45 +
46 + failure_reasons = []
47 + for kernel in _POSSIBLE_KERNELS:
48 + if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\
49 + .split(","):
50 + failure_reasons.append(
51 + f' {kernel.__name__} disabled by environment variable')
52 + continue
53 +
54 + if kernel.get_min_capability() > compute_capability:
55 + failure_reasons.append(
56 + f"{kernel.__name__} requires capability "
57 + f"{kernel.get_min_capability()}, current compute capability "
58 + f"is {compute_capability}")
59 + continue
60 +
61 + can_implement, failure_reason = kernel.can_implement(config)
62 + if can_implement:
63 + return kernel
64 + else:
65 + failure_reasons.append(
66 + f' {kernel.__name__} cannot implement due to: {failure_reason}'
67 + )
68 +
69 + raise ValueError(
70 + "Failed to find a kernel that can implement the "\
71 + "WNA16 linear layer. Reasons: \n"
72 + + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/machete.py new
+118
@@ -0,0 +1,118 @@
1 +from functools import partial
2 +from typing import Optional, Tuple
3 +
4 +import torch
5 +
6 +from vllm import _custom_ops as ops
7 +from vllm.model_executor.layers.quantization.utils.machete_utils import (
8 + MACHETE_SUPPORTED_GROUP_SIZES, check_machete_supports_shape,
9 + query_machete_supported_quant_types)
10 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
11 + pack_weights_into_int32, unpack_weights_into_int32)
12 +from vllm.model_executor.parameter import (BasevLLMParameter,
13 + permute_param_layout_)
14 +
15 +from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
16 +
17 +
18 +class MacheteLinearKernel(MPLinearKernel):
19 +
20 + @classmethod
21 + def get_min_capability(cls) -> int:
22 + return 90
23 +
24 + @classmethod
25 + def can_implement(cls,
26 + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
27 + if c.has_g_idx and\
28 + c.partition_weight_shape[0] != c.full_weight_shape[0]:
29 + return False, "Act reordering currently not supported by Machete, "\
30 + "when the input features are partitioned across "\
31 + "devices"
32 +
33 + if c.zero_points:
34 + return False, "Zero points currently not supported by "\
35 + " Compressed Tensors + Machete. (Kernel supports it"\
36 + " but CompressedTensorsWNA16 does not so support has"\
37 + " not been added to MacheteWNA16Kernel yet"
38 +
39 + if c.weight_type not in query_machete_supported_quant_types(
40 + c.zero_points):
41 + return False, f"Quant type ({c.weight_type}) not supported by "\
42 + "Machete, supported types are: "\
43 + f"{query_machete_supported_quant_types(c.zero_points)}"
44 +
45 + if c.group_size not in MACHETE_SUPPORTED_GROUP_SIZES:
46 + return False, f"Group size ({c.group_size}) not supported by "\
47 + "Machete, supported group sizes are: "\
48 + f"{MACHETE_SUPPORTED_GROUP_SIZES}"
49 +
50 + return check_machete_supports_shape(c.partition_weight_shape[0],
51 + c.partition_weight_shape[1])
52 +
53 + # note assumes that
54 + # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
55 + # `weight_scale` is: {input_dim = 0, output_dim = 1}
56 + def process_weights_after_loading(self, layer: torch.nn.Module):
57 + c = self.config
58 +
59 + if c.has_g_idx:
60 + assert self.w_gidx_name is not None
61 + perm = torch.argsort(getattr(layer, self.w_gidx_name))\
62 + .to(torch.int)
63 +
64 + self.act_perm = lambda x: x[:, perm]
65 + # use `ops.permute_cols` if possible
66 + if c.act_type in [torch.float16, torch.bfloat16] \
67 + and c.partition_weight_shape[0] % 8 == 0:
68 + self.act_perm = partial(ops.permute_cols, perm=perm)
69 +
70 + def transform_w_q(x):
71 + assert isinstance(x, BasevLLMParameter)
72 + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
73 + if c.has_g_idx:
74 + x_unpacked = unpack_weights_into_int32(x.data,
75 + c.weight_type,
76 + packed_dim=0)
77 + x_perm = x_unpacked[perm, :]
78 + x.data = pack_weights_into_int32(x_perm,
79 + c.weight_type,
80 + packed_dim=0)
81 + x.data = ops.machete_prepack_B(x.data.t().contiguous().t(),
82 + self.config.weight_type)
83 + return x
84 +
85 + def transform_w_s(x):
86 + assert isinstance(x, BasevLLMParameter)
87 + permute_param_layout_(x, input_dim=0, output_dim=1)
88 + x.data = x.data.contiguous()
89 + return x
90 +
91 + # Repack weights and scales for Machete
92 + self._transform_param(layer, self.w_q_name, transform_w_q)
93 + self._transform_param(layer, self.w_s_name, transform_w_s)
94 +
95 + def apply_weights(self,
96 + layer: torch.nn.Module,
97 + x: torch.Tensor,
98 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
99 + c = self.config
100 + w_q, w_s, _, _ = self._get_weight_params(layer)
101 +
102 + x_2d = x.reshape(-1, x.shape[-1])
103 + out_shape = x.shape[:-1] + (c.partition_weight_shape[1], )
104 +
105 + if c.has_g_idx:
106 + x_2d = self.act_perm(x_2d)
107 +
108 + output = ops.machete_gemm(a=x_2d,
109 + b_q=w_q,
110 + b_type=c.weight_type,
111 + b_zeros=None,
112 + b_scales=w_s,
113 + b_group_size=c.group_size)
114 +
115 + if bias is not None:
116 + output.add_(bias) # In-place add
117 +
118 + return output.reshape(out_shape)
vllm/model_executor/layers/quantization/kernels/marlin.py new
+132
@@ -0,0 +1,132 @@
1 +from typing import Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm import _custom_ops as ops
6 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
7 + MARLIN_SUPPORTED_GROUP_SIZES, apply_gptq_marlin_linear,
8 + check_marlin_supports_shape, marlin_is_k_full, marlin_make_empty_g_idx,
9 + marlin_make_workspace, marlin_permute_scales, marlin_sort_g_idx,
10 + query_marlin_supported_quant_types)
11 +from vllm.model_executor.parameter import (BasevLLMParameter,
12 + permute_param_layout_)
13 +
14 +from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
15 +
16 +
17 +class MarlinLinearKernel(MPLinearKernel):
18 +
19 + @classmethod
20 + def get_min_capability(cls) -> int:
21 + return 80
22 +
23 + @classmethod
24 + def can_implement(cls,
25 + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
26 + if c.zero_points:
27 + return False, "Zero points currently not supported by "\
28 + " MarlinLinearKernel. Will be added when AWQMarlin "\
29 + "is migrated over to using MPLinearKernel backend"
30 +
31 + quant_types = query_marlin_supported_quant_types(c.zero_points)
32 + if c.weight_type not in quant_types:
33 + return False, f"Quant type ({c.weight_type}) not supported by"\
34 + f" Marlin, supported types are: {quant_types}"
35 +
36 + if c.group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
37 + return False, f"Group size ({c.group_size}) not supported by "\
38 + "Marlin, supported group sizes are: "\
39 + f"{MARLIN_SUPPORTED_GROUP_SIZES}"
40 +
41 + return check_marlin_supports_shape(c.partition_weight_shape[0],
42 + c.partition_weight_shape[1],
43 + c.full_weight_shape[1],
44 + c.group_size)
45 +
46 + # note assumes that
47 + # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
48 + # `weight_scale` is: {input_dim = 0, output_dim = 1}
49 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
50 + device = getattr(layer, self.w_q_name).device
51 + c = self.config
52 +
53 + row_parallel = (c.partition_weight_shape[0] != c.full_weight_shape[0])
54 + self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel)
55 +
56 + # Allocate marlin workspace.
57 + self.workspace = marlin_make_workspace(c.partition_weight_shape[1],
58 + device)
59 +
60 + # Default names since marlin requires empty parameters for these,
61 + # TODO: remove this requirement from marlin (allow optional tensors)
62 + if self.w_gidx_name is None:
63 + self.w_gidx_name = "g_idx"
64 + if self.w_zp_name is None:
65 + self.w_zp_name = "w_zp"
66 +
67 + if c.has_g_idx:
68 + g_idx, g_idx_sort_indices = marlin_sort_g_idx(
69 + getattr(layer, self.w_gidx_name))
70 + self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
71 + layer.g_idx_sort_indices = g_idx_sort_indices
72 + else:
73 + setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
74 + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
75 +
76 + if c.zero_points:
77 + pass
78 + # TODO (lucas): add the following when AWQMarlin is migrated over to
79 + # using MPLinearKernel backend
80 + # self._transform_param(layer, self.w_zp_name, lambda x: \
81 + # marlin_zero_points(
82 + # x,
83 + # size_k=c.partition_weight_shape[0],
84 + # size_n=c.partition_weight_shape[1],
85 + # num_bits=c.weight_type.size_bits))
86 + else:
87 + setattr(layer, self.w_zp_name, marlin_make_empty_g_idx(device))
88 +
89 + def transform_w_q(x):
90 + assert isinstance(x, BasevLLMParameter)
91 + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
92 + x.data = ops.gptq_marlin_repack(x.data.contiguous(),
93 + perm=layer.g_idx_sort_indices,
94 + size_k=c.partition_weight_shape[0],
95 + size_n=c.partition_weight_shape[1],
96 + num_bits=c.weight_type.size_bits)
97 + return x
98 +
99 + def transform_w_s(x):
100 + assert isinstance(x, BasevLLMParameter)
101 + permute_param_layout_(x, input_dim=0, output_dim=1)
102 + x.data = marlin_permute_scales(x.data.contiguous(),
103 + size_k=c.partition_weight_shape[0],
104 + size_n=c.partition_weight_shape[1],
105 + group_size=c.group_size)
106 + return x
107 +
108 + self._transform_param(layer, self.w_q_name, transform_w_q)
109 + self._transform_param(layer, self.w_s_name, transform_w_s)
110 +
111 + def apply_weights(self,
112 + layer: torch.nn.Module,
113 + x: torch.Tensor,
114 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
115 + c = self.config
116 + w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
117 +
118 + # `process_weights_after_loading` will ensure w_zp and w_gidx are not
119 + # None for marlin
120 + return apply_gptq_marlin_linear(
121 + input=x,
122 + weight=w_q,
123 + weight_scale=w_s,
124 + weight_zp=w_zp, # type: ignore
125 + g_idx=w_gidx, # type: ignore
126 + g_idx_sort_indices=layer.g_idx_sort_indices,
127 + workspace=self.workspace,
128 + wtype=c.weight_type,
129 + input_size_per_partition=c.partition_weight_shape[0],
130 + output_size_per_partition=c.partition_weight_shape[1],
131 + is_k_full=self.is_k_full,
132 + bias=bias)
vllm/model_executor/layers/quantization/utils/__init__.py
+3
@@ -0,0 +1,3 @@
1 +from .layer_utils import replace_parameter, update_tensor_inplace
2 +
3 +__all__ = ['update_tensor_inplace', 'replace_parameter']
vllm/model_executor/layers/quantization/utils/layer_utils.py new
+33
@@ -0,0 +1,33 @@
1 +from typing import Union
2 +
3 +import torch
4 +
5 +
6 +def update_tensor_inplace(dst: torch.Tensor, src: torch.Tensor):
7 + assert dst.dtype == src.dtype, "Tensors must have the same dtype"
8 +
9 + # update tensor shape and stride
10 + dst.as_strided_(src.shape, src.stride())
11 +
12 + # If not the same underlying storage move tensor data
13 + if dst.data_ptr() != src.data_ptr():
14 + dst.copy_(src)
15 + del src
16 +
17 +
18 +# Newly generated tensors need to replace existing tensors that are
19 +# already registered as parameters by vLLM (and won't be freed)
20 +def replace_parameter(mod: torch.nn.Module, name: str,
21 + new: Union[torch.Tensor, torch.nn.Parameter]) -> None:
22 +
23 + old = getattr(mod, name)
24 + if old.dtype == new.dtype and \
25 + old.untyped_storage().nbytes() == new.untyped_storage().nbytes():
26 + # If we can just update in-place to avoid re-registering
27 + # can be faster if the underlying storage is the same
28 + update_tensor_inplace(old, new)
29 + else:
30 + # Fallback re-register parameter
31 + if not isinstance(new, torch.nn.Parameter):
32 + new = torch.nn.Parameter(new)
33 + mod.register_parameter(name, torch.nn.Parameter(new))
vllm/model_executor/layers/quantization/utils/machete_utils.py new
+30
@@ -0,0 +1,30 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.scalar_type import ScalarType, scalar_types
6 +
7 +MACHETE_SUPPORTED_GROUP_SIZES = [-1, 128]
8 +MACHETE_PREPACKED_BLOCK_SHAPE = [64, 128]
9 +
10 +
11 +def query_machete_supported_quant_types(zero_points: bool) -> List[ScalarType]:
12 + if zero_points:
13 + return [scalar_types.uint4, scalar_types.uint8]
14 + else:
15 + return [scalar_types.uint4b8, scalar_types.uint8b128]
16 +
17 +
18 +def query_machete_supported_act_types(zero_points: bool) -> List[ScalarType]:
19 + return [torch.float16, torch.bfloat16]
20 +
21 +
22 +def check_machete_supports_shape(in_features: int, out_featrues: int) \
23 + -> Tuple[bool, Optional[str]]:
24 + if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
25 + return False, "Input features size must be divisible by "\
26 + f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}"
27 + if out_featrues % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
28 + return False, "Output features size must be divisible by "\
29 + f"{MACHETE_PREPACKED_BLOCK_SHAPE[1]}"
30 + return True, None
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+18 -11
@@ -120,6 +120,19 @@ def verify_marlin_supports_shape(output_size_per_partition: int,
120 "with --quantization gptq.")
121
122
123 +def check_marlin_supports_shape(output_size_per_partition: int,
124 + input_size_per_partition: int,
125 + input_size: int, group_size: int) \
126 + -> Tuple[bool, Optional[str]]:
127 + try:
128 + verify_marlin_supports_shape(output_size_per_partition,
129 + input_size_per_partition, input_size,
130 + group_size)
131 + except ValueError as e:
132 + return False, e.__str__()
133 + return True, None
134 +
135 +
136 def marlin_make_workspace(output_size_per_partition: int,
137 device: torch.device) -> torch.Tensor:
138 max_workspace_size = (output_size_per_partition //
@@ -148,6 +161,11 @@ def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
161 requires_grad=False)
162
163
164 +def marlin_make_empty_zp(device: torch.device) -> torch.Tensor:
165 + return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
166 + requires_grad=False)
167 +
168 +
169 def marlin_sort_g_idx(
170 g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
171 g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
@@ -240,17 +258,6 @@ def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
258 return marlin_zp
259
260
243 -# Newly generated tensors need to replace existing tensors that are
244 -# already registered as parameters by vLLM (and won't be freed)
245 -def replace_tensor(layer: torch.nn.Module, name: str,
246 - new_t: torch.Tensor) -> None:
247 - # It is important to use resize_() here since it ensures
248 - # the same buffer is reused
249 - getattr(layer, name).resize_(new_t.shape)
250 - getattr(layer, name).copy_(new_t)
251 - del new_t
252 -
253 -
261 def apply_gptq_marlin_linear(
262 input: torch.Tensor,
263 weight: torch.Tensor,
vllm/model_executor/layers/quantization/utils/quant_utils.py
+43
@@ -20,6 +20,49 @@ FUSED_LAYER_NAME_MAPPING = {
20 }
21
22
23 +def pack_weights_into_int32(w_q: torch.Tensor,
24 + wtype: ScalarType,
25 + packed_dim: int = 0):
26 + # move dim to pack to the end
27 + perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
28 + inv_perm = tuple(perm.index(i) for i in range(len(perm)))
29 + w_q_perm = w_q.permute(perm)
30 +
31 + pack_factor = 32 // wtype.size_bits
32 + mask = (1 << wtype.size_bits) - 1
33 +
34 + new_shape_perm = list(w_q_perm.shape)
35 + assert w_q_perm.shape[-1] % pack_factor == 0
36 + new_shape_perm[-1] //= pack_factor
37 +
38 + res = torch.zeros(new_shape_perm, dtype=torch.int32, device=w_q.device)
39 + for i in range(pack_factor):
40 + res |= (w_q_perm[..., i::pack_factor] & mask) << wtype.size_bits * i
41 +
42 + return res.permute(inv_perm)
43 +
44 +
45 +def unpack_weights_into_int32(w_q: torch.Tensor,
46 + wtype: ScalarType,
47 + packed_dim: int = 0):
48 + # move dim to pack to the end
49 + perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
50 + inv_perm = tuple(perm.index(i) for i in range(len(perm)))
51 + w_q_perm = w_q.permute(perm)
52 +
53 + pack_factor = 32 // wtype.size_bits
54 + mask = (1 << wtype.size_bits) - 1
55 +
56 + new_shape_perm = list(w_q_perm.shape)
57 + new_shape_perm[-1] *= pack_factor
58 +
59 + res = torch.zeros(new_shape_perm, dtype=torch.int32, device=w_q.device)
60 + for i in range(pack_factor):
61 + res[..., i::pack_factor] = (w_q_perm >> wtype.size_bits * i) & mask
62 +
63 + return res.permute(inv_perm)
64 +
65 +
66 def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
67 # prefix: model.layers.0.self_attn.q_proj
68 # proj_name: q_proj
vllm/model_executor/parameter.py
+58
@@ -328,6 +328,64 @@ class PackedvLLMParameter(ModelWeightParameter):
328 marlin_tile_size=self.marlin_tile_size)
329
330
331 +def permute_param_layout_(param: BasevLLMParameter, input_dim: int,
332 + output_dim: int, **kwargs) -> BasevLLMParameter:
333 + """
334 + Permute a parameter's layout to the specified input and output dimensions,
335 + useful for forcing the parameter into a known layout, for example, if I need
336 + a packed (quantized) weight matrix to be in the layout
337 + {input_dim = 0, output_dim = 1, packed_dim = 0}
338 + then I can call:
339 + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
340 + to ensure x is in the correct layout (permuting it to the correct layout if
341 + required, asserting if it cannot get it to the correct layout)
342 + """
343 +
344 + curr_input_dim = getattr(param, "input_dim", None)
345 + curr_output_dim = getattr(param, "output_dim", None)
346 +
347 + if curr_input_dim is None or curr_output_dim is None:
348 + assert param.data.dim() == 2,\
349 + "permute_param_layout_ only supports 2D parameters when either "\
350 + "input_dim or output_dim is not set"
351 +
352 + # if one of the dimensions is not set, set it to the opposite of the other
353 + # we can only do this since we asserted the parameter is 2D above
354 + if curr_input_dim is None:
355 + assert curr_output_dim is not None,\
356 + "either input or output dim must be set"
357 + curr_input_dim = (curr_output_dim + 1) % 2
358 + if curr_output_dim is None:
359 + assert curr_input_dim is not None,\
360 + "either input or output dim must be set"
361 + curr_output_dim = (curr_input_dim + 1) % 2
362 +
363 + # create permutation from the current layout to the layout with
364 + # self.input_dim at input_dim and self.output_dim at output_dim preserving
365 + # other dimensions
366 + perm = [
367 + i for i in range(param.data.dim())
368 + if i not in [curr_input_dim, curr_output_dim]
369 + ]
370 + perm.insert(input_dim, curr_input_dim)
371 + perm.insert(output_dim, curr_output_dim)
372 +
373 + if "packed_dim" in kwargs:
374 + assert hasattr(param, "packed_dim") and\
375 + param.packed_dim == perm[kwargs["packed_dim"]],\
376 + "permute_param_layout_ currently doesn't support repacking"
377 +
378 + param.data = param.data.permute(*perm)
379 + if hasattr(param, "_input_dim"):
380 + param._input_dim = input_dim
381 + if hasattr(param, "_output_dim"):
382 + param._output_dim = output_dim
383 + if "packed_dim" in kwargs and hasattr(param, "_packed_dim"):
384 + param._packed_dim = kwargs["packed_dim"]
385 +
386 + return param
387 +
388 +
389 def _adjust_shard_indexes_for_marlin(shard_size, shard_offset,
390 marlin_tile_size):
391 return shard_size * marlin_tile_size, shard_offset * marlin_tile_size