A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin (#7701)
Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Divakar Verma <137818590+divakar-amd@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Lucas Wilkinson committed
Sep 23, 2024 at 13:46 UTC
86e9c8df29a954a7a2fc46e9985fecc2a2e15ae8
27 files changed
+1005
-246
CMakeLists.txt
+1
@@ -223,6 +223,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
223
"csrc/quantization/gguf/gguf_kernel.cu"
224
"csrc/quantization/fp8/fp8_marlin.cu"
225
"csrc/custom_all_reduce.cu"
226
+ "csrc/permute_cols.cu"
227
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
228
"csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
229
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
benchmarks/kernels/benchmark_machete.py
+61
-13
@@ -4,8 +4,10 @@ import itertools
4
import math
5
import pickle as pkl
6
import time
7
-from typing import Callable, Iterable, List, Tuple
7
+from itertools import product
8
+from typing import Callable, Iterable, List, Optional, Tuple
9
10
+import pandas as pd
11
import torch
12
import torch.utils.benchmark as TBenchmark
13
from torch.utils.benchmark import Measurement as TMeasurement
@@ -84,6 +86,10 @@ def loop_over_weights(
86
fn(a, w_ref, w_q, w_s)
87
88
89
+_SWEEP_SCHEDULES_RESULTS: Optional[pd.DataFrame] = None
90
+_SWEEP_SCHEDULES_RESULTS_CSV: Optional[str] = None
91
+
92
+
93
def bench(atype: torch.dtype,
94
wtype: ScalarType,
95
group_size: int,
@@ -94,6 +100,8 @@ def bench(atype: torch.dtype,
100
sub_label: str,
101
benchmark_marlinv1: bool = True,
102
sweep_schedules: bool = True) -> Iterable[TMeasurement]:
103
+ global _SWEEP_SCHEDULES_RESULTS
104
+
105
a, weights = make_bench_tensors(atype, wtype, group_size, m, n, k)
106
sub_label += f", L={len(weights)}"
107
@@ -163,6 +171,11 @@ def bench(atype: torch.dtype,
171
best_schedule = None
172
schedules = ops.machete_supported_schedules(wtype)
173
for schedule in reversed(schedules):
174
+ schedule_M = int(schedule.split("_")[0].split("x")[1])
175
+
176
+ # Prune known bad schedules
177
+ if schedule_M >= 2 * max(m, 16) or schedule_M < m // 4:
178
+ continue
179
180
def run(a, _, w_q, w_s, schedule=schedule):
181
ops.machete_gemm(a,
@@ -175,6 +188,20 @@ def bench(atype: torch.dtype,
188
res = bench_fn(label, sub_label, "machete_best",
189
lambda: loop_over_weights(a, weights_machete, run))
190
191
+ results_row = {
192
+ "M": m,
193
+ "K": k,
194
+ "N": n,
195
+ "group_size": group_size,
196
+ "schedule": schedule,
197
+ "median": res.median,
198
+ }
199
+ if _SWEEP_SCHEDULES_RESULTS is None:
200
+ _SWEEP_SCHEDULES_RESULTS = pd.DataFrame(
201
+ columns=results_row.keys())
202
+ _SWEEP_SCHEDULES_RESULTS.\
203
+ loc[len(_SWEEP_SCHEDULES_RESULTS)] = results_row
204
+
205
print(f" {res.median:5.5} ", schedule)
206
if not best or res.median < best.median:
207
best = res
@@ -235,18 +262,22 @@ def run_square_bench(args):
262
dim_sizes = list(
263
range(args.dim_start, args.dim_end + 1, args.dim_increment))
264
MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
265
+
266
data = run(args.dtype, args.sweep_schedules, MKNs)
267
268
make_output(data, MKNs, f"square_bench-{args.dtype}")
269
270
271
def run_range_bench(args):
244
- dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
245
- n = len(dim_sizes)
246
- Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
247
- Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
248
- Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
249
- MKNs = list(zip(Ms, Ks, Ns))
272
+ m_start, k_start, n_start = [int(x) for x in args.dim_start.split(",")]
273
+ m_end, k_end, n_end = [int(x) for x in args.dim_end.split(",")]
274
+ m_increment, k_increment, n_increment = \
275
+ [int(x) for x in args.dim_increment.split(",")]
276
+ Ms = list(range(m_start, m_end + 1, m_increment))
277
+ Ks = list(range(k_start, k_end + 1, k_increment))
278
+ Ns = list(range(n_start, n_end + 1, n_increment))
279
+ MKNs = list(product(Ms, Ks, Ns))
280
+
281
data = run(args.dtype, args.sweep_schedules, MKNs)
282
283
make_output(data, MKNs, f"range_bench-{args.dtype}")
@@ -333,6 +364,9 @@ Benchmark Machete GEMM.
364
action="store_true",
365
help="Run a sweep over all supported schedules",
366
)
367
+ parser.add_argument("--sweep-csv-out",
368
+ help="CSV to store sweep results",
369
+ default="sch_sweep_results.csv")
370
subparsers = parser.add_subparsers(dest="cmd", required=True)
371
372
square_parser = subparsers.add_parser("square_bench")
@@ -342,12 +376,21 @@ Benchmark Machete GEMM.
376
square_parser.set_defaults(func=run_square_bench)
377
378
range_parser = subparsers.add_parser("range_bench")
345
- range_parser.add_argument("--dim-start", type=int, required=True)
346
- range_parser.add_argument("--dim-end", type=int, required=True)
347
- range_parser.add_argument("--dim-increment", type=int, required=True)
348
- range_parser.add_argument("--m-constant", type=int, default=None)
349
- range_parser.add_argument("--n-constant", type=int, default=None)
350
- range_parser.add_argument("--k-constant", type=int, default=None)
379
+ range_parser.add_argument(
380
+ "--dim-start",
381
+ type=str,
382
+ required=True,
383
+ help="Start value for M,K,N as common separated list")
384
+ range_parser.add_argument(
385
+ "--dim-end",
386
+ type=str,
387
+ required=True,
388
+ help="End value (inclusive) for M,K,N as common separated list")
389
+ range_parser.add_argument(
390
+ "--dim-increment",
391
+ type=str,
392
+ required=True,
393
+ help="Increment value for M,K,N as common separated list")
394
range_parser.set_defaults(func=run_range_bench)
395
396
model_parser = subparsers.add_parser("model_bench")
@@ -369,4 +412,9 @@ Benchmark Machete GEMM.
412
model_parser.set_defaults(func=run_model_bench)
413
414
args = parser.parse_args()
415
+
416
+ _SWEEP_SCHEDULES_RESULTS_CSV = args.sweep_csv_out
417
args.func(args)
418
+
419
+ if _SWEEP_SCHEDULES_RESULTS is not None:
420
+ _SWEEP_SCHEDULES_RESULTS.to_csv(_SWEEP_SCHEDULES_RESULTS_CSV)
benchmarks/kernels/requirements.txt
new
+1
@@ -0,0 +1 @@
1
+pandas
\ No newline at end of file
csrc/cutlass_extensions/torch_utils.hpp
+7
-1
@@ -68,7 +68,13 @@ static inline auto make_cute_layout(torch::Tensor const& tensor,
68
name, ".stride(", idx, ") to be ", StrideEle::value);
69
return StrideEle{};
70
} else {
71
- return tensor.stride(idx);
71
+ if (tensor.size(idx) == 1) {
72
+ // use 0 stride for dim with size 1, this is easier for
73
+ // cute/cutlass to optimize (helps the TMA code flatten dims)
74
+ return StrideEle{0};
75
+ } else {
76
+ return tensor.stride(idx);
77
+ }
78
}
79
} else {
80
// Extra strides are assumed to be 0 or 1
csrc/ops.h
+2
@@ -113,6 +113,8 @@ torch::Tensor prepack_B(torch::Tensor const& B,
113
114
}; // namespace machete
115
116
+torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
117
+
118
torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
119
torch::Tensor& b_meta,
120
torch::Tensor& b_scales,
csrc/permute_cols.cu
new
+88
@@ -0,0 +1,88 @@
1
+#include <torch/all.h>
2
+
3
+#include <ATen/cuda/CUDAContext.h>
4
+#include <c10/cuda/CUDAGuard.h>
5
+
6
+#include <cuda_fp16.h>
7
+
8
+static constexpr int default_threads = 256;
9
+static constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
10
+
11
+// For a given "a" of size [M,K] performs a permutation of the K columns based
12
+// on the given "perm" indices.
13
+// Currently only supports 16bit types (since we permute half types)
14
+__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
15
+ int const* __restrict__ perm_int_ptr,
16
+ int4* __restrict__ out_int4_ptr, int size_m,
17
+ int size_k, int block_rows) {
18
+ int start_row = block_rows * blockIdx.x;
19
+ int finish_row = start_row + block_rows;
20
+ if (finish_row > size_m) {
21
+ finish_row = size_m;
22
+ }
23
+ int cur_block_rows = std::max(finish_row - start_row, 0);
24
+
25
+ int row_stride = size_k * sizeof(half) / 16;
26
+
27
+ auto permute_row = [&](int row) {
28
+ int iters = size_k / default_threads;
29
+ int rest = size_k % default_threads;
30
+
31
+ int offset = row * row_stride;
32
+
33
+ half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
34
+ half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
35
+
36
+ int base_k = 0;
37
+
38
+ for (int i = 0; i < iters; i++) {
39
+ int cur_k = base_k + threadIdx.x;
40
+ int src_pos = perm_int_ptr[cur_k];
41
+
42
+ out_half[cur_k] = a_row_half[src_pos];
43
+
44
+ base_k += default_threads;
45
+ }
46
+
47
+ if (rest) {
48
+ if (threadIdx.x < rest) {
49
+ int cur_k = base_k + threadIdx.x;
50
+ int src_pos = perm_int_ptr[cur_k];
51
+
52
+ out_half[cur_k] = a_row_half[src_pos];
53
+ }
54
+ }
55
+ };
56
+
57
+ for (int i = 0; i < cur_block_rows; i++) {
58
+ int cur_row = start_row + i;
59
+ if (cur_row < size_m) {
60
+ permute_row(cur_row);
61
+ }
62
+ }
63
+}
64
+
65
+// More efficient version of A[..., perm]
66
+// taken from gptq_marlin.cu
67
+torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm) {
68
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
69
+ auto dev = A.get_device();
70
+ auto stream = at::cuda::getCurrentCUDAStream(dev);
71
+
72
+ TORCH_CHECK(A.scalar_type() == at::kHalf || A.scalar_type() == at::kBFloat16,
73
+ "Currently only 16bit types are supported");
74
+ TORCH_CHECK(A.is_contiguous(), "A must be contiguous");
75
+ TORCH_CHECK(A.size(-1) % 8 == 0,
76
+ "A columns must be a multiple of 8 (128bits)");
77
+ auto A_2d = A.view({-1, A.size(-1)});
78
+
79
+ torch::Tensor D = torch::empty_like(A);
80
+ int sms;
81
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
82
+ int block_rows = div_ceil(A_2d.size(0), sms);
83
+ permute_cols_kernel<<<sms, default_threads, 0, stream>>>(
84
+ reinterpret_cast<int4 const*>(A_2d.const_data_ptr()),
85
+ perm.const_data_ptr<int>(), reinterpret_cast<int4*>(D.mutable_data_ptr()),
86
+ A_2d.size(0), A_2d.size(1), block_rows);
87
+ return D;
88
+}
\ No newline at end of file
csrc/quantization/machete/generate.py
+127
-46
@@ -157,7 +157,7 @@ TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput
157
TmaCoop = EpilogueScheduleType.TmaWarpSpecializedCooperative
158
159
160
-@dataclass
160
+@dataclass(frozen=True)
161
class ScheduleConfig:
162
tile_shape_mn: Tuple[int, int]
163
cluster_shape_mnk: Tuple[int, int, int]
@@ -328,56 +328,137 @@ def generate():
328
# about how this works
329
SCRIPT_DIR = os.path.dirname(__file__)
330
331
- schedules = [
332
- ScheduleConfig(
333
- tile_shape_mn=tile_shape_mn,
334
- cluster_shape_mnk=cluster_shape_mnk,
335
- kernel_schedule=kernel_schedule,
336
- epilogue_schedule=epilogue_schedule,
337
- tile_scheduler=tile_scheduler,
338
- ) for tile_shape_mn, cluster_shape_mnk in (
339
- ((128, 16), (1, 1, 1)),
340
- ((128, 32), (1, 1, 1)),
341
- ((128, 64), (1, 1, 1)),
342
- ((128, 128), (1, 1, 1)),
343
- ) for kernel_schedule in (TmaMI, ) for epilogue_schedule in (TmaCoop, )
344
- for tile_scheduler in (TileSchedulerType.StreamK, )
345
- ]
331
+ schedule_common_params = dict(
332
+ kernel_schedule=TmaMI,
333
+ epilogue_schedule=TmaCoop,
334
+ tile_scheduler=TileSchedulerType.StreamK,
335
+ )
336
337
# For now we use the same heuristic for all types
338
+ # Heuristic is currently tuned for H100s
339
default_heuristic = [
349
- ("M > 64",
350
- ScheduleConfig(
351
- tile_shape_mn=(128, 128),
352
- cluster_shape_mnk=(1, 1, 1),
353
- kernel_schedule=TmaMI,
354
- epilogue_schedule=TmaCoop,
355
- tile_scheduler=TileSchedulerType.StreamK,
356
- )),
357
- ("M > 32",
358
- ScheduleConfig(
359
- tile_shape_mn=(128, 64),
360
- cluster_shape_mnk=(1, 1, 1),
361
- kernel_schedule=TmaMI,
362
- epilogue_schedule=TmaCoop,
363
- tile_scheduler=TileSchedulerType.StreamK,
364
- )),
365
- ("M > 16",
366
- ScheduleConfig(
367
- tile_shape_mn=(128, 32),
368
- cluster_shape_mnk=(1, 1, 1),
369
- kernel_schedule=TmaMI,
370
- epilogue_schedule=TmaCoop,
371
- tile_scheduler=TileSchedulerType.StreamK,
372
- )),
373
- (None,
374
- ScheduleConfig(tile_shape_mn=(128, 16),
375
- cluster_shape_mnk=(1, 1, 1),
376
- kernel_schedule=TmaMI,
377
- epilogue_schedule=TmaCoop,
378
- tile_scheduler=TileSchedulerType.StreamK))
340
+ #### M = 257+
341
+ (
342
+ "M > 256 && K <= 16384 && N <= 4096",
343
+ ScheduleConfig(
344
+ tile_shape_mn=(128, 128),
345
+ cluster_shape_mnk=(2, 1, 1),
346
+ **schedule_common_params # type: ignore
347
+ )),
348
+ (
349
+ "M > 256",
350
+ ScheduleConfig(
351
+ tile_shape_mn=(128, 256),
352
+ cluster_shape_mnk=(2, 1, 1),
353
+ **schedule_common_params # type: ignore
354
+ )),
355
+ #### M = 129-256
356
+ (
357
+ "M > 128 && K <= 4096 && N <= 4096",
358
+ ScheduleConfig(
359
+ tile_shape_mn=(128, 64),
360
+ cluster_shape_mnk=(2, 1, 1),
361
+ **schedule_common_params # type: ignore
362
+ )),
363
+ (
364
+ "M > 128 && K <= 8192 && N <= 8192",
365
+ ScheduleConfig(
366
+ tile_shape_mn=(128, 128),
367
+ cluster_shape_mnk=(2, 1, 1),
368
+ **schedule_common_params # type: ignore
369
+ )),
370
+ (
371
+ "M > 128",
372
+ ScheduleConfig(
373
+ tile_shape_mn=(128, 256),
374
+ cluster_shape_mnk=(2, 1, 1),
375
+ **schedule_common_params # type: ignore
376
+ )),
377
+ #### M = 65-128
378
+ (
379
+ "M > 64 && K <= 4069 && N <= 4069",
380
+ ScheduleConfig(
381
+ tile_shape_mn=(128, 32),
382
+ cluster_shape_mnk=(2, 1, 1),
383
+ **schedule_common_params # type: ignore
384
+ )),
385
+ (
386
+ "M > 64 && K <= 4069 && N <= 8192",
387
+ ScheduleConfig(
388
+ tile_shape_mn=(128, 64),
389
+ cluster_shape_mnk=(2, 1, 1),
390
+ **schedule_common_params # type: ignore
391
+ )),
392
+ (
393
+ "M > 64 && K >= 8192 && N >= 12288",
394
+ ScheduleConfig(
395
+ tile_shape_mn=(256, 128),
396
+ cluster_shape_mnk=(2, 1, 1),
397
+ **schedule_common_params # type: ignore
398
+ )),
399
+ (
400
+ "M > 64",
401
+ ScheduleConfig(
402
+ tile_shape_mn=(128, 128),
403
+ cluster_shape_mnk=(2, 1, 1),
404
+ **schedule_common_params # type: ignore
405
+ )),
406
+ #### M = 33-64
407
+ (
408
+ "M > 32 && K <= 6144 && N <= 6144",
409
+ ScheduleConfig(
410
+ tile_shape_mn=(128, 16),
411
+ cluster_shape_mnk=(1, 1, 1),
412
+ **schedule_common_params # type: ignore
413
+ )),
414
+ (
415
+ "M > 32 && K >= 16384 && N >= 12288",
416
+ ScheduleConfig(
417
+ tile_shape_mn=(256, 64),
418
+ cluster_shape_mnk=(2, 1, 1),
419
+ **schedule_common_params # type: ignore
420
+ )),
421
+ (
422
+ "M > 32",
423
+ ScheduleConfig(
424
+ tile_shape_mn=(128, 64),
425
+ cluster_shape_mnk=(2, 1, 1),
426
+ **schedule_common_params # type: ignore
427
+ )),
428
+ #### M = 17-32
429
+ (
430
+ "M > 16 && K <= 12288 && N <= 8192",
431
+ ScheduleConfig(
432
+ tile_shape_mn=(128, 32),
433
+ cluster_shape_mnk=(2, 1, 1),
434
+ **schedule_common_params # type: ignore
435
+ )),
436
+ (
437
+ "M > 16",
438
+ ScheduleConfig(
439
+ tile_shape_mn=(256, 32),
440
+ cluster_shape_mnk=(2, 1, 1),
441
+ **schedule_common_params # type: ignore
442
+ )),
443
+ #### M = 1-16
444
+ (
445
+ "N >= 26624",
446
+ ScheduleConfig(
447
+ tile_shape_mn=(256, 16),
448
+ cluster_shape_mnk=(1, 1, 1),
449
+ **schedule_common_params # type: ignore
450
+ )),
451
+ (
452
+ None,
453
+ ScheduleConfig(
454
+ tile_shape_mn=(128, 16),
455
+ cluster_shape_mnk=(1, 1, 1),
456
+ **schedule_common_params # type: ignore
457
+ )),
458
]
459
460
+ schedules = list(set([x[1] for x in default_heuristic]))
461
+
462
impl_configs = []
463
464
GPTQ_kernel_type_configs = list(
csrc/quantization/machete/machete_mm_kernel.cuh
+2
-1
@@ -152,7 +152,8 @@ struct MacheteKernelTemplate {
152
153
int M = size<0>(layout_A), N = size<1>(layout_D), K = size<1>(layout_A);
154
155
- int const group_size = maybe_group_size.value_or(K);
155
+ int const group_size =
156
+ maybe_group_size == -1 ? K : maybe_group_size.value_or(K);
157
int const scale_k = (K + group_size - 1) / group_size;
158
159
TORCH_CHECK(size<0>(layout_A) == M && size<1>(layout_A) == K);
csrc/quantization/machete/machete_mm_launcher.cuh
+1
-1
@@ -71,7 +71,7 @@ torch::Tensor run_impl(PyTorchArguments args) {
71
auto arguments = MacheteKernel::create_arguments(
72
stream, A_ptr, layout_A, B_ptr, D_ptr, layout_D, C_ptr, layout_C, S_ptr,
73
layout_S, Z_ptr, layout_Z, args.alpha.value_or(1), args.beta.value_or(0),
74
- args.group_size.value_or(K));
74
+ args.group_size);
75
TORCH_CHECK(MacheteKernel::can_implement(arguments),
76
"Machete kernel cannot be run with these arguments");
77
csrc/quantization/machete/machete_prepack_launcher.cuh
+1
-1
@@ -53,7 +53,7 @@ torch::Tensor prepack_impl(torch::Tensor const B) {
53
// clang-format on
54
55
// Allocate output
56
- torch::Tensor D = torch::empty_like(B);
56
+ torch::Tensor D = torch::empty_like(B, {}, at::MemoryFormat::Contiguous);
57
58
prepack_B<PrepackedLayoutB>(stream, B_ptr, layout_Bt,
59
static_cast<ElementB*>(D.mutable_data_ptr()));
csrc/torch_bindings.cpp
+3
@@ -192,6 +192,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
192
"-> Tensor");
193
ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
194
195
+ ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
196
+ ops.impl("permute_cols", torch::kCUDA, &permute_cols);
197
+
198
// gptq_marlin Optimized Quantized GEMM for GPTQ.
199
ops.def(
200
"gptq_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
tests/kernels/test_machete_gemm.py
+3
@@ -31,6 +31,8 @@ MNK_SHAPES = [
31
(257, 4224, 4160),
32
(257, 4096, 4096),
33
(64, 4096, 4096),
34
+ (1024, 4096, 8192),
35
+ (1024, 8192, 4096),
36
]
37
38
ACT_TYPES = [torch.float16, torch.bfloat16]
@@ -139,6 +141,7 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
141
output_ref = torch.matmul(a, w_ref)
142
143
for schedule in ops.machete_supported_schedules(wtype):
144
+ print(f"Testing schedule {schedule}")
145
output = ops.machete_gemm(
146
a,
147
b_q=w_q_machete,
tests/kernels/test_permute_cols.py
new
+15
@@ -0,0 +1,15 @@
1
+import pytest
2
+import torch
3
+
4
+from tests.kernels.utils import opcheck
5
+from vllm._custom_ops import permute_cols
6
+
7
+
8
+@pytest.mark.parametrize('shape', [(1, 512), (544, 4096), (67, 8192)])
9
+@pytest.mark.parametrize('dtype', [torch.bfloat16, torch.float16])
10
+def test_permute_cols(shape, dtype):
11
+ x = torch.randn(shape, dtype=dtype).cuda()
12
+ perm = torch.randperm(x.shape[1]).to(torch.int).cuda()
13
+ opcheck(torch.ops._C.permute_cols, (x, perm))
14
+ y = permute_cols(x, perm)
15
+ torch.testing.assert_close(y, x[:, perm])
\ No newline at end of file
vllm/_custom_ops.py
+18
-1
@@ -438,7 +438,8 @@ try:
438
@torch.library.register_fake("_C::machete_prepack_B")
439
def machete_prepack_B_fake(b_q_weight: torch.Tensor,
440
b_type: ScalarType) -> torch.Tensor:
441
- return torch.empty_like(b_q_weight)
441
+ return torch.empty_like(b_q_weight,
442
+ memory_format=torch.contiguous_format)
443
444
@torch.library.register_fake("_C::causal_conv1d_fwd")
445
def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
@@ -625,6 +626,22 @@ def machete_prepack_B(b_q_weight: torch.Tensor,
626
return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
627
628
629
+# TODO: has to be a better way to do this
630
+try:
631
+ torch.ops._C.permute_cols # noqa B018
632
+
633
+ @torch.library.register_fake("_C::permute_cols")
634
+ def _permute_cols_fake(a: torch.Tensor,
635
+ perm: torch.Tensor) -> torch.Tensor:
636
+ return torch.empty_like(a)
637
+except Exception:
638
+ pass
639
+
640
+
641
+def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:
642
+ return torch.ops._C.permute_cols(a, perm)
643
+
644
+
645
# fp8
646
def scaled_fp8_quant(
647
input: torch.Tensor,
vllm/model_executor/layers/quantization/awq_marlin.py
+5
-4
@@ -7,10 +7,11 @@ from vllm.logger import init_logger
7
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
10
+from vllm.model_executor.layers.quantization.utils import replace_parameter
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
13
marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
13
- replace_tensor, verify_marlin_supported, verify_marlin_supports_shape)
14
+ verify_marlin_supported, verify_marlin_supports_shape)
15
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
16
from vllm.model_executor.parameter import (GroupQuantScaleParameter,
17
PackedvLLMParameter)
@@ -231,7 +232,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
232
size_k=layer.input_size_per_partition,
233
size_n=layer.output_size_per_partition,
234
num_bits=self.quant_config.quant_type.size_bits)
234
- replace_tensor(layer, "qweight", marlin_qweight)
235
+ replace_parameter(layer, "qweight", marlin_qweight)
236
237
# Permute scales from AWQ format to marlin format.
238
marlin_scales = marlin_permute_scales(
@@ -239,7 +240,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
240
size_k=layer.input_size_per_partition,
241
size_n=layer.output_size_per_partition,
242
group_size=self.quant_config.group_size)
242
- replace_tensor(layer, "scales", marlin_scales)
243
+ replace_parameter(layer, "scales", marlin_scales)
244
245
# Permute zero-points from AWQ format to marlin format.
246
marlin_zp = awq_to_marlin_zero_points(
@@ -247,7 +248,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
248
size_k=layer.num_groups,
249
size_n=layer.output_size_per_partition,
250
num_bits=self.quant_config.quant_type.size_bits)
250
- replace_tensor(layer, "qzeros", marlin_zp)
251
+ replace_parameter(layer, "qzeros", marlin_zp)
252
253
# Not-used
254
layer.g_idx = marlin_make_empty_g_idx(device)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+36
-78
@@ -1,17 +1,16 @@
1
-from typing import Callable, List, Optional
1
+from typing import Callable, List, Optional, Set
2
3
import torch
4
5
-from vllm import _custom_ops as ops
5
+from vllm.logger import init_logger
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
8
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
ActivationOrdering)
10
+from vllm.model_executor.layers.quantization.kernels import (
11
+ MPLinearLayerConfig, choose_mp_linear_kernel)
12
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11
- apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
12
- marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
13
- marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
14
- verify_marlin_supports_shape)
13
+ marlin_repeat_scales_on_all_ranks)
14
from vllm.model_executor.parameter import (BasevLLMParameter,
15
ChannelQuantScaleParameter,
16
GroupQuantScaleParameter,
@@ -19,6 +18,8 @@ from vllm.model_executor.parameter import (BasevLLMParameter,
18
RowvLLMParameter)
19
from vllm.scalar_type import scalar_types
20
21
+logger = init_logger(__name__)
22
+
23
__all__ = ["CompressedTensorsWNA16"]
24
WNA16_SUPPORTED_TYPES_MAP = {
25
4: scalar_types.uint4b8,
@@ -28,6 +29,7 @@ WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
29
30
31
class CompressedTensorsWNA16(CompressedTensorsScheme):
32
+ _kernel_backends_being_used: Set[str] = set()
33
34
def __init__(self,
35
strategy: str,
@@ -52,35 +54,43 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
54
55
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
56
55
- # Verify supported on platform.
56
- verify_marlin_supported(quant_type=self.quant_type,
57
- group_size=self.group_size)
58
-
57
@classmethod
58
def get_min_capability(cls) -> int:
59
# ampere and up
60
return 80
61
64
- def create_weights(self, layer: torch.nn.Module, input_size: int,
65
- output_partition_sizes: List[int],
62
+ def create_weights(self, layer: torch.nn.Module, output_size: int,
63
+ input_size: int, output_partition_sizes: List[int],
64
input_size_per_partition: int,
65
params_dtype: torch.dtype, weight_loader: Callable,
66
**kwargs):
67
68
output_size_per_partition = sum(output_partition_sizes)
69
70
+ mp_linear_kernel_config = MPLinearLayerConfig(
71
+ full_weight_shape=(input_size, output_size),
72
+ partition_weight_shape=\
73
+ (input_size_per_partition, output_size_per_partition),
74
+ weight_type=self.quant_type,
75
+ act_type=params_dtype,
76
+ group_size=self.group_size,
77
+ zero_points=False,
78
+ has_g_idx=self.has_g_idx
79
+ )
80
+
81
+ kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
82
+
83
+ if kernel_type.__name__ not in self._kernel_backends_being_used:
84
+ logger.info("Using %s for CompressedTensorsWNA16",
85
+ kernel_type.__name__)
86
+ self._kernel_backends_being_used.add(kernel_type.__name__)
87
+
88
# If group_size is -1, we are in channelwise case.
89
group_size = self.group_size if self.group_size != -1 else input_size
90
row_parallel = (input_size != input_size_per_partition)
91
partition_scales = not marlin_repeat_scales_on_all_ranks(
92
self.has_g_idx, self.group_size, row_parallel)
93
78
- verify_marlin_supports_shape(
79
- output_size_per_partition=output_size_per_partition,
80
- input_size_per_partition=input_size_per_partition,
81
- input_size=input_size,
82
- group_size=group_size)
83
-
94
scales_and_zp_size = input_size // group_size
95
96
if partition_scales:
@@ -137,69 +147,17 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
147
weight_loader=weight_loader)
148
layer.register_parameter("weight_g_idx", weight_g_idx)
149
140
- layer.input_size_per_partition = input_size_per_partition
141
- layer.output_size_per_partition = output_size_per_partition
142
- layer.input_size = input_size
143
- layer.group_size = group_size
150
+ self.kernel = kernel_type(mp_linear_kernel_config,
151
+ w_q_param_name="weight_packed",
152
+ w_s_param_name="weight_scale",
153
+ w_zp_param_name=None,
154
+ w_gidx_param_name="weight_g_idx")
155
156
# Checkpoints are serialized in compressed-tensors format, which is
146
- # different from marlin format. Handle repacking here.
157
+ # different from the format the kernel may want. Handle repacking here.
158
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
148
- device = layer.weight_packed.device
149
-
150
- # Allocate marlin workspace.
151
- layer.workspace = marlin_make_workspace(
152
- layer.output_size_per_partition, device)
153
-
154
- # Handle sorting for activation reordering if needed.
155
- if self.has_g_idx:
156
- g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.weight_g_idx)
157
- layer.g_idx_sort_indices = g_idx_sort_indices
158
- replace_tensor(layer, "weight_g_idx", g_idx)
159
- else:
160
- layer.weight_g_idx = marlin_make_empty_g_idx(device)
161
- layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
162
-
163
- # No zero-point
164
- layer.weight_zp = marlin_make_empty_g_idx(device)
165
- # Update for kernel
166
- layer.weight_packed = torch.nn.Parameter(
167
- layer.weight_packed.t().contiguous(), requires_grad=False)
168
- layer.weight_scale = torch.nn.Parameter(
169
- layer.weight_scale.squeeze().t().contiguous(), requires_grad=False)
170
-
171
- # Repack weights from compressed-tensors format to marlin format.
172
- marlin_qweight = ops.gptq_marlin_repack(
173
- layer.weight_packed,
174
- perm=layer.g_idx_sort_indices,
175
- size_k=layer.input_size_per_partition,
176
- size_n=layer.output_size_per_partition,
177
- num_bits=self.quant_type.size_bits)
178
- replace_tensor(layer, "weight_packed", marlin_qweight)
179
-
180
- # Permute scales from compressed-tensors format to marlin format.
181
- # scale is required on all partitions if activation reordering
182
- marlin_scales = marlin_permute_scales(
183
- layer.weight_scale,
184
- size_k=(layer.input_size
185
- if self.has_g_idx else layer.input_size_per_partition),
186
- size_n=layer.output_size_per_partition,
187
- group_size=layer.group_size)
188
- replace_tensor(layer, "weight_scale", marlin_scales)
159
+ self.kernel.process_weights_after_loading(layer)
160
161
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
162
bias: Optional[torch.Tensor]) -> torch.Tensor:
192
-
193
- return apply_gptq_marlin_linear(
194
- input=x,
195
- weight=layer.weight_packed,
196
- weight_scale=layer.weight_scale,
197
- weight_zp=layer.weight_zp,
198
- g_idx=layer.weight_g_idx,
199
- g_idx_sort_indices=layer.g_idx_sort_indices,
200
- workspace=layer.workspace,
201
- wtype=self.quant_type,
202
- output_size_per_partition=layer.output_size_per_partition,
203
- input_size_per_partition=layer.input_size_per_partition,
204
- is_k_full=True,
205
- bias=bias)
163
+ return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/gptq_marlin.py
+44
-89
@@ -1,7 +1,6 @@
1
-from typing import Any, Callable, Dict, List, Optional, Union
1
+from typing import Any, Callable, Dict, List, Optional, Set, Union
2
3
import torch
4
-from torch.nn import Parameter
4
5
from vllm import _custom_ops as ops
6
from vllm.logger import init_logger
@@ -11,12 +10,12 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10
set_weight_attrs)
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
13
+from vllm.model_executor.layers.quantization.kernels import (
14
+ MPLinearLayerConfig, choose_mp_linear_kernel)
15
+from vllm.model_executor.layers.quantization.utils import replace_parameter
16
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
- apply_gptq_marlin_linear, check_marlin_supported, marlin_is_k_full,
16
- marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
17
- marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
18
- marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
19
- verify_marlin_supports_shape)
17
+ check_marlin_supported, marlin_moe_permute_scales,
18
+ marlin_repeat_scales_on_all_ranks, verify_marlin_supported)
19
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
21
GroupQuantScaleParameter,
@@ -159,6 +158,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
158
quant_config: The GPTQ Marlin quantization config.
159
"""
160
161
+ _kernel_backends_being_used: Set[str] = set()
162
+
163
def __init__(self, quant_config: GPTQMarlinConfig) -> None:
164
self.quant_config = quant_config
165
@@ -176,25 +177,34 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
177
params_dtype: torch.dtype,
178
**extra_weight_attrs,
179
) -> None:
179
-
180
- del output_size
180
output_size_per_partition = sum(output_partition_sizes)
181
is_row_parallel = input_size != input_size_per_partition
182
weight_loader = extra_weight_attrs.get("weight_loader")
183
184
+ mp_linear_kernel_config = MPLinearLayerConfig(
185
+ full_weight_shape=(input_size, output_size),
186
+ partition_weight_shape=\
187
+ (input_size_per_partition, output_size_per_partition),
188
+ weight_type=self.quant_config.quant_type,
189
+ act_type=params_dtype,
190
+ group_size=self.quant_config.group_size,
191
+ zero_points=False,
192
+ has_g_idx=self.quant_config.desc_act
193
+ )
194
+
195
+ kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
196
+
197
+ if kernel_type.__name__ not in self._kernel_backends_being_used:
198
+ logger.info("Using %s for GPTQMarlinLinearMethod",
199
+ kernel_type.__name__)
200
+ self._kernel_backends_being_used.add(kernel_type.__name__)
201
+
202
# Normalize group_size
203
if self.quant_config.group_size != -1:
204
group_size = self.quant_config.group_size
205
else:
206
group_size = input_size
207
191
- verify_marlin_supports_shape(
192
- output_size_per_partition=output_size_per_partition,
193
- input_size_per_partition=input_size_per_partition,
194
- input_size=input_size,
195
- group_size=group_size,
196
- )
197
-
208
# Determine sharding
209
if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act,
210
self.quant_config.group_size,
@@ -275,57 +285,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
285
layer.register_parameter("g_idx", g_idx)
286
layer.register_parameter("scales", scales)
287
layer.register_parameter("qzeros", qzeros)
278
- layer.input_size_per_partition = input_size_per_partition
279
- layer.output_size_per_partition = output_size_per_partition
280
- layer.input_size = input_size
281
- layer.is_k_full = marlin_is_k_full(self.quant_config.desc_act,
282
- is_row_parallel)
283
-
284
- # Checkpoints are serialized in AutoGPTQ format, which is different from the
285
- # marlin format. This function is called after the weights are loaded.
286
- # Here, we handle the repacking, including the activation reordering case.
287
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
288
- device = layer.qweight.device
288
290
- # required by torch.compile
291
- layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
292
- layer.scales = Parameter(layer.scales.data, requires_grad=False)
289
+ self.kernel = kernel_type(mp_linear_kernel_config,
290
+ w_q_param_name="qweight",
291
+ w_s_param_name="scales",
292
+ w_zp_param_name="qzeros",
293
+ w_gidx_param_name="g_idx")
294
294
- # Allocate marlin workspace
295
- layer.workspace = marlin_make_workspace(
296
- layer.output_size_per_partition, device)
297
-
298
- # Handle sorting for activation reordering if needed.
299
- if self.quant_config.desc_act:
300
- g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.g_idx)
301
- layer.g_idx_sort_indices = g_idx_sort_indices
302
- replace_tensor(layer, "g_idx", g_idx)
303
- else:
304
- layer.g_idx = marlin_make_empty_g_idx(device)
305
- layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
306
-
307
- # No zero-point
308
- layer.zp = marlin_make_empty_g_idx(device)
309
-
310
- # Repack weights from autogptq format to marlin format.
311
- marlin_qweight = ops.gptq_marlin_repack(
312
- layer.qweight,
313
- perm=layer.g_idx_sort_indices,
314
- size_k=layer.input_size_per_partition,
315
- size_n=layer.output_size_per_partition,
316
- num_bits=self.quant_config.quant_type.size_bits,
317
- )
318
- replace_tensor(layer, "qweight", marlin_qweight)
319
-
320
- # Permute scales from autogptq format to marlin format.
321
- marlin_scales = marlin_permute_scales(
322
- layer.scales,
323
- size_k=(layer.input_size if self.quant_config.desc_act else
324
- layer.input_size_per_partition),
325
- size_n=layer.output_size_per_partition,
326
- group_size=self.quant_config.group_size,
327
- )
328
- replace_tensor(layer, "scales", marlin_scales)
295
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
296
+ self.kernel.process_weights_after_loading(layer)
297
298
def apply(
299
self,
@@ -333,20 +301,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
301
x: torch.Tensor,
302
bias: Optional[torch.Tensor] = None,
303
) -> torch.Tensor:
336
- return apply_gptq_marlin_linear(
337
- input=x,
338
- weight=layer.qweight,
339
- weight_scale=layer.scales,
340
- weight_zp=layer.zp,
341
- g_idx=layer.g_idx,
342
- g_idx_sort_indices=layer.g_idx_sort_indices,
343
- workspace=layer.workspace,
344
- wtype=self.quant_config.quant_type,
345
- output_size_per_partition=layer.output_size_per_partition,
346
- input_size_per_partition=layer.input_size_per_partition,
347
- is_k_full=layer.is_k_full,
348
- bias=bias,
349
- )
304
+ return self.kernel.apply_weights(layer, x, bias)
305
306
307
class GPTQMarlinMoEMethod(FusedMoEMethodBase):
@@ -506,12 +461,12 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
461
w13_g_idx_sort_indices[e]]
462
w2_sorted_g_idx[e] = layer.w2_g_idx[e][
463
w2_g_idx_sort_indices[e]]
509
- replace_tensor(layer, "w13_g_idx", w13_sorted_g_idx)
510
- replace_tensor(layer, "w2_g_idx", w2_sorted_g_idx)
511
- replace_tensor(layer, "w13_g_idx_sort_indices",
512
- w13_g_idx_sort_indices)
513
- replace_tensor(layer, "w2_g_idx_sort_indices",
514
- w2_g_idx_sort_indices)
464
+ replace_parameter(layer, "w13_g_idx", w13_sorted_g_idx)
465
+ replace_parameter(layer, "w2_g_idx", w2_sorted_g_idx)
466
+ replace_parameter(layer, "w13_g_idx_sort_indices",
467
+ w13_g_idx_sort_indices)
468
+ replace_parameter(layer, "w2_g_idx_sort_indices",
469
+ w2_g_idx_sort_indices)
470
else:
471
# Reset g_idx related tensors
472
num_experts = layer.w13_g_idx.shape[0]
@@ -544,7 +499,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
499
layer.w13_qweight.shape[2],
500
self.quant_config.quant_type.size_bits,
501
)
547
- replace_tensor(layer, "w13_qweight", marlin_w13_qweight)
502
+ replace_parameter(layer, "w13_qweight", marlin_w13_qweight)
503
marlin_w2_qweight = ops.gptq_marlin_moe_repack(
504
layer.w2_qweight,
505
layer.w2_g_idx_sort_indices,
@@ -552,7 +507,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
507
layer.w2_qweight.shape[2],
508
self.quant_config.quant_type.size_bits,
509
)
555
- replace_tensor(layer, "w2_qweight", marlin_w2_qweight)
510
+ replace_parameter(layer, "w2_qweight", marlin_w2_qweight)
511
# Repack scales
512
marlin_w13_scales = marlin_moe_permute_scales(
513
s=layer.w13_scales,
@@ -560,14 +515,14 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
515
size_n=layer.w13_scales.shape[2],
516
group_size=self.quant_config.group_size,
517
)
563
- replace_tensor(layer, "w13_scales", marlin_w13_scales)
518
+ replace_parameter(layer, "w13_scales", marlin_w13_scales)
519
marlin_w2_scales = marlin_moe_permute_scales(
520
s=layer.w2_scales,
521
size_k=layer.w2_scales.shape[1] * self.quant_config.pack_factor,
522
size_n=layer.w2_scales.shape[2],
523
group_size=self.quant_config.group_size,
524
)
570
- replace_tensor(layer, "w2_scales", marlin_w2_scales)
525
+ replace_parameter(layer, "w2_scales", marlin_w2_scales)
526
527
def apply(
528
self,
vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py
new
+83
@@ -0,0 +1,83 @@
1
+from abc import ABC, abstractmethod
2
+from dataclasses import dataclass
3
+from typing import Callable, Optional, Tuple
4
+
5
+import torch
6
+
7
+from vllm.model_executor.layers.quantization.utils import replace_parameter
8
+from vllm.scalar_type import ScalarType
9
+
10
+
11
+@dataclass
12
+class MPLinearLayerConfig:
13
+ full_weight_shape: Tuple[int, int] # [in, out]
14
+ partition_weight_shape: Tuple[int, int]
15
+ weight_type: ScalarType
16
+ act_type: torch.dtype
17
+ group_size: int
18
+ zero_points: bool
19
+ has_g_idx: bool
20
+
21
+
22
+class MPLinearKernel(ABC):
23
+
24
+ @classmethod
25
+ @abstractmethod
26
+ def get_min_capability(cls) -> int:
27
+ raise NotImplementedError
28
+
29
+ @classmethod
30
+ @abstractmethod
31
+ def can_implement(cls,
32
+ c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
33
+ raise NotImplementedError
34
+
35
+ def __init__(self,
36
+ c: MPLinearLayerConfig,
37
+ w_q_param_name: str,
38
+ w_s_param_name: str,
39
+ w_zp_param_name: Optional[str] = None,
40
+ w_gidx_param_name: Optional[str] = None) -> None:
41
+ assert self.can_implement(c)
42
+ self.config = c
43
+ self.w_q_name = w_q_param_name
44
+ self.w_s_name = w_s_param_name
45
+ self.w_zp_name = w_zp_param_name
46
+ self.w_gidx_name = w_gidx_param_name
47
+
48
+ @abstractmethod
49
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
50
+ raise NotImplementedError
51
+
52
+ @abstractmethod
53
+ def apply_weights(self,
54
+ layer: torch.nn.Module,
55
+ x: torch.Tensor,
56
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
57
+ raise NotImplementedError
58
+
59
+ def _transform_param(self, layer: torch.nn.Module, name: Optional[str],
60
+ fn: Callable) -> None:
61
+ if name is not None and getattr(layer, name, None) is not None:
62
+
63
+ old_param = getattr(layer, name)
64
+ new_param = fn(old_param)
65
+ # replace the parameter with torch.nn.Parameter for TorchDynamo
66
+ # compatibility
67
+ replace_parameter(
68
+ layer, name,
69
+ torch.nn.Parameter(new_param.data, requires_grad=False))
70
+
71
+ def _get_weight_params(
72
+ self, layer: torch.nn.Module
73
+ ) -> Tuple[torch.Tensor, # w_q
74
+ torch.Tensor, # w_s
75
+ Optional[torch.Tensor], # w_zp,
76
+ Optional[torch.Tensor] # w_gidx
77
+ ]:
78
+ return (
79
+ getattr(layer, self.w_q_name),
80
+ getattr(layer, self.w_s_name),
81
+ getattr(layer, self.w_zp_name or "", None),
82
+ getattr(layer, self.w_gidx_name or "", None),
83
+ )
vllm/model_executor/layers/quantization/kernels/__init__.py
new
+72
@@ -0,0 +1,72 @@
1
+import os
2
+from typing import List, Optional, Type
3
+
4
+from vllm.model_executor.layers.quantization.kernels.machete import (
5
+ MacheteLinearKernel)
6
+from vllm.model_executor.layers.quantization.kernels.marlin import (
7
+ MarlinLinearKernel)
8
+from vllm.model_executor.layers.quantization.kernels.MPLinearKernel import (
9
+ MPLinearKernel, MPLinearLayerConfig)
10
+from vllm.platforms import current_platform
11
+
12
+# in priority/performance order (when available)
13
+_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
14
+ MacheteLinearKernel,
15
+ MarlinLinearKernel,
16
+]
17
+
18
+
19
+def choose_mp_linear_kernel(
20
+ config: MPLinearLayerConfig,
21
+ compute_capability: Optional[int] = None) -> Type[MPLinearKernel]:
22
+ """
23
+ Choose an MPLinearKernel that can implement the given config for the given
24
+ compute capability. Attempts to choose the best kernel in terms of
25
+ performance.
26
+
27
+ Args:
28
+ config (MPLinearLayerConfig): Description of the linear layer to be
29
+ implemented.
30
+ compute_capability (Optional[int], optional): The compute capability of
31
+ the target device, if None uses `current_platform` to get the compute
32
+ capability. Defaults to None.
33
+
34
+ Raises:
35
+ ValueError: If no kernel can implement the given config.
36
+
37
+ Returns:
38
+ Type[MPLinearKernel]: Chosen kernel.
39
+ """
40
+ if compute_capability is None:
41
+ if current_platform is None:
42
+ raise ValueError("Cannot determine compute capability")
43
+ _cc = current_platform.get_device_capability()
44
+ compute_capability = _cc[0] * 10 + _cc[1]
45
+
46
+ failure_reasons = []
47
+ for kernel in _POSSIBLE_KERNELS:
48
+ if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\
49
+ .split(","):
50
+ failure_reasons.append(
51
+ f' {kernel.__name__} disabled by environment variable')
52
+ continue
53
+
54
+ if kernel.get_min_capability() > compute_capability:
55
+ failure_reasons.append(
56
+ f"{kernel.__name__} requires capability "
57
+ f"{kernel.get_min_capability()}, current compute capability "
58
+ f"is {compute_capability}")
59
+ continue
60
+
61
+ can_implement, failure_reason = kernel.can_implement(config)
62
+ if can_implement:
63
+ return kernel
64
+ else:
65
+ failure_reasons.append(
66
+ f' {kernel.__name__} cannot implement due to: {failure_reason}'
67
+ )
68
+
69
+ raise ValueError(
70
+ "Failed to find a kernel that can implement the "\
71
+ "WNA16 linear layer. Reasons: \n"
72
+ + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/machete.py
new
+118
@@ -0,0 +1,118 @@
1
+from functools import partial
2
+from typing import Optional, Tuple
3
+
4
+import torch
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.model_executor.layers.quantization.utils.machete_utils import (
8
+ MACHETE_SUPPORTED_GROUP_SIZES, check_machete_supports_shape,
9
+ query_machete_supported_quant_types)
10
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
11
+ pack_weights_into_int32, unpack_weights_into_int32)
12
+from vllm.model_executor.parameter import (BasevLLMParameter,
13
+ permute_param_layout_)
14
+
15
+from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
16
+
17
+
18
+class MacheteLinearKernel(MPLinearKernel):
19
+
20
+ @classmethod
21
+ def get_min_capability(cls) -> int:
22
+ return 90
23
+
24
+ @classmethod
25
+ def can_implement(cls,
26
+ c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
27
+ if c.has_g_idx and\
28
+ c.partition_weight_shape[0] != c.full_weight_shape[0]:
29
+ return False, "Act reordering currently not supported by Machete, "\
30
+ "when the input features are partitioned across "\
31
+ "devices"
32
+
33
+ if c.zero_points:
34
+ return False, "Zero points currently not supported by "\
35
+ " Compressed Tensors + Machete. (Kernel supports it"\
36
+ " but CompressedTensorsWNA16 does not so support has"\
37
+ " not been added to MacheteWNA16Kernel yet"
38
+
39
+ if c.weight_type not in query_machete_supported_quant_types(
40
+ c.zero_points):
41
+ return False, f"Quant type ({c.weight_type}) not supported by "\
42
+ "Machete, supported types are: "\
43
+ f"{query_machete_supported_quant_types(c.zero_points)}"
44
+
45
+ if c.group_size not in MACHETE_SUPPORTED_GROUP_SIZES:
46
+ return False, f"Group size ({c.group_size}) not supported by "\
47
+ "Machete, supported group sizes are: "\
48
+ f"{MACHETE_SUPPORTED_GROUP_SIZES}"
49
+
50
+ return check_machete_supports_shape(c.partition_weight_shape[0],
51
+ c.partition_weight_shape[1])
52
+
53
+ # note assumes that
54
+ # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
55
+ # `weight_scale` is: {input_dim = 0, output_dim = 1}
56
+ def process_weights_after_loading(self, layer: torch.nn.Module):
57
+ c = self.config
58
+
59
+ if c.has_g_idx:
60
+ assert self.w_gidx_name is not None
61
+ perm = torch.argsort(getattr(layer, self.w_gidx_name))\
62
+ .to(torch.int)
63
+
64
+ self.act_perm = lambda x: x[:, perm]
65
+ # use `ops.permute_cols` if possible
66
+ if c.act_type in [torch.float16, torch.bfloat16] \
67
+ and c.partition_weight_shape[0] % 8 == 0:
68
+ self.act_perm = partial(ops.permute_cols, perm=perm)
69
+
70
+ def transform_w_q(x):
71
+ assert isinstance(x, BasevLLMParameter)
72
+ permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
73
+ if c.has_g_idx:
74
+ x_unpacked = unpack_weights_into_int32(x.data,
75
+ c.weight_type,
76
+ packed_dim=0)
77
+ x_perm = x_unpacked[perm, :]
78
+ x.data = pack_weights_into_int32(x_perm,
79
+ c.weight_type,
80
+ packed_dim=0)
81
+ x.data = ops.machete_prepack_B(x.data.t().contiguous().t(),
82
+ self.config.weight_type)
83
+ return x
84
+
85
+ def transform_w_s(x):
86
+ assert isinstance(x, BasevLLMParameter)
87
+ permute_param_layout_(x, input_dim=0, output_dim=1)
88
+ x.data = x.data.contiguous()
89
+ return x
90
+
91
+ # Repack weights and scales for Machete
92
+ self._transform_param(layer, self.w_q_name, transform_w_q)
93
+ self._transform_param(layer, self.w_s_name, transform_w_s)
94
+
95
+ def apply_weights(self,
96
+ layer: torch.nn.Module,
97
+ x: torch.Tensor,
98
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
99
+ c = self.config
100
+ w_q, w_s, _, _ = self._get_weight_params(layer)
101
+
102
+ x_2d = x.reshape(-1, x.shape[-1])
103
+ out_shape = x.shape[:-1] + (c.partition_weight_shape[1], )
104
+
105
+ if c.has_g_idx:
106
+ x_2d = self.act_perm(x_2d)
107
+
108
+ output = ops.machete_gemm(a=x_2d,
109
+ b_q=w_q,
110
+ b_type=c.weight_type,
111
+ b_zeros=None,
112
+ b_scales=w_s,
113
+ b_group_size=c.group_size)
114
+
115
+ if bias is not None:
116
+ output.add_(bias) # In-place add
117
+
118
+ return output.reshape(out_shape)
vllm/model_executor/layers/quantization/kernels/marlin.py
new
+132
@@ -0,0 +1,132 @@
1
+from typing import Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm import _custom_ops as ops
6
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
7
+ MARLIN_SUPPORTED_GROUP_SIZES, apply_gptq_marlin_linear,
8
+ check_marlin_supports_shape, marlin_is_k_full, marlin_make_empty_g_idx,
9
+ marlin_make_workspace, marlin_permute_scales, marlin_sort_g_idx,
10
+ query_marlin_supported_quant_types)
11
+from vllm.model_executor.parameter import (BasevLLMParameter,
12
+ permute_param_layout_)
13
+
14
+from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
15
+
16
+
17
+class MarlinLinearKernel(MPLinearKernel):
18
+
19
+ @classmethod
20
+ def get_min_capability(cls) -> int:
21
+ return 80
22
+
23
+ @classmethod
24
+ def can_implement(cls,
25
+ c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
26
+ if c.zero_points:
27
+ return False, "Zero points currently not supported by "\
28
+ " MarlinLinearKernel. Will be added when AWQMarlin "\
29
+ "is migrated over to using MPLinearKernel backend"
30
+
31
+ quant_types = query_marlin_supported_quant_types(c.zero_points)
32
+ if c.weight_type not in quant_types:
33
+ return False, f"Quant type ({c.weight_type}) not supported by"\
34
+ f" Marlin, supported types are: {quant_types}"
35
+
36
+ if c.group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
37
+ return False, f"Group size ({c.group_size}) not supported by "\
38
+ "Marlin, supported group sizes are: "\
39
+ f"{MARLIN_SUPPORTED_GROUP_SIZES}"
40
+
41
+ return check_marlin_supports_shape(c.partition_weight_shape[0],
42
+ c.partition_weight_shape[1],
43
+ c.full_weight_shape[1],
44
+ c.group_size)
45
+
46
+ # note assumes that
47
+ # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
48
+ # `weight_scale` is: {input_dim = 0, output_dim = 1}
49
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
50
+ device = getattr(layer, self.w_q_name).device
51
+ c = self.config
52
+
53
+ row_parallel = (c.partition_weight_shape[0] != c.full_weight_shape[0])
54
+ self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel)
55
+
56
+ # Allocate marlin workspace.
57
+ self.workspace = marlin_make_workspace(c.partition_weight_shape[1],
58
+ device)
59
+
60
+ # Default names since marlin requires empty parameters for these,
61
+ # TODO: remove this requirement from marlin (allow optional tensors)
62
+ if self.w_gidx_name is None:
63
+ self.w_gidx_name = "g_idx"
64
+ if self.w_zp_name is None:
65
+ self.w_zp_name = "w_zp"
66
+
67
+ if c.has_g_idx:
68
+ g_idx, g_idx_sort_indices = marlin_sort_g_idx(
69
+ getattr(layer, self.w_gidx_name))
70
+ self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
71
+ layer.g_idx_sort_indices = g_idx_sort_indices
72
+ else:
73
+ setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
74
+ layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
75
+
76
+ if c.zero_points:
77
+ pass
78
+ # TODO (lucas): add the following when AWQMarlin is migrated over to
79
+ # using MPLinearKernel backend
80
+ # self._transform_param(layer, self.w_zp_name, lambda x: \
81
+ # marlin_zero_points(
82
+ # x,
83
+ # size_k=c.partition_weight_shape[0],
84
+ # size_n=c.partition_weight_shape[1],
85
+ # num_bits=c.weight_type.size_bits))
86
+ else:
87
+ setattr(layer, self.w_zp_name, marlin_make_empty_g_idx(device))
88
+
89
+ def transform_w_q(x):
90
+ assert isinstance(x, BasevLLMParameter)
91
+ permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
92
+ x.data = ops.gptq_marlin_repack(x.data.contiguous(),
93
+ perm=layer.g_idx_sort_indices,
94
+ size_k=c.partition_weight_shape[0],
95
+ size_n=c.partition_weight_shape[1],
96
+ num_bits=c.weight_type.size_bits)
97
+ return x
98
+
99
+ def transform_w_s(x):
100
+ assert isinstance(x, BasevLLMParameter)
101
+ permute_param_layout_(x, input_dim=0, output_dim=1)
102
+ x.data = marlin_permute_scales(x.data.contiguous(),
103
+ size_k=c.partition_weight_shape[0],
104
+ size_n=c.partition_weight_shape[1],
105
+ group_size=c.group_size)
106
+ return x
107
+
108
+ self._transform_param(layer, self.w_q_name, transform_w_q)
109
+ self._transform_param(layer, self.w_s_name, transform_w_s)
110
+
111
+ def apply_weights(self,
112
+ layer: torch.nn.Module,
113
+ x: torch.Tensor,
114
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
115
+ c = self.config
116
+ w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
117
+
118
+ # `process_weights_after_loading` will ensure w_zp and w_gidx are not
119
+ # None for marlin
120
+ return apply_gptq_marlin_linear(
121
+ input=x,
122
+ weight=w_q,
123
+ weight_scale=w_s,
124
+ weight_zp=w_zp, # type: ignore
125
+ g_idx=w_gidx, # type: ignore
126
+ g_idx_sort_indices=layer.g_idx_sort_indices,
127
+ workspace=self.workspace,
128
+ wtype=c.weight_type,
129
+ input_size_per_partition=c.partition_weight_shape[0],
130
+ output_size_per_partition=c.partition_weight_shape[1],
131
+ is_k_full=self.is_k_full,
132
+ bias=bias)
vllm/model_executor/layers/quantization/utils/__init__.py
+3
@@ -0,0 +1,3 @@
1
+from .layer_utils import replace_parameter, update_tensor_inplace
2
+
3
+__all__ = ['update_tensor_inplace', 'replace_parameter']
vllm/model_executor/layers/quantization/utils/layer_utils.py
new
+33
@@ -0,0 +1,33 @@
1
+from typing import Union
2
+
3
+import torch
4
+
5
+
6
+def update_tensor_inplace(dst: torch.Tensor, src: torch.Tensor):
7
+ assert dst.dtype == src.dtype, "Tensors must have the same dtype"
8
+
9
+ # update tensor shape and stride
10
+ dst.as_strided_(src.shape, src.stride())
11
+
12
+ # If not the same underlying storage move tensor data
13
+ if dst.data_ptr() != src.data_ptr():
14
+ dst.copy_(src)
15
+ del src
16
+
17
+
18
+# Newly generated tensors need to replace existing tensors that are
19
+# already registered as parameters by vLLM (and won't be freed)
20
+def replace_parameter(mod: torch.nn.Module, name: str,
21
+ new: Union[torch.Tensor, torch.nn.Parameter]) -> None:
22
+
23
+ old = getattr(mod, name)
24
+ if old.dtype == new.dtype and \
25
+ old.untyped_storage().nbytes() == new.untyped_storage().nbytes():
26
+ # If we can just update in-place to avoid re-registering
27
+ # can be faster if the underlying storage is the same
28
+ update_tensor_inplace(old, new)
29
+ else:
30
+ # Fallback re-register parameter
31
+ if not isinstance(new, torch.nn.Parameter):
32
+ new = torch.nn.Parameter(new)
33
+ mod.register_parameter(name, torch.nn.Parameter(new))
vllm/model_executor/layers/quantization/utils/machete_utils.py
new
+30
@@ -0,0 +1,30 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.scalar_type import ScalarType, scalar_types
6
+
7
+MACHETE_SUPPORTED_GROUP_SIZES = [-1, 128]
8
+MACHETE_PREPACKED_BLOCK_SHAPE = [64, 128]
9
+
10
+
11
+def query_machete_supported_quant_types(zero_points: bool) -> List[ScalarType]:
12
+ if zero_points:
13
+ return [scalar_types.uint4, scalar_types.uint8]
14
+ else:
15
+ return [scalar_types.uint4b8, scalar_types.uint8b128]
16
+
17
+
18
+def query_machete_supported_act_types(zero_points: bool) -> List[ScalarType]:
19
+ return [torch.float16, torch.bfloat16]
20
+
21
+
22
+def check_machete_supports_shape(in_features: int, out_featrues: int) \
23
+ -> Tuple[bool, Optional[str]]:
24
+ if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
25
+ return False, "Input features size must be divisible by "\
26
+ f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}"
27
+ if out_featrues % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
28
+ return False, "Output features size must be divisible by "\
29
+ f"{MACHETE_PREPACKED_BLOCK_SHAPE[1]}"
30
+ return True, None
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+18
-11
@@ -120,6 +120,19 @@ def verify_marlin_supports_shape(output_size_per_partition: int,
120
"with --quantization gptq.")
121
122
123
+def check_marlin_supports_shape(output_size_per_partition: int,
124
+ input_size_per_partition: int,
125
+ input_size: int, group_size: int) \
126
+ -> Tuple[bool, Optional[str]]:
127
+ try:
128
+ verify_marlin_supports_shape(output_size_per_partition,
129
+ input_size_per_partition, input_size,
130
+ group_size)
131
+ except ValueError as e:
132
+ return False, e.__str__()
133
+ return True, None
134
+
135
+
136
def marlin_make_workspace(output_size_per_partition: int,
137
device: torch.device) -> torch.Tensor:
138
max_workspace_size = (output_size_per_partition //
@@ -148,6 +161,11 @@ def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
161
requires_grad=False)
162
163
164
+def marlin_make_empty_zp(device: torch.device) -> torch.Tensor:
165
+ return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
166
+ requires_grad=False)
167
+
168
+
169
def marlin_sort_g_idx(
170
g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
171
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
@@ -240,17 +258,6 @@ def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
258
return marlin_zp
259
260
243
-# Newly generated tensors need to replace existing tensors that are
244
-# already registered as parameters by vLLM (and won't be freed)
245
-def replace_tensor(layer: torch.nn.Module, name: str,
246
- new_t: torch.Tensor) -> None:
247
- # It is important to use resize_() here since it ensures
248
- # the same buffer is reused
249
- getattr(layer, name).resize_(new_t.shape)
250
- getattr(layer, name).copy_(new_t)
251
- del new_t
252
-
253
-
261
def apply_gptq_marlin_linear(
262
input: torch.Tensor,
263
weight: torch.Tensor,
vllm/model_executor/layers/quantization/utils/quant_utils.py
+43
@@ -20,6 +20,49 @@ FUSED_LAYER_NAME_MAPPING = {
20
}
21
22
23
+def pack_weights_into_int32(w_q: torch.Tensor,
24
+ wtype: ScalarType,
25
+ packed_dim: int = 0):
26
+ # move dim to pack to the end
27
+ perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
28
+ inv_perm = tuple(perm.index(i) for i in range(len(perm)))
29
+ w_q_perm = w_q.permute(perm)
30
+
31
+ pack_factor = 32 // wtype.size_bits
32
+ mask = (1 << wtype.size_bits) - 1
33
+
34
+ new_shape_perm = list(w_q_perm.shape)
35
+ assert w_q_perm.shape[-1] % pack_factor == 0
36
+ new_shape_perm[-1] //= pack_factor
37
+
38
+ res = torch.zeros(new_shape_perm, dtype=torch.int32, device=w_q.device)
39
+ for i in range(pack_factor):
40
+ res |= (w_q_perm[..., i::pack_factor] & mask) << wtype.size_bits * i
41
+
42
+ return res.permute(inv_perm)
43
+
44
+
45
+def unpack_weights_into_int32(w_q: torch.Tensor,
46
+ wtype: ScalarType,
47
+ packed_dim: int = 0):
48
+ # move dim to pack to the end
49
+ perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
50
+ inv_perm = tuple(perm.index(i) for i in range(len(perm)))
51
+ w_q_perm = w_q.permute(perm)
52
+
53
+ pack_factor = 32 // wtype.size_bits
54
+ mask = (1 << wtype.size_bits) - 1
55
+
56
+ new_shape_perm = list(w_q_perm.shape)
57
+ new_shape_perm[-1] *= pack_factor
58
+
59
+ res = torch.zeros(new_shape_perm, dtype=torch.int32, device=w_q.device)
60
+ for i in range(pack_factor):
61
+ res[..., i::pack_factor] = (w_q_perm >> wtype.size_bits * i) & mask
62
+
63
+ return res.permute(inv_perm)
64
+
65
+
66
def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
67
# prefix: model.layers.0.self_attn.q_proj
68
# proj_name: q_proj
vllm/model_executor/parameter.py
+58
@@ -328,6 +328,64 @@ class PackedvLLMParameter(ModelWeightParameter):
328
marlin_tile_size=self.marlin_tile_size)
329
330
331
+def permute_param_layout_(param: BasevLLMParameter, input_dim: int,
332
+ output_dim: int, **kwargs) -> BasevLLMParameter:
333
+ """
334
+ Permute a parameter's layout to the specified input and output dimensions,
335
+ useful for forcing the parameter into a known layout, for example, if I need
336
+ a packed (quantized) weight matrix to be in the layout
337
+ {input_dim = 0, output_dim = 1, packed_dim = 0}
338
+ then I can call:
339
+ permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
340
+ to ensure x is in the correct layout (permuting it to the correct layout if
341
+ required, asserting if it cannot get it to the correct layout)
342
+ """
343
+
344
+ curr_input_dim = getattr(param, "input_dim", None)
345
+ curr_output_dim = getattr(param, "output_dim", None)
346
+
347
+ if curr_input_dim is None or curr_output_dim is None:
348
+ assert param.data.dim() == 2,\
349
+ "permute_param_layout_ only supports 2D parameters when either "\
350
+ "input_dim or output_dim is not set"
351
+
352
+ # if one of the dimensions is not set, set it to the opposite of the other
353
+ # we can only do this since we asserted the parameter is 2D above
354
+ if curr_input_dim is None:
355
+ assert curr_output_dim is not None,\
356
+ "either input or output dim must be set"
357
+ curr_input_dim = (curr_output_dim + 1) % 2
358
+ if curr_output_dim is None:
359
+ assert curr_input_dim is not None,\
360
+ "either input or output dim must be set"
361
+ curr_output_dim = (curr_input_dim + 1) % 2
362
+
363
+ # create permutation from the current layout to the layout with
364
+ # self.input_dim at input_dim and self.output_dim at output_dim preserving
365
+ # other dimensions
366
+ perm = [
367
+ i for i in range(param.data.dim())
368
+ if i not in [curr_input_dim, curr_output_dim]
369
+ ]
370
+ perm.insert(input_dim, curr_input_dim)
371
+ perm.insert(output_dim, curr_output_dim)
372
+
373
+ if "packed_dim" in kwargs:
374
+ assert hasattr(param, "packed_dim") and\
375
+ param.packed_dim == perm[kwargs["packed_dim"]],\
376
+ "permute_param_layout_ currently doesn't support repacking"
377
+
378
+ param.data = param.data.permute(*perm)
379
+ if hasattr(param, "_input_dim"):
380
+ param._input_dim = input_dim
381
+ if hasattr(param, "_output_dim"):
382
+ param._output_dim = output_dim
383
+ if "packed_dim" in kwargs and hasattr(param, "_packed_dim"):
384
+ param._packed_dim = kwargs["packed_dim"]
385
+
386
+ return param
387
+
388
+
389
def _adjust_shard_indexes_for_marlin(shard_size, shard_offset,
390
marlin_tile_size):
391
return shard_size * marlin_tile_size, shard_offset * marlin_tile_size