A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
AQLM CUDA support (#3287)
Co-authored-by: mgoin <michael@neuralmagic.com>
James Fleming committed
Apr 23, 2024 at 13:59 UTC
2b7949c1c2e34de41d9cfc84dd0e377cc6bd58c2
14 files changed
+1592
-11
CMakeLists.txt
+1
@@ -173,6 +173,7 @@ set(VLLM_EXT_SRC
173
174
if(VLLM_GPU_LANG STREQUAL "CUDA")
175
list(APPEND VLLM_EXT_SRC
176
+ "csrc/quantization/aqlm/gemm_kernels.cu"
177
"csrc/quantization/awq/gemm_kernels.cu"
178
"csrc/quantization/marlin/marlin_cuda_kernel.cu"
179
"csrc/custom_all_reduce.cu")
benchmarks/kernels/benchmark_aqlm.py
new
+302
@@ -0,0 +1,302 @@
1
+import argparse
2
+import os
3
+import sys
4
+from typing import Optional
5
+
6
+import torch
7
+import torch.nn.functional as F
8
+
9
+from vllm._C import ops
10
+from vllm.model_executor.layers.quantization.aqlm import (
11
+ dequantize_weight, generic_dequantize_gemm, get_int_dtype,
12
+ optimized_dequantize_gemm)
13
+
14
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
15
+
16
+
17
+def torch_mult(
18
+ input: torch.Tensor, # [..., in_features]
19
+ weights: torch.Tensor,
20
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
21
+) -> torch.Tensor:
22
+ output = F.linear(input, weights)
23
+ return output
24
+
25
+
26
+def dequant_out_scale(
27
+ input: torch.Tensor, # [..., in_features]
28
+ codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
29
+ codebooks: torch.
30
+ Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
31
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
32
+ output_partition_sizes: torch.IntTensor,
33
+ bias: Optional[torch.Tensor],
34
+) -> torch.Tensor:
35
+
36
+ weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
37
+
38
+ if bias is None:
39
+ output = F.linear(input, weights, bias)
40
+ orig_shape = output.shape
41
+ flattened_output = output.view(-1, output.size(-1))
42
+ f_scales = scales.view(-1, scales.shape[0])
43
+ b_scales = f_scales.expand(flattened_output.shape[0], -1)
44
+ flattened_output *= b_scales
45
+ return flattened_output.view(orig_shape)
46
+ else:
47
+ b_scales = scales.view(scales.shape[:-3] + (-1, )).expand(
48
+ -1, weights.shape[1])
49
+ weights *= b_scales
50
+ return F.linear(input, weights, bias)
51
+
52
+
53
+def dequant_weight_scale(
54
+ input: torch.Tensor, # [..., in_features]
55
+ codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
56
+ codebooks: torch.
57
+ Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
58
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
59
+ output_partition_sizes: torch.IntTensor,
60
+ bias: Optional[torch.Tensor],
61
+) -> torch.Tensor:
62
+
63
+ weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
64
+
65
+ b_scales = scales.view(scales.shape[:-3] + (-1, )).expand(
66
+ -1, weights.shape[1])
67
+ weights *= b_scales
68
+ return F.linear(input, weights, bias)
69
+
70
+
71
+def dequant_no_scale(
72
+ input: torch.Tensor, # [..., in_features]
73
+ codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
74
+ codebooks: torch.
75
+ Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
76
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
77
+ output_partition_sizes: torch.IntTensor,
78
+ bias: Optional[torch.Tensor],
79
+) -> torch.Tensor:
80
+
81
+ weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
82
+
83
+ return F.linear(input, weights, bias)
84
+
85
+
86
+# Compare the optimized 1x16 and 2x8 cuda decompression/dequant kernels against
87
+# the generic pytorch version.
88
+# Just visual comparison.
89
+def dequant_test(k: int, parts: torch.tensor, nbooks: int, bits: int) -> None:
90
+
91
+ n = parts.sum().item()
92
+
93
+ device = torch.device('cuda:0')
94
+
95
+ code_range = (1 << bits) // 2
96
+ ingroups = 8
97
+
98
+ codes = torch.randint(-code_range,
99
+ code_range,
100
+ size=(n, k // ingroups, nbooks),
101
+ dtype=get_int_dtype(bits),
102
+ device=device)
103
+
104
+ codebooks = torch.randn(size=(parts.shape[0] * nbooks, 1 << bits, 1, 8),
105
+ dtype=torch.float16,
106
+ device=device)
107
+
108
+ count = 0
109
+ for index in range(16):
110
+ for i in range(8):
111
+ for book in range(nbooks):
112
+ codebooks[book, index, 0, i] = count * (10**book)
113
+ count += 1
114
+
115
+ print("codes shape", codes.shape)
116
+
117
+ for i in range(16):
118
+ for book in range(nbooks):
119
+ codes[0, i, book] = i
120
+ codes[0, -i, book] = i
121
+
122
+ weights = dequantize_weight(codes, codebooks, None)
123
+ weights2 = ops.aqlm_dequant(codes, codebooks, parts)
124
+
125
+ print("weights shape:", weights.shape)
126
+ print("weights2 shape:", weights2.shape)
127
+
128
+ print("weights are:", weights)
129
+ print("weights2 are:", weights2)
130
+
131
+ print("first 128 weights are", weights[0, 0:128].to(torch.int32))
132
+ print("first 128 weights2 are:", weights2[0, 0:128].to(torch.int32))
133
+
134
+ print("last 128 weights are", weights[0, -128:])
135
+ print("last 128 weights2 are:", weights2[0, -128:])
136
+
137
+
138
+def main():
139
+
140
+ parser = argparse.ArgumentParser(description="Benchmark aqlm performance.")
141
+
142
+ # Add arguments
143
+ parser.add_argument("--nbooks",
144
+ type=int,
145
+ default=1,
146
+ help="Number of codebooks (default: 1)")
147
+ parser.add_argument("--bits",
148
+ type=int,
149
+ default=16,
150
+ help="Number of bits per code element (default: 16)")
151
+ parser.add_argument(
152
+ "--test",
153
+ type=bool,
154
+ default=False,
155
+ help="Run the decompression/dequant tester rather than benchmarking "
156
+ "(default: False)")
157
+
158
+ # Parse the arguments
159
+ args = parser.parse_args()
160
+
161
+ # Extract values
162
+ nbooks = args.nbooks
163
+ bits = args.bits
164
+
165
+ if args.test:
166
+ dequant_test(4096, torch.tensor((4096, )), nbooks, bits)
167
+ return
168
+
169
+ # Otherwise, benchmark.
170
+ methods = [
171
+ ops.aqlm_gemm,
172
+ dequant_out_scale,
173
+ generic_dequantize_gemm,
174
+ optimized_dequantize_gemm,
175
+ dequant_weight_scale,
176
+ torch_mult,
177
+ dequant_no_scale,
178
+ ]
179
+
180
+ filename = f"./aqlm_benchmark_{nbooks}x{bits}.csv"
181
+ print(f"writing benchmarks to file {filename}")
182
+ with open(filename, "w") as f:
183
+ sys.stdout = f
184
+
185
+ print('m | k | n | n parts', end='')
186
+ for method in methods:
187
+ print(f" | {method.__name__.replace('_', ' ')} (µs)", end='')
188
+ print('')
189
+
190
+ # These are reasonable prefill sizes.
191
+ ksandpartions = ((4096, (4096, 4096, 4096)), (4096, (4096, )),
192
+ (4096, (11008, 11008)), (11008, (4096, )))
193
+
194
+ # reasonable ranges for m.
195
+ for m in [
196
+ 1, 2, 4, 8, 10, 12, 14, 16, 24, 32, 48, 52, 56, 64, 96, 112,
197
+ 128, 256, 512, 1024, 1536, 2048, 3072, 4096
198
+ ]:
199
+ print(f'{m}', file=sys.__stdout__)
200
+ for ksp in ksandpartions:
201
+ run_grid(m, ksp[0], torch.tensor(ksp[1]), nbooks, bits,
202
+ methods)
203
+
204
+ sys.stdout = sys.__stdout__
205
+
206
+
207
+def run_grid(m: int, k: int, parts: torch.tensor, nbooks: int, bits: int,
208
+ methods):
209
+
210
+ # I didn't see visible improvements from increasing these, but feel free :)
211
+ num_warmup_trials = 1
212
+ num_trials = 1
213
+
214
+ num_calls = 100
215
+
216
+ # warmup.
217
+ for method in methods:
218
+ for _ in range(num_warmup_trials):
219
+ run_timing(
220
+ num_calls=num_calls,
221
+ m=m,
222
+ k=k,
223
+ parts=parts,
224
+ nbooks=nbooks,
225
+ bits=bits,
226
+ method=method,
227
+ )
228
+
229
+ n = parts.sum().item()
230
+ print(f'{m} | {k} | {n} | {parts.tolist()}', end='')
231
+
232
+ for method in methods:
233
+ best_time_us = 1e20
234
+ for _ in range(num_trials):
235
+ kernel_dur_ms = run_timing(
236
+ num_calls=num_calls,
237
+ m=m,
238
+ k=k,
239
+ parts=parts,
240
+ nbooks=nbooks,
241
+ bits=bits,
242
+ method=method,
243
+ )
244
+
245
+ kernel_dur_us = 1000 * kernel_dur_ms
246
+
247
+ if kernel_dur_us < best_time_us:
248
+ best_time_us = kernel_dur_us
249
+
250
+ print(f' | {kernel_dur_us:.0f}', end='')
251
+
252
+ print('')
253
+
254
+
255
+def run_timing(num_calls: int, m: int, k: int, parts: torch.tensor,
256
+ nbooks: int, bits: int, method) -> float:
257
+
258
+ n = parts.sum().item()
259
+
260
+ device = torch.device('cuda:0')
261
+
262
+ input = torch.randn((1, m, k), dtype=torch.float16, device=device)
263
+
264
+ code_range = (1 << bits) // 2
265
+ ingroups = 8
266
+
267
+ codes = torch.randint(-code_range,
268
+ code_range,
269
+ size=(n, k // ingroups, nbooks),
270
+ dtype=get_int_dtype(bits),
271
+ device=device)
272
+
273
+ codebooks = torch.randn(size=(parts.shape[0] * nbooks, 1 << bits, 1, 8),
274
+ dtype=torch.float16,
275
+ device=device)
276
+
277
+ scales = torch.randn(size=(n, 1, 1, 1), dtype=torch.float16, device=device)
278
+
279
+ # for comparison to just a pytorch mult.
280
+ weights = torch.randn((n, k), dtype=torch.float16, device=device)
281
+
282
+ start_event = torch.cuda.Event(enable_timing=True)
283
+ end_event = torch.cuda.Event(enable_timing=True)
284
+
285
+ start_event.record()
286
+
287
+ if method is torch_mult:
288
+ for i in range(num_calls):
289
+ torch_mult(input, weights, scales)
290
+ else:
291
+ for i in range(num_calls):
292
+ method(input, codes, codebooks, scales, parts, None)
293
+
294
+ end_event.record()
295
+ end_event.synchronize()
296
+
297
+ dur_ms = start_event.elapsed_time(end_event) / num_calls
298
+ return dur_ms
299
+
300
+
301
+if __name__ == "__main__":
302
+ sys.exit(main())
csrc/ops.h
+15
@@ -86,6 +86,21 @@ void gelu_fast(
86
torch::Tensor& input);
87
88
#ifndef USE_ROCM
89
+torch::Tensor aqlm_gemm(
90
+ const torch::Tensor& input,
91
+ const torch::Tensor& codes,
92
+ const torch::Tensor& codebooks,
93
+ const torch::Tensor& scales,
94
+ const torch::Tensor& codebook_partition_sizes,
95
+ const std::optional<torch::Tensor>& bias
96
+);
97
+
98
+torch::Tensor aqlm_dequant(
99
+ const torch::Tensor& codes,
100
+ const torch::Tensor& codebooks,
101
+ const torch::Tensor& codebook_partition_sizes
102
+);
103
+
104
torch::Tensor awq_gemm(
105
torch::Tensor _in_feats,
106
torch::Tensor _kernel,
csrc/pybind.cpp
+2
@@ -63,6 +63,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
63
64
// Quantization ops
65
#ifndef USE_ROCM
66
+ ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
67
+ ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
68
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69
ops.def("marlin_gemm", &marlin_gemm, "Marlin Optimized Quantized GEMM for GPTQ");
70
ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
csrc/quantization/aqlm/gemm_kernels.cu
new
+712
@@ -0,0 +1,712 @@
1
+/*
2
+ * Modified by Neural Magic
3
+ * Adapted from https://github.com/Vahe1994/AQLM
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * http://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+
18
+#include <cuda.h>
19
+#include <cuda_fp16.h>
20
+#include <cuda_runtime.h>
21
+#include <torch/extension.h>
22
+#include <c10/cuda/CUDAStream.h>
23
+#include <c10/cuda/CUDAGuard.h>
24
+
25
+#include <iostream>
26
+#include <cstdlib>
27
+
28
+
29
+namespace vllm {
30
+namespace aqlm {
31
+
32
+__global__ void Code1x16MatVec(
33
+ const int4* __restrict__ A,
34
+ const int4* __restrict__ B,
35
+ int4* __restrict__ C,
36
+ const int4* __restrict__ codebook,
37
+ const int prob_m,
38
+ const int prob_k,
39
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
40
+ const int codebook_stride // as int4.
41
+) {
42
+ int a_gl_stride = prob_k / 8 / 8;
43
+ int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
44
+ bool pred = a_gl_rd < prob_m;
45
+
46
+ if (pred)
47
+ {
48
+ // advance to the correct codebook, this easy because we only multiply one column of the codebook.
49
+ auto codebook_size = &codebook_a_sizes.x;
50
+ while (a_gl_rd >= *codebook_size)
51
+ {
52
+ codebook += codebook_stride;
53
+ ++codebook_size;
54
+ }
55
+ }
56
+
57
+ int b_gl_rd = 0;
58
+ int c_gl_wr = a_gl_rd;
59
+ a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
60
+ int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
61
+
62
+ __shared__ int4 sh_b[32 * 9];
63
+ float res = 0;
64
+
65
+ int iters = (prob_k / 8 + 8 * 32 - 1) / (8 * 32);
66
+ while (iters--) {
67
+ // We pad shared memory to avoid bank conflicts during reads
68
+ __syncthreads();
69
+ for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
70
+ if (b_gl_rd + i < prob_k / 8)
71
+ sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
72
+ }
73
+ __syncthreads();
74
+ b_gl_rd += 32 * 8;
75
+
76
+ int b_sh_rd = 9 * (threadIdx.x % 32);
77
+ if (pred && a_gl_rd < a_gl_end) {
78
+ const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
79
+ #pragma unroll
80
+ for (int i = 0; i < 8; i++) {
81
+ uint32_t dec[4];
82
+ // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
83
+ // actually help us; this brings > 2x speedup.
84
+ asm volatile (
85
+ "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
86
+ : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
87
+ : "l"((void*) &codebook[enc[i]])
88
+ );
89
+ half2* a = reinterpret_cast<half2*>(&dec);
90
+ half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
91
+ half2 res2 = {};
92
+ #pragma unroll
93
+ for (int j = 0; j < 4; j++)
94
+ res2 = __hfma2(a[j], b[j], res2);
95
+ res += __half2float(res2.x) + __half2float(res2.y);
96
+ b_sh_rd++;
97
+ }
98
+ a_gl_rd += 32;
99
+ }
100
+ }
101
+
102
+ if (pred) {
103
+ #pragma unroll
104
+ for (int i = 16; i > 0; i /= 2)
105
+ res += __shfl_down_sync(0xffffffff, res, i);
106
+ if (threadIdx.x % 32 == 0)
107
+ reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
108
+ }
109
+}
110
+
111
+__global__ void Code2x8MatVec(
112
+ const int4* __restrict__ A,
113
+ const int4* __restrict__ B,
114
+ int4* __restrict__ C,
115
+ const int4* __restrict__ codebook,
116
+ int prob_m,
117
+ int prob_k,
118
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
119
+ const int codebook_stride // as int4.
120
+
121
+) {
122
+ int a_gl_stride = prob_k / 8 / 8;
123
+ int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
124
+ bool pred = a_gl_rd < prob_m;
125
+
126
+ if (pred)
127
+ {
128
+ // advance to the correct codebook, this easy because we only multiply one column of the codebook.
129
+ auto codebook_size = &codebook_a_sizes.x;
130
+ while (a_gl_rd >= *codebook_size)
131
+ {
132
+ codebook += codebook_stride;
133
+ ++codebook_size;
134
+ }
135
+ }
136
+
137
+ int b_gl_rd = 0;
138
+ int c_gl_wr = a_gl_rd;
139
+ a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
140
+ int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
141
+ int lane = threadIdx.x % 8;
142
+
143
+ extern __shared__ int4 sh[];
144
+ int4* sh_b = sh;
145
+ int4* sh_code = sh_b + 32 * 9;
146
+ int4* sh_code0 = sh_code;
147
+ int4* sh_code1 = sh_code + 256 * 8;
148
+
149
+ for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
150
+ int4 dec = codebook[i];
151
+ #pragma unroll
152
+ for (int j = 0; j < 8; j++)
153
+ sh_code[8 * i + (j + lane) % 8] = dec;
154
+ }
155
+ __syncthreads();
156
+
157
+ float res = 0;
158
+
159
+ int iters = (prob_k / 8 + 8 * 32 - 1) / (8 * 32);
160
+ while (iters--) {
161
+ // We pad shared memory to avoid bank conflicts during reads
162
+ __syncthreads();
163
+ for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
164
+ if (b_gl_rd + i < prob_k / 8)
165
+ sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
166
+ }
167
+ __syncthreads();
168
+ b_gl_rd += 32 * 8;
169
+
170
+ int b_sh_rd = 9 * (threadIdx.x % 32);
171
+ if (pred && a_gl_rd < a_gl_end) {
172
+ const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
173
+ #pragma unroll
174
+ for (int i = 0; i < 8; i++) {
175
+ half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
176
+ half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
177
+ half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
178
+ half2 res2 = {};
179
+ #pragma unroll
180
+ for (int j = 0; j < 4; j++)
181
+ res2 = __hfma2(__hadd2(a0[j], a1[j]), b[j], res2);
182
+ res += __half2float(res2.x) + __half2float(res2.y);
183
+ b_sh_rd++;
184
+ }
185
+ a_gl_rd += 32;
186
+ }
187
+ }
188
+
189
+ if (pred) {
190
+ #pragma unroll
191
+ for (int i = 16; i > 0; i /= 2)
192
+ res += __shfl_down_sync(0xffffffff, res, i);
193
+ if (threadIdx.x % 32 == 0)
194
+ reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
195
+ }
196
+}
197
+
198
+
199
+__global__ void Code1x16Dequant(
200
+ const int4* __restrict__ A,
201
+ int4* __restrict__ C,
202
+ const int4* __restrict__ codebook,
203
+ int prob_m,
204
+ int prob_k,
205
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, sums to m.
206
+ const int codebook_stride // as int4
207
+) {
208
+ int a_gl_stride = prob_k / 8 / 8;
209
+ int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
210
+ bool pred = a_gl_rd < prob_m;
211
+
212
+ if (pred)
213
+ {
214
+ // advance to the correct codebook, this easy because we only multiply one column of the codebook.
215
+ auto codebook_size = &codebook_a_sizes.x;
216
+ while (a_gl_rd >= *codebook_size)
217
+ {
218
+ codebook += codebook_stride;
219
+ ++codebook_size;
220
+ }
221
+ }
222
+
223
+ a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
224
+ int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
225
+
226
+ int c_gl_stride = prob_k / 8;
227
+ int c_gl_wr = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
228
+ c_gl_wr = c_gl_stride * c_gl_wr + (threadIdx.x % 32) * 8;
229
+
230
+ int iters = (prob_k / 8 - 1) / (8 * 32) + 1;
231
+ while (iters--) {
232
+ if (pred && a_gl_rd < a_gl_end) {
233
+ const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
234
+ #pragma unroll
235
+ for (int i = 0; i < 8; i++) {
236
+ int4 chunk;
237
+ auto dec = reinterpret_cast<uint32_t*>(&chunk);
238
+ // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
239
+ // actually help us; this brings > 2x speedup.
240
+ asm volatile (
241
+ "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
242
+ : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
243
+ : "l"((void*) &codebook[enc[i]])
244
+ );
245
+
246
+ C[a_gl_rd * 8 + i] = chunk;
247
+ }
248
+ }
249
+ a_gl_rd += 32;
250
+ }
251
+}
252
+
253
+
254
+__global__ void Code2x8Dequant(
255
+ const int4* __restrict__ A,
256
+ int4* __restrict__ C,
257
+ const int4* __restrict__ codebook,
258
+ int prob_m,
259
+ int prob_k,
260
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
261
+ const int codebook_stride // as int4
262
+) {
263
+ int a_gl_stride = prob_k / 8 / 8;
264
+ int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
265
+ bool pred = a_gl_rd < prob_m;
266
+
267
+ if (pred)
268
+ {
269
+ // advance to the correct codebook, this easy because we only multiply one column of the codebook.
270
+ auto codebook_size = &codebook_a_sizes.x;
271
+ while (a_gl_rd >= *codebook_size)
272
+ {
273
+ codebook += codebook_stride;
274
+ ++codebook_size;
275
+ }
276
+ }
277
+
278
+ a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
279
+ int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
280
+ int lane = threadIdx.x % 8;
281
+
282
+ int c_gl_stride = prob_k / 8;
283
+ int c_gl_wr = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
284
+ c_gl_wr = c_gl_stride * c_gl_wr + (threadIdx.x % 32) * 8;
285
+
286
+ extern __shared__ int4 sh[];
287
+ int4* sh_code = sh;
288
+ int4* sh_code0 = sh_code;
289
+ int4* sh_code1 = sh_code + 256 * 8;
290
+
291
+ for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
292
+ int4 dec = codebook[i];
293
+ #pragma unroll
294
+ for (int j = 0; j < 8; j++)
295
+ sh_code[8 * i + (j + lane) % 8] = dec;
296
+ }
297
+ __syncthreads();
298
+
299
+ float res = 0;
300
+
301
+ int iters = (prob_k / 8 - 1) / (8 * 32) + 1;
302
+ while (iters--) {
303
+ if (pred && a_gl_rd < a_gl_end) {
304
+ const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
305
+ #pragma unroll
306
+ for (int i = 0; i < 8; i++) {
307
+ int4 chunk;
308
+ half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
309
+ half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
310
+ #pragma unroll
311
+ for (int j = 0; j < 4; j++)
312
+ reinterpret_cast<half2*>(&chunk)[j] = __hadd2(a0[j], a1[j]);
313
+ C[a_gl_rd * 8 + i] = chunk;
314
+ }
315
+ }
316
+ a_gl_rd += 32;
317
+ }
318
+}
319
+
320
+inline int ceildiv(int a, int b) {
321
+ return (a + b - 1) / b;
322
+}
323
+
324
+const int THREAD_M = 16;
325
+
326
+void code1x16_matvec_cuda(
327
+ const void* __restrict__ A,
328
+ const void* __restrict__ B,
329
+ void* __restrict__ C,
330
+ const void* __restrict__ codebook,
331
+ int prob_m,
332
+ int prob_k,
333
+ const int4 codebook_a_sizes,
334
+ const int codebook_stride
335
+) {
336
+ int sms;
337
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
338
+ int waves = 0;
339
+ int thread_m;
340
+ do {
341
+ waves++;
342
+ thread_m = ceildiv(prob_m, waves * sms);
343
+ } while (thread_m > THREAD_M);
344
+
345
+ int blocks = ceildiv(prob_m, thread_m);
346
+ int threads = 32 * thread_m;
347
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
348
+ Code1x16MatVec<<<blocks, threads, 16*32*9, stream>>>(
349
+ (const int4*) A,
350
+ (const int4*) B,
351
+ (int4*) C,
352
+ (const int4*) codebook,
353
+ prob_m,
354
+ prob_k,
355
+ codebook_a_sizes,
356
+ codebook_stride
357
+ );
358
+}
359
+
360
+void code2x8_matvec_cuda(
361
+ const void* __restrict__ A,
362
+ const void* __restrict__ B,
363
+ void* __restrict__ C,
364
+ const void* __restrict__ codebook,
365
+ int prob_m,
366
+ int prob_k,
367
+ const int4 codebook_a_sizes,
368
+ const int codebook_stride
369
+) {
370
+ int sms;
371
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
372
+ int waves = 0;
373
+ int thread_m;
374
+ do {
375
+ waves++;
376
+ thread_m = ceildiv(prob_m, waves * sms);
377
+ } while (thread_m > THREAD_M);
378
+
379
+ int blocks = ceildiv(prob_m, thread_m);
380
+ int threads = 32 * thread_m;
381
+ int shared = 16 * (2 * 256 * 8 + 32 * 9);
382
+ cudaFuncSetAttribute(
383
+ Code2x8MatVec, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
384
+ );
385
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
386
+ Code2x8MatVec<<<blocks, threads, shared, stream>>>(
387
+ (const int4*) A,
388
+ (const int4*) B,
389
+ (int4*) C,
390
+ (const int4*) codebook,
391
+ prob_m,
392
+ prob_k,
393
+ codebook_a_sizes,
394
+ codebook_stride
395
+ );
396
+}
397
+
398
+void code1x16_dequant_cuda(
399
+ const void* __restrict__ A,
400
+ void* __restrict__ C,
401
+ const void* __restrict__ codebook,
402
+ int prob_m,
403
+ int prob_k,
404
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
405
+ const int codebook_stride // as int4.
406
+) {
407
+ int sms;
408
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
409
+ int waves = 0;
410
+ int thread_m;
411
+ do {
412
+ waves++;
413
+ thread_m = ceildiv(prob_m, waves * sms);
414
+ } while (thread_m > THREAD_M);
415
+
416
+ int blocks = ceildiv(prob_m, thread_m);
417
+ int threads = 32 * thread_m;
418
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
419
+ Code1x16Dequant<<<blocks, threads, 0, stream>>>(
420
+ (const int4*) A,
421
+ (int4*) C,
422
+ (const int4*) codebook,
423
+ prob_m,
424
+ prob_k,
425
+ codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
426
+ codebook_stride // as int4.
427
+ );
428
+}
429
+
430
+// Dequantizes the code and codebook into weights.
431
+void code2x8_dequant_cuda(
432
+ const void* __restrict__ A,
433
+ void* __restrict__ C,
434
+ const void* __restrict__ codebook,
435
+ int prob_m,
436
+ int prob_k,
437
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
438
+ const int codebook_stride // as int4
439
+) {
440
+ int sms;
441
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
442
+ int waves = 0;
443
+ int thread_m;
444
+ do {
445
+ waves++;
446
+ thread_m = ceildiv(prob_m, waves * sms);
447
+ } while (thread_m > THREAD_M);
448
+
449
+ int blocks = ceildiv(prob_m, thread_m);
450
+ int threads = 32 * thread_m;
451
+ int shared = 16 * (2 * 256 * 8 + 32 * 9);
452
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
453
+
454
+ cudaFuncSetAttribute(
455
+ Code2x8Dequant, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
456
+ );
457
+ Code2x8Dequant<<<blocks, threads, shared, stream>>>(
458
+ (const int4*) A,
459
+ (int4*) C,
460
+ (const int4*) codebook,
461
+ prob_m,
462
+ prob_k,
463
+ codebook_a_sizes,
464
+ codebook_stride
465
+ );
466
+}
467
+
468
+int codebook_stride(const torch::Tensor& codebooks)
469
+{
470
+ return codebooks.stride(0) * codebooks.element_size() / sizeof(int4);
471
+}
472
+
473
+void code1x16_matvec(
474
+ const torch::Tensor& A,
475
+ const torch::Tensor& B,
476
+ torch::Tensor& C,
477
+ const torch::Tensor& codebook,
478
+ const int4 codebook_a_sizes // cumulative sizes of A spanning each codebook, at most 3 long.
479
+) {
480
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
481
+ int prob_m = C.size(0);
482
+ int prob_k = B.size(0);
483
+
484
+ code1x16_matvec_cuda(
485
+ A.data_ptr(),
486
+ B.data_ptr(),
487
+ C.data_ptr(),
488
+ codebook.data_ptr(),
489
+ prob_m,
490
+ prob_k,
491
+ codebook_a_sizes,
492
+ codebook_stride(codebook)
493
+ );
494
+}
495
+
496
+torch::Tensor code1x16_matmat(
497
+ const torch::Tensor& input,
498
+ const torch::Tensor& codes,
499
+ const torch::Tensor& codebooks,
500
+ const torch::Tensor& scales,
501
+ const int4 codebook_a_sizes,
502
+ const std::optional<torch::Tensor>& bias) {
503
+ auto input_sizes = input.sizes();
504
+ auto out_features = codes.size(0) * codebooks.size(2);
505
+ auto flat_input = input.reshape({-1, input.size(-1)});
506
+ auto flat_output = torch::empty({flat_input.size(0), out_features},
507
+ torch::TensorOptions()
508
+ .dtype(input.dtype())
509
+ .device(input.device())
510
+ );
511
+
512
+ for (int i = 0; i < flat_input.size(0); ++i) {
513
+ auto input_vec = flat_input.index({i});
514
+ auto output_vec = flat_output.index({i});
515
+ code1x16_matvec(
516
+ codes.squeeze(2),
517
+ input_vec,
518
+ output_vec,
519
+ codebooks,
520
+ codebook_a_sizes
521
+ );
522
+ }
523
+ flat_output *= scales.flatten().unsqueeze(0);
524
+
525
+ if (bias.has_value()) {
526
+ flat_output += bias->unsqueeze(0);
527
+ }
528
+
529
+ auto output_sizes = input_sizes.vec();
530
+ output_sizes.pop_back();
531
+ output_sizes.push_back(-1);
532
+ auto output = flat_output.reshape(output_sizes);
533
+ return output;
534
+}
535
+
536
+void code2x8_matvec(
537
+ const torch::Tensor& A,
538
+ const torch::Tensor& B,
539
+ torch::Tensor& C,
540
+ const torch::Tensor& codebook,
541
+ const int4 codebook_a_sizes
542
+) {
543
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
544
+ int prob_m = C.size(0);
545
+ int prob_k = B.size(0);
546
+ code2x8_matvec_cuda(
547
+ A.data_ptr(),
548
+ B.data_ptr(),
549
+ C.data_ptr(),
550
+ codebook.data_ptr(),
551
+ prob_m,
552
+ prob_k,
553
+ codebook_a_sizes,
554
+ 2 * codebook_stride(codebook)
555
+ );
556
+}
557
+
558
+torch::Tensor code2x8_matmat(
559
+ const torch::Tensor& input,
560
+ const torch::Tensor& codes,
561
+ const torch::Tensor& codebooks,
562
+ const torch::Tensor& scales,
563
+ const int4 codebook_a_sizes,
564
+ const std::optional<torch::Tensor>& bias
565
+) {
566
+ auto input_sizes = input.sizes();
567
+ auto out_features = codes.size(0) * codebooks.size(2);
568
+ auto flat_input = input.reshape({-1, input.size(-1)});
569
+ auto flat_output = torch::empty({flat_input.size(0), out_features},
570
+ torch::TensorOptions()
571
+ .dtype(input.dtype())
572
+ .device(input.device())
573
+ );
574
+
575
+ for (int i = 0; i < flat_input.size(0); ++i) {
576
+ auto input_vec = flat_input.index({i});
577
+ auto output_vec = flat_output.index({i});
578
+ code2x8_matvec(
579
+ codes.squeeze(2),
580
+ input_vec,
581
+ output_vec,
582
+ codebooks,
583
+ codebook_a_sizes
584
+ );
585
+ }
586
+ flat_output *= scales.flatten().unsqueeze(0);
587
+ if (bias.has_value()) {
588
+ flat_output += bias->unsqueeze(0);
589
+ }
590
+
591
+ auto output_sizes = input_sizes.vec();
592
+ output_sizes.pop_back();
593
+ output_sizes.push_back(-1);
594
+ auto output = flat_output.reshape(output_sizes);
595
+ return output;
596
+}
597
+
598
+// Accumulate the partition sizes.
599
+int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes)
600
+{
601
+ int4 cumulative_sizes;
602
+ auto cumulative_size = &cumulative_sizes.x;
603
+ int i = 0;
604
+ int last = 0;
605
+ assert(codebook_partition_sizes.size(0) <= 4);
606
+ for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size)
607
+ {
608
+ *cumulative_size = codebook_partition_sizes[i].item<int>() + last;
609
+ last = *cumulative_size;
610
+ }
611
+ // fill in the rest with unreachable.
612
+ for (; i < 4; ++i, ++cumulative_size)
613
+ {
614
+ *cumulative_size = last*10;
615
+ }
616
+ return cumulative_sizes;
617
+}
618
+
619
+} // namespace aqlm
620
+} // namespace vllm
621
+
622
+
623
+torch::Tensor aqlm_gemm(
624
+ const torch::Tensor& input,
625
+ const torch::Tensor& codes,
626
+ const torch::Tensor& codebooks,
627
+ const torch::Tensor& scales,
628
+ const torch::Tensor& codebook_partition_sizes,
629
+ const std::optional<torch::Tensor>& bias
630
+)
631
+{
632
+ int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
633
+
634
+ int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
635
+ int const entries = codebooks.size(1);
636
+
637
+ if (nbooks == 1 && entries == (1 << 16))
638
+ {
639
+ return vllm::aqlm::code1x16_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
640
+ }
641
+ if (nbooks == 2 && entries == (1 << 8))
642
+ {
643
+ return vllm::aqlm::code2x8_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
644
+ }
645
+
646
+ TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
647
+ return {};
648
+}
649
+
650
+torch::Tensor aqlm_dequant(
651
+ const torch::Tensor& codes,
652
+ const torch::Tensor& codebooks,
653
+ const torch::Tensor& codebook_partition_sizes
654
+)
655
+{
656
+ int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
657
+
658
+ int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
659
+ int const entries = codebooks.size(1);
660
+
661
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(codes));
662
+ int rows = codes.size(1);
663
+ int cols = codes.size(0);
664
+
665
+ auto in_features = codes.size(1) * 8;
666
+ auto out_features = codes.size(0);
667
+
668
+ assert(out_features = codebook_partition_sizes.sum().item<int>());
669
+
670
+ auto weights = torch::empty({out_features, in_features},
671
+ torch::TensorOptions()
672
+ .dtype(codebooks.dtype())
673
+ .device(codebooks.device())
674
+ );
675
+
676
+ if (nbooks == 1 && entries == (1 << 16))
677
+ {
678
+ vllm::aqlm::code1x16_dequant_cuda(
679
+ codes.data_ptr(),
680
+ weights.data_ptr(),
681
+ codebooks.data_ptr(),
682
+ out_features,
683
+ in_features,
684
+ cumulative_sizes,
685
+ vllm::aqlm::codebook_stride(codebooks));
686
+
687
+ // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation.)
688
+ // weights *= scales.index({"...", 0, 0});
689
+
690
+ return weights;
691
+ }
692
+
693
+ if (nbooks == 2 && entries == (1 << 8))
694
+ {
695
+ vllm::aqlm::code2x8_dequant_cuda(
696
+ codes.data_ptr(),
697
+ weights.data_ptr(),
698
+ codebooks.data_ptr(),
699
+ out_features,
700
+ in_features,
701
+ cumulative_sizes,
702
+ vllm::aqlm::codebook_stride(codebooks));
703
+
704
+ // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation)
705
+ // weights *= scales.index({"...", 0, 0});
706
+
707
+ return weights;
708
+ }
709
+
710
+ TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
711
+ return {};
712
+}
examples/aqlm_example.py
new
+46
@@ -0,0 +1,46 @@
1
+import argparse
2
+
3
+from vllm import LLM, SamplingParams
4
+
5
+
6
+def main():
7
+
8
+ parser = argparse.ArgumentParser(description='AQLM examples')
9
+
10
+ parser.add_argument('--model',
11
+ '-m',
12
+ type=str,
13
+ default=None,
14
+ help='model path, as for HF')
15
+ parser.add_argument('--choice',
16
+ '-c',
17
+ type=int,
18
+ default=0,
19
+ help='known good models by index, [0-4]')
20
+ parser.add_argument('--tensor_parallel_size',
21
+ '-t',
22
+ type=int,
23
+ default=1,
24
+ help='tensor parallel size')
25
+
26
+ args = parser.parse_args()
27
+
28
+ models = [
29
+ "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf",
30
+ "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-2x8-hf",
31
+ "ISTA-DASLab/Llama-2-13b-AQLM-2Bit-1x16-hf",
32
+ "ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf",
33
+ "BlackSamorez/TinyLlama-1_1B-Chat-v1_0-AQLM-2Bit-1x16-hf",
34
+ ]
35
+
36
+ model = LLM(args.model if args.model is not None else models[args.choice],
37
+ tensor_parallel_size=args.tensor_parallel_size)
38
+
39
+ sampling_params = SamplingParams(max_tokens=100, temperature=0)
40
+ outputs = model.generate("Hello my name is",
41
+ sampling_params=sampling_params)
42
+ print(outputs[0].outputs[0].text)
43
+
44
+
45
+if __name__ == '__main__':
46
+ main()
tests/models/test_aqlm.py
new
+95
@@ -0,0 +1,95 @@
1
+"""Compare the outputs of a AQLM model between vLLM and HF Transformers
2
+
3
+Run `pytest tests/models/test_aqlm.py`.
4
+"""
5
+
6
+import pytest
7
+import torch
8
+
9
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10
+
11
+capability = torch.cuda.get_device_capability()
12
+capability = capability[0] * 10 + capability[1]
13
+aqlm_not_supported = (capability <
14
+ QUANTIZATION_METHODS["aqlm"].get_min_capability())
15
+
16
+# In this test we hardcode prompts and generations for the model so we don't
17
+# need to require the AQLM package as a dependency
18
+example_prompts = [
19
+ 'vLLM is a high-throughput and memory-efficient inference and serving '
20
+ 'engine for LLMs.\n',
21
+ 'Briefly describe the major milestones in the development of artificial '
22
+ 'intelligence from 1950 to 2020.\n',
23
+ 'Compare and contrast artificial intelligence with human intelligence in '
24
+ 'terms of processing information.\n',
25
+ 'Describe the basic components of a neural network and how it can be '
26
+ 'trained.\n',
27
+ 'Write a short story about a robot that dreams for the first time.\n',
28
+ 'Analyze the impact of the COVID-19 pandemic on global economic structures '
29
+ 'and future business models.\n',
30
+ 'Explain the cultural significance of the Mona Lisa painting, and how its '
31
+ 'perception might vary in Western versus Eastern societies.\n',
32
+ "Translate the following English sentence into Japanese, French, and "
33
+ "Swahili: 'The early bird catches the worm.'\n"
34
+]
35
+
36
+# These ground truth generations were generated using `transformers==4.38.1
37
+# aqlm==1.1.0 torch==2.2.0`
38
+# and the below code:
39
+# ```python
40
+# from transformers import AutoTokenizer, AutoModelForCausalLM
41
+# model_id = "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"
42
+# quantized_model = AutoModelForCausalLM.from_pretrained(model_id,
43
+# torch_dtype="auto", device_map="cuda").cuda()
44
+# tokenizer = AutoTokenizer.from_pretrained(model_id)
45
+# outputs = []
46
+# for prompt in example_prompts:
47
+# input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"].to("cuda")
48
+# hf_outputs = quantized_model.generate(input_ids, max_new_tokens=32)
49
+# outputs.append(tokenizer.decode(hf_outputs[0][input_ids.shape[1]:]))
50
+# print(outputs)
51
+# ```
52
+ground_truth_generations = [
53
+ '\n### Features\n\n- **High-throughput**: v',
54
+ 'The major milestones in the development of artificial intelligence from '
55
+ '195',
56
+ 'Compare and contrast artificial intelligence with human intelligence in '
57
+ 'terms of processing information. The',
58
+ 'Explain the difference between supervised and unsupervised learning.'
59
+ '\nExplain',
60
+ 'Write a short story about a robot that dreams for the first time. The',
61
+ 'Analyze the impact of the COVID-19 pandemic on global economic',
62
+ 'The Mona Lisa is a painting by Leonardo da Vinci, and it',
63
+ 'The early bird catches the worm.\nThe early bird catches the'
64
+]
65
+
66
+
67
+@pytest.mark.skipif(aqlm_not_supported,
68
+ reason="AQLM is not supported on this GPU type.")
69
+@pytest.mark.parametrize("model", ["ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"])
70
+@pytest.mark.parametrize("dtype", ["half"])
71
+@pytest.mark.parametrize("max_tokens", [16])
72
+@pytest.mark.parametrize("num_logprobs", [1])
73
+def test_models(
74
+ vllm_runner,
75
+ example_prompts,
76
+ model: str,
77
+ dtype: str,
78
+ max_tokens: int,
79
+ num_logprobs: int,
80
+) -> None:
81
+
82
+ vllm_model = vllm_runner(model, dtype=dtype)
83
+ vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
84
+ max_tokens,
85
+ num_logprobs)
86
+
87
+ # loop through the prompts to compare against the ground truth generations
88
+ for prompt_idx in range(len(example_prompts)):
89
+ vllm_output_ids, vllm_output_str, vllm_logprobs = vllm_outputs[
90
+ prompt_idx]
91
+
92
+ print("Prompt: ", repr(example_prompts[prompt_idx]))
93
+ print("Reference output:", repr(ground_truth_generations[prompt_idx]))
94
+ print("Output output: ", repr(vllm_output_str))
95
+ assert vllm_output_str == ground_truth_generations[prompt_idx]
vllm/model_executor/layers/linear.py
+34
-7
@@ -31,7 +31,7 @@ class LinearMethodBase(ABC):
31
@abstractmethod
32
def create_weights(self, layer: torch.nn.Module,
33
input_size_per_partition: int,
34
- output_size_per_partition: int, input_size: int,
34
+ output_partition_sizes: List[int], input_size: int,
35
output_size: int, params_dtype: torch.dtype,
36
**extra_weight_attrs):
37
"""Create weights for a linear layer.
@@ -70,9 +70,10 @@ class UnquantizedLinearMethod(LinearMethodBase):
70
71
def create_weights(self, layer: torch.nn.Module,
72
input_size_per_partition: int,
73
- output_size_per_partition: int, input_size: int,
73
+ output_partition_sizes: List[int], input_size: int,
74
output_size: int, params_dtype: torch.dtype,
75
**extra_weight_attrs):
76
+ output_size_per_partition = sum(output_partition_sizes)
77
weight = Parameter(torch.empty(output_size_per_partition,
78
input_size_per_partition,
79
dtype=params_dtype),
@@ -127,7 +128,7 @@ class ReplicatedLinear(torch.nn.Module):
128
linear_method = UnquantizedLinearMethod()
129
self.linear_method = linear_method
130
self.linear_method.create_weights(self, self.input_size,
130
- self.output_size, self.input_size,
131
+ [self.output_size], self.input_size,
132
self.output_size, self.params_dtype)
133
if bias:
134
self.bias = Parameter(
@@ -161,6 +162,8 @@ class ColumnParallelLinear(torch.nn.Module):
162
skip adding bias but instead return it.
163
params_dtype: Data type for the parameters.
164
linear_method: (Maybe quantized) linear method.
165
+ output_sizes: list of output sizes packed into one output, like for QKV
166
+ the list would be size 3.
167
"""
168
169
def __init__(
@@ -172,6 +175,7 @@ class ColumnParallelLinear(torch.nn.Module):
175
skip_bias_add: bool = False,
176
params_dtype: Optional[torch.dtype] = None,
177
linear_method: Optional[LinearMethodBase] = None,
178
+ output_sizes: Optional[List[int]] = None,
179
):
180
super().__init__()
181
@@ -188,10 +192,12 @@ class ColumnParallelLinear(torch.nn.Module):
192
self.params_dtype = params_dtype
193
if linear_method is None:
194
linear_method = UnquantizedLinearMethod()
195
+ if output_sizes is None:
196
+ output_sizes = [output_size]
197
self.linear_method = linear_method
198
self.linear_method.create_weights(self,
199
self.input_size,
194
- self.output_size_per_partition,
200
+ [x // tp_size for x in output_sizes],
201
self.input_size,
202
self.output_size,
203
self.params_dtype,
@@ -268,14 +274,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
274
tp_size = get_tensor_model_parallel_world_size()
275
assert all(output_size % tp_size == 0 for output_size in output_sizes)
276
super().__init__(input_size, sum(output_sizes), bias, gather_output,
271
- skip_bias_add, params_dtype, linear_method)
277
+ skip_bias_add, params_dtype, linear_method,
278
+ self.output_sizes)
279
280
def weight_loader(self,
281
param: Parameter,
282
loaded_weight: torch.Tensor,
283
loaded_shard_id: Optional[int] = None):
284
+
285
param_data = param.data
286
output_dim = getattr(param, "output_dim", None)
287
+ is_metadata = getattr(param, "is_metadata", False)
288
if loaded_shard_id is None:
289
# Loaded weight is already packed.
290
if output_dim is None:
@@ -328,6 +337,11 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
337
start_idx = tp_rank * shard_size
338
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
339
shard_size)
340
+ elif is_metadata:
341
+ # metadata indicates fixed size concatenated along dim 0
342
+ shard_size = loaded_weight.shape[0]
343
+ shard_offset = loaded_shard_id * shard_size
344
+ param_data = param_data.narrow(0, shard_offset, shard_size)
345
else:
346
ignore_warning = getattr(param, "ignore_warning", False)
347
if not ignore_warning:
@@ -393,8 +407,14 @@ class QKVParallelLinear(ColumnParallelLinear):
407
input_size = self.hidden_size
408
output_size = (self.num_heads +
409
2 * self.num_kv_heads) * tp_size * self.head_size
410
+ output_sizes = [
411
+ self.num_heads * tp_size * self.head_size,
412
+ self.num_kv_heads * tp_size * self.head_size,
413
+ self.num_kv_heads * tp_size * self.head_size
414
+ ]
415
+
416
super().__init__(input_size, output_size, bias, False, skip_bias_add,
397
- params_dtype, linear_method)
417
+ params_dtype, linear_method, output_sizes)
418
419
def weight_loader(self,
420
param: Parameter,
@@ -402,6 +422,7 @@ class QKVParallelLinear(ColumnParallelLinear):
422
loaded_shard_id: Optional[str] = None):
423
param_data = param.data
424
output_dim = getattr(param, "output_dim", None)
425
+ is_metadata = getattr(param, "is_metadata", False)
426
427
if loaded_shard_id is None:
428
# Loaded weight is already packed.
@@ -469,6 +490,12 @@ class QKVParallelLinear(ColumnParallelLinear):
490
start_idx = shard_id * shard_size
491
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
492
shard_size)
493
+ elif is_metadata:
494
+ # metadata indicates fixed size concatenated along dim 0
495
+ shard_size = loaded_weight.shape[0]
496
+ shard_index = ["q", "k", "v"].index(loaded_shard_id)
497
+ param_data = param_data.narrow(0, shard_index * shard_size,
498
+ shard_size)
499
else:
500
ignore_warning = getattr(param, "ignore_warning", False)
501
if not ignore_warning:
@@ -536,7 +563,7 @@ class RowParallelLinear(torch.nn.Module):
563
self.linear_method = linear_method
564
self.linear_method.create_weights(self,
565
self.input_size_per_partition,
539
- self.output_size,
566
+ [self.output_size],
567
self.input_size,
568
self.output_size,
569
self.params_dtype,
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -1,5 +1,6 @@
1
from typing import Type
2
3
+from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
@@ -9,6 +10,7 @@ from vllm.model_executor.layers.quantization.marlin import MarlinConfig
10
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
11
12
QUANTIZATION_METHODS = {
13
+ "aqlm": AQLMConfig,
14
"awq": AWQConfig,
15
"fp8": FP8Config,
16
"gptq": GPTQConfig,
vllm/model_executor/layers/quantization/aqlm.py
new
+373
@@ -0,0 +1,373 @@
1
+# Supports AQLM compression, see https://github.com/Vahe1994/AQLM
2
+# and https://arxiv.org/pdf/2401.06118.pdf
3
+
4
+import math
5
+from typing import Any, Dict, List, Optional
6
+
7
+import torch
8
+import torch.nn.functional as F
9
+from torch.nn.parameter import Parameter
10
+
11
+from vllm._C import ops
12
+from vllm.model_executor.layers.linear import (LinearMethodBase,
13
+ set_weight_attrs)
14
+from vllm.model_executor.layers.quantization.base_config import (
15
+ QuantizationConfig)
16
+
17
+
18
+def get_int_dtype(nbits: int) -> torch.dtype:
19
+ if nbits <= 8:
20
+ return torch.int8
21
+ if nbits <= 16:
22
+ return torch.int16
23
+ if nbits <= 32:
24
+ return torch.int32
25
+ if nbits <= 64:
26
+ return torch.int64
27
+ raise ValueError(f"No dtype available for {nbits}-bit codebooks")
28
+
29
+
30
+@torch.inference_mode()
31
+def unpack_int_data(data: torch.IntTensor, nbits: int) -> torch.IntTensor:
32
+ return data.to(torch.int64) % (2**nbits)
33
+
34
+
35
+def dequantize_weight(codes: torch.Tensor,
36
+ codebooks: torch.Tensor,
37
+ scales: Optional[torch.Tensor] = None) -> torch.Tensor:
38
+ """
39
+ Decode float weights from quantization codes. Differentiable.
40
+ :param codes: tensor of integer quantization codes, shape
41
+ [*dims, num_out_groups, num_in_groups, num_codebooks]
42
+ :param codebooks: tensor of vectors for each quantization code,
43
+ [num_codebooks, codebook_size, out_group_size, in_group_size]
44
+ :param scales: weight will be multiplied by this factor, must be
45
+ broadcastble with
46
+ [*dims, out_groups, num_in_groups, out_group_size, in_group_size]
47
+ :return: reconstructed weight tensor of shape
48
+ [*dims, num_in_groups*group_size]
49
+ """
50
+ num_out_groups, num_in_groups, num_codebooks = codes.shape[-3:]
51
+ num_codebooks, codebook_size, out_group_size, in_group_size = \
52
+ codebooks.shape
53
+ out_features = num_out_groups * out_group_size
54
+ in_features = num_in_groups * in_group_size
55
+ codebook_offsets = torch.arange(
56
+ 0, num_codebooks * codebook_size, codebook_size,
57
+ device=codes.device) # shape: [num_codebooks]
58
+ reconstructed_weight_flat = F.embedding_bag(
59
+ codes.flatten(0, -2) + codebook_offsets,
60
+ codebooks.flatten(0, 1).flatten(-2, -1),
61
+ mode="sum"
62
+ ) # [prod(dims) * num_out_groups * num_in_groups, out_group_size
63
+ # * in_group_size]
64
+
65
+ reconstructed_weight_groupwise = reconstructed_weight_flat.view(
66
+ list(codes.shape[:-3]) +
67
+ [num_out_groups, num_in_groups, out_group_size, in_group_size])
68
+ if scales is not None:
69
+ reconstructed_weight_groupwise = reconstructed_weight_groupwise.mul(
70
+ scales)
71
+ return reconstructed_weight_groupwise.swapaxes(
72
+ -3, -2).reshape(list(codes.shape[:-3]) + [out_features, in_features])
73
+
74
+
75
+def dequantize_gemm(
76
+ input: torch.Tensor, # [..., in_features]
77
+ codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
78
+ codebooks: torch.
79
+ Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
80
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
81
+ bias: Optional[torch.Tensor],
82
+) -> torch.Tensor:
83
+ dequantized_weight = dequantize_weight(
84
+ unpack_int_data(codes, codebooks.shape[1].bit_length() - 1),
85
+ codebooks,
86
+ scales,
87
+ )
88
+ return F.linear(input, dequantized_weight, bias)
89
+
90
+
91
+# Generic dequantization, slow but flexible.
92
+def generic_dequantize_gemm(
93
+ input: torch.Tensor, # [..., in_features]
94
+ codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
95
+ codebooks: torch.
96
+ Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
97
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
98
+ output_partition_sizes: torch.IntTensor,
99
+ bias: Optional[torch.Tensor],
100
+) -> torch.Tensor:
101
+ output_shape = input.shape[:-1] + (scales.shape[0], )
102
+ output = torch.empty(output_shape, dtype=input.dtype, device=input.device)
103
+ num_outputs = len(output_partition_sizes)
104
+
105
+ # break the inputs and codebooks apart then combine the outputs.
106
+ # Surprisingly (to me) this is faster than doing 3 de-quants and 1 big
107
+ # multiply at the end.
108
+ num_codebooks = codebooks.shape[0] // num_outputs
109
+ assert (scales.shape[0] == codes.shape[0])
110
+ assert (sum(output_partition_sizes) == scales.shape[0])
111
+ output_offset = 0
112
+ codebooks_offset = 0
113
+ for output_size in output_partition_sizes:
114
+ shard_output = dequantize_gemm(
115
+ input, codes.narrow(0, output_offset, output_size),
116
+ codebooks.narrow(0, codebooks_offset, num_codebooks),
117
+ scales.narrow(0, output_offset, output_size), None
118
+ if bias is None else bias.narrow(0, output_offset, output_size))
119
+
120
+ output_slice = output.narrow(-1, output_offset, output_size)
121
+ assert (output_slice.shape == shard_output.shape)
122
+ output_slice.copy_(shard_output)
123
+ output_offset += output_size
124
+ codebooks_offset += num_codebooks
125
+ return output
126
+
127
+
128
+# Optimized dequnantize/decompression kernels, supports 1x16 and 2x8
129
+# at 6 and 9 times faster than the generic version above, respectively.
130
+def optimized_dequantize_gemm(
131
+ input: torch.Tensor, # [..., in_features]
132
+ codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
133
+ codebooks: torch.
134
+ Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
135
+ scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
136
+ output_partition_sizes: torch.IntTensor,
137
+ bias: Optional[torch.Tensor],
138
+) -> torch.Tensor:
139
+ weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
140
+
141
+ if bias is None:
142
+ # scaling the output is fastest, so we do that when possible.
143
+ output = F.linear(input, weights, bias)
144
+ orig_shape = output.shape
145
+ flattened_output = output.view(-1, output.size(-1))
146
+ f_scales = scales.view(-1, scales.shape[0])
147
+ b_scales = f_scales.expand(flattened_output.shape[0], -1)
148
+ flattened_output *= b_scales
149
+ return output.view(orig_shape)
150
+ else:
151
+ b_scales = scales.view(scales.shape[:-3] + (-1, )).expand(
152
+ -1, weights.shape[1])
153
+ weights *= b_scales
154
+ return F.linear(input, weights, bias)
155
+
156
+
157
+class AQLMConfig(QuantizationConfig):
158
+ """Config class for AQLM.
159
+
160
+ Reference: https://github.com/Vahe1994/AQLM
161
+ """
162
+
163
+ def __init__(
164
+ self,
165
+ in_group_size: int,
166
+ nbits_per_codebook: int,
167
+ num_codebooks: int,
168
+ out_group_size: int,
169
+ ) -> None:
170
+ self.in_group_size = in_group_size
171
+ self.nbits_per_codebook = nbits_per_codebook
172
+ self.num_codebooks = num_codebooks
173
+ self.out_group_size = out_group_size
174
+
175
+ # out_group_size > 1 is untested, and probably won't work as-is.
176
+ assert (self.out_group_size == 1)
177
+ self.pack_factor = (self.in_group_size * self.out_group_size)
178
+
179
+ def __repr__(self) -> str:
180
+ return (f"AQLMConfig(in_group_size={self.in_group_size}, "
181
+ f"nbits_per_codebook={self.nbits_per_codebook}, "
182
+ f"num_codebooks={self.num_codebooks}, "
183
+ f"out_group_size={self.out_group_size})")
184
+
185
+ @classmethod
186
+ def get_name(cls) -> str:
187
+ return "aqlm"
188
+
189
+ @classmethod
190
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
191
+ return [torch.half]
192
+
193
+ @classmethod
194
+ def get_min_capability(cls) -> int:
195
+ return 70
196
+
197
+ @classmethod
198
+ def get_config_filenames(cls) -> List[str]:
199
+ return [] # no extra configs.
200
+
201
+ @classmethod
202
+ def from_config(cls, config: Dict[str, Any]) -> "AQLMConfig":
203
+ in_group_size = cls.get_from_keys(config, ["in_group_size"])
204
+ nbits_per_codebook = cls.get_from_keys(config, ["nbits_per_codebook"])
205
+ num_code_books = cls.get_from_keys(config, ["num_codebooks"])
206
+ out_group_size = cls.get_from_keys(config, ["out_group_size"])
207
+ return cls(in_group_size, nbits_per_codebook, num_code_books,
208
+ out_group_size)
209
+
210
+ def get_linear_method(self) -> "AQLMLinearMethod":
211
+ return AQLMLinearMethod(self)
212
+
213
+ def get_scaled_act_names(self) -> List[str]:
214
+ return []
215
+
216
+
217
+class AQLMLinearMethod(LinearMethodBase):
218
+ """Linear method for AQLM.
219
+
220
+ Args:
221
+ quant_config: The AQLM quantization config.
222
+ """
223
+
224
+ def __init__(self, quant_config: AQLMConfig):
225
+ self.quant_config = quant_config
226
+
227
+ def create_weights(self, layer: torch.nn.Module,
228
+ input_size_per_partition: int,
229
+ output_partition_sizes: List[int], input_size: int,
230
+ output_size: int, params_dtype: torch.dtype,
231
+ **extra_weight_attrs):
232
+ del output_size # Unused.
233
+ del input_size # Unused.
234
+
235
+ if params_dtype != torch.half:
236
+ raise ValueError("Only half is currently supported by aqlm")
237
+ if input_size_per_partition % self.quant_config.in_group_size != 0:
238
+ raise ValueError(
239
+ "The input size is not aligned with the quantized "
240
+ "weight shape. This can be caused by too large "
241
+ "tensor parallel size.")
242
+
243
+ output_size_per_partition = sum(output_partition_sizes)
244
+ if output_size_per_partition % self.quant_config.out_group_size != 0:
245
+ raise ValueError(
246
+ "The output size is not aligned with the quantized "
247
+ "weight shape. This can be caused by too large "
248
+ "tensor parallel size.")
249
+
250
+ codes = Parameter(
251
+ torch.empty(
252
+ # There could actually be two pack factors, one along input and
253
+ # one along output, but we don't currently support
254
+ # out_group_size, and only the one along output needs to be
255
+ # marked with "packed_dim" in order for QKVLinear to work.
256
+ output_size_per_partition,
257
+ input_size_per_partition // self.quant_config.pack_factor,
258
+ self.quant_config.num_codebooks,
259
+ dtype=get_int_dtype(self.quant_config.nbits_per_codebook),
260
+ ),
261
+ requires_grad=False,
262
+ )
263
+
264
+ set_weight_attrs(
265
+ codes,
266
+ {
267
+ "input_dim": 1,
268
+ "output_dim": 0,
269
+ "packed_dim": 1,
270
+ "pack_factor": self.quant_config.pack_factor,
271
+ },
272
+ )
273
+
274
+ codebooks = Parameter(
275
+ torch.empty(
276
+ self.quant_config.num_codebooks * len(output_partition_sizes),
277
+ 2**self.quant_config.nbits_per_codebook,
278
+ self.quant_config.out_group_size,
279
+ self.quant_config.in_group_size,
280
+ dtype=params_dtype,
281
+ ),
282
+ requires_grad=False,
283
+ )
284
+ set_weight_attrs(
285
+ codebooks,
286
+ {
287
+ # metadata indicates fixed size concatenated along dim 0
288
+ "is_metadata":
289
+ True,
290
+ "output_partition_sizes":
291
+ torch.tensor(output_partition_sizes, device='cpu'),
292
+ },
293
+ )
294
+
295
+ scales = Parameter(
296
+ torch.empty(
297
+ (
298
+ output_size_per_partition //
299
+ self.quant_config.out_group_size,
300
+ 1,
301
+ 1,
302
+ 1,
303
+ ),
304
+ dtype=params_dtype,
305
+ ),
306
+ requires_grad=False,
307
+ )
308
+ set_weight_attrs(
309
+ scales,
310
+ {
311
+ "output_dim": 0,
312
+ "packed_dim": 0,
313
+ "pack_factor": self.quant_config.out_group_size
314
+ },
315
+ )
316
+
317
+ layer.register_parameter("codes", codes)
318
+ set_weight_attrs(codes, extra_weight_attrs)
319
+ layer.register_parameter("codebooks", codebooks)
320
+ set_weight_attrs(codebooks, extra_weight_attrs)
321
+ layer.register_parameter("scales", scales)
322
+ set_weight_attrs(scales, extra_weight_attrs)
323
+
324
+ def apply_weights(
325
+ self,
326
+ layer: torch.nn.Module,
327
+ x: torch.Tensor,
328
+ bias: Optional[torch.Tensor] = None,
329
+ ) -> torch.Tensor:
330
+ codebooks = layer.codebooks
331
+ codes = layer.codes
332
+ scales = layer.scales
333
+ output_partition_sizes = getattr(codebooks, "output_partition_sizes",
334
+ None)
335
+
336
+ nbooks = codes.shape[2]
337
+ ingroups = codebooks.shape[3]
338
+ outgroups = codebooks.shape[2]
339
+ bits = codebooks.shape[1]
340
+
341
+ # We support these formats with dedicated gemm and decompression
342
+ # kernels.
343
+ if ingroups == 8 and outgroups == 1 and (
344
+ (bits == 256 and nbooks == 2) or (bits == 65536 and nbooks == 1)):
345
+
346
+ # thresholds determined by timings on an A6000, one GPU
347
+ use_gemv = math.prod(x.shape[:-1]) <= 6
348
+
349
+ return ops.aqlm_gemm(
350
+ x,
351
+ codes,
352
+ codebooks,
353
+ scales,
354
+ output_partition_sizes,
355
+ bias,
356
+ ) if use_gemv else optimized_dequantize_gemm(
357
+ x,
358
+ codes,
359
+ codebooks,
360
+ scales,
361
+ output_partition_sizes,
362
+ bias,
363
+ )
364
+
365
+ # fall back all unoptimized formats
366
+ return generic_dequantize_gemm(
367
+ x,
368
+ codes,
369
+ codebooks,
370
+ scales,
371
+ output_partition_sizes,
372
+ bias,
373
+ )
vllm/model_executor/layers/quantization/awq.py
+3
-1
@@ -81,7 +81,7 @@ class AWQLinearMethod(LinearMethodBase):
81
82
def create_weights(self, layer: torch.nn.Module,
83
input_size_per_partition: int,
84
- output_size_per_partition: int, input_size: int,
84
+ output_partition_sizes: List[int], input_size: int,
85
output_size: int, params_dtype: torch.dtype,
86
**extra_weight_attrs):
87
if input_size_per_partition % self.quant_config.group_size != 0:
@@ -89,6 +89,8 @@ class AWQLinearMethod(LinearMethodBase):
89
"The input size is not aligned with the quantized "
90
"weight shape. This can be caused by too large "
91
"tensor parallel size.")
92
+
93
+ output_size_per_partition = sum(output_partition_sizes)
94
if output_size_per_partition % self.quant_config.pack_factor != 0:
95
raise ValueError(
96
"The output size is not aligned with the quantized "
vllm/model_executor/layers/quantization/gptq.py
+2
-1
@@ -91,7 +91,7 @@ class GPTQLinearMethod(LinearMethodBase):
91
self,
92
layer: torch.nn.Module,
93
input_size_per_partition: int,
94
- output_size_per_partition: int,
94
+ output_partition_sizes: List[int],
95
input_size: int,
96
output_size: int,
97
params_dtype: torch.dtype,
@@ -103,6 +103,7 @@ class GPTQLinearMethod(LinearMethodBase):
103
"The input size is not aligned with the quantized "
104
"weight shape. This can be caused by too large "
105
"tensor parallel size.")
106
+ output_size_per_partition = sum(output_partition_sizes)
107
if (output_size_per_partition % self.quant_config.pack_factor.numerator
108
!= 0):
109
raise ValueError(
vllm/model_executor/layers/quantization/marlin.py
+2
-1
@@ -93,7 +93,7 @@ class MarlinLinearMethod(LinearMethodBase):
93
self,
94
layer: torch.nn.Module,
95
input_size_per_partition: int,
96
- output_size_per_partition: int,
96
+ output_partition_sizes: List[int],
97
input_size: int,
98
output_size: int,
99
params_dtype: torch.dtype,
@@ -106,6 +106,7 @@ class MarlinLinearMethod(LinearMethodBase):
106
f"The params dtype must be float16, but got {params_dtype}")
107
108
# Validate output_size_per_partition
109
+ output_size_per_partition = sum(output_partition_sizes)
110
if output_size_per_partition % self.quant_config.min_n_threads != 0:
111
raise ValueError(
112
f"Weight output_size_per_partition = "
vllm/model_executor/layers/quantization/squeezellm.py
+3
-1
@@ -70,7 +70,7 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
70
71
def create_weights(self, layer: torch.nn.Module,
72
input_size_per_partition: int,
73
- output_size_per_partition: int, input_size: int,
73
+ output_partition_sizes: List[int], input_size: int,
74
output_size: int, params_dtype: torch.dtype,
75
**extra_weight_attrs):
76
if input_size_per_partition % self.quant_config.pack_factor != 0:
@@ -78,6 +78,8 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
78
"The input size is not aligned with the quantized "
79
"weight shape. This can be caused by too large "
80
"tensor parallel size.")
81
+
82
+ output_size_per_partition = sum(output_partition_sizes)
83
qweight = Parameter(
84
torch.empty(
85
input_size_per_partition // self.quant_config.pack_factor,