@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b7949c1c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

AQLM CUDA support (#3287)

Co-authored-by: mgoin <michael@neuralmagic.com>

James Fleming committed Apr 23, 2024 at 13:59 UTC 2b7949c1c2e34de41d9cfc84dd0e377cc6bd58c2
14 files changed +1592 -11
CMakeLists.txt
+1
@@ -173,6 +173,7 @@ set(VLLM_EXT_SRC
173
174 if(VLLM_GPU_LANG STREQUAL "CUDA")
175 list(APPEND VLLM_EXT_SRC
176 + "csrc/quantization/aqlm/gemm_kernels.cu"
177 "csrc/quantization/awq/gemm_kernels.cu"
178 "csrc/quantization/marlin/marlin_cuda_kernel.cu"
179 "csrc/custom_all_reduce.cu")
benchmarks/kernels/benchmark_aqlm.py new
+302
@@ -0,0 +1,302 @@
1 +import argparse
2 +import os
3 +import sys
4 +from typing import Optional
5 +
6 +import torch
7 +import torch.nn.functional as F
8 +
9 +from vllm._C import ops
10 +from vllm.model_executor.layers.quantization.aqlm import (
11 + dequantize_weight, generic_dequantize_gemm, get_int_dtype,
12 + optimized_dequantize_gemm)
13 +
14 +os.environ['CUDA_VISIBLE_DEVICES'] = '0'
15 +
16 +
17 +def torch_mult(
18 + input: torch.Tensor, # [..., in_features]
19 + weights: torch.Tensor,
20 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
21 +) -> torch.Tensor:
22 + output = F.linear(input, weights)
23 + return output
24 +
25 +
26 +def dequant_out_scale(
27 + input: torch.Tensor, # [..., in_features]
28 + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
29 + codebooks: torch.
30 + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
31 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
32 + output_partition_sizes: torch.IntTensor,
33 + bias: Optional[torch.Tensor],
34 +) -> torch.Tensor:
35 +
36 + weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
37 +
38 + if bias is None:
39 + output = F.linear(input, weights, bias)
40 + orig_shape = output.shape
41 + flattened_output = output.view(-1, output.size(-1))
42 + f_scales = scales.view(-1, scales.shape[0])
43 + b_scales = f_scales.expand(flattened_output.shape[0], -1)
44 + flattened_output *= b_scales
45 + return flattened_output.view(orig_shape)
46 + else:
47 + b_scales = scales.view(scales.shape[:-3] + (-1, )).expand(
48 + -1, weights.shape[1])
49 + weights *= b_scales
50 + return F.linear(input, weights, bias)
51 +
52 +
53 +def dequant_weight_scale(
54 + input: torch.Tensor, # [..., in_features]
55 + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
56 + codebooks: torch.
57 + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
58 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
59 + output_partition_sizes: torch.IntTensor,
60 + bias: Optional[torch.Tensor],
61 +) -> torch.Tensor:
62 +
63 + weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
64 +
65 + b_scales = scales.view(scales.shape[:-3] + (-1, )).expand(
66 + -1, weights.shape[1])
67 + weights *= b_scales
68 + return F.linear(input, weights, bias)
69 +
70 +
71 +def dequant_no_scale(
72 + input: torch.Tensor, # [..., in_features]
73 + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
74 + codebooks: torch.
75 + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
76 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
77 + output_partition_sizes: torch.IntTensor,
78 + bias: Optional[torch.Tensor],
79 +) -> torch.Tensor:
80 +
81 + weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
82 +
83 + return F.linear(input, weights, bias)
84 +
85 +
86 +# Compare the optimized 1x16 and 2x8 cuda decompression/dequant kernels against
87 +# the generic pytorch version.
88 +# Just visual comparison.
89 +def dequant_test(k: int, parts: torch.tensor, nbooks: int, bits: int) -> None:
90 +
91 + n = parts.sum().item()
92 +
93 + device = torch.device('cuda:0')
94 +
95 + code_range = (1 << bits) // 2
96 + ingroups = 8
97 +
98 + codes = torch.randint(-code_range,
99 + code_range,
100 + size=(n, k // ingroups, nbooks),
101 + dtype=get_int_dtype(bits),
102 + device=device)
103 +
104 + codebooks = torch.randn(size=(parts.shape[0] * nbooks, 1 << bits, 1, 8),
105 + dtype=torch.float16,
106 + device=device)
107 +
108 + count = 0
109 + for index in range(16):
110 + for i in range(8):
111 + for book in range(nbooks):
112 + codebooks[book, index, 0, i] = count * (10**book)
113 + count += 1
114 +
115 + print("codes shape", codes.shape)
116 +
117 + for i in range(16):
118 + for book in range(nbooks):
119 + codes[0, i, book] = i
120 + codes[0, -i, book] = i
121 +
122 + weights = dequantize_weight(codes, codebooks, None)
123 + weights2 = ops.aqlm_dequant(codes, codebooks, parts)
124 +
125 + print("weights shape:", weights.shape)
126 + print("weights2 shape:", weights2.shape)
127 +
128 + print("weights are:", weights)
129 + print("weights2 are:", weights2)
130 +
131 + print("first 128 weights are", weights[0, 0:128].to(torch.int32))
132 + print("first 128 weights2 are:", weights2[0, 0:128].to(torch.int32))
133 +
134 + print("last 128 weights are", weights[0, -128:])
135 + print("last 128 weights2 are:", weights2[0, -128:])
136 +
137 +
138 +def main():
139 +
140 + parser = argparse.ArgumentParser(description="Benchmark aqlm performance.")
141 +
142 + # Add arguments
143 + parser.add_argument("--nbooks",
144 + type=int,
145 + default=1,
146 + help="Number of codebooks (default: 1)")
147 + parser.add_argument("--bits",
148 + type=int,
149 + default=16,
150 + help="Number of bits per code element (default: 16)")
151 + parser.add_argument(
152 + "--test",
153 + type=bool,
154 + default=False,
155 + help="Run the decompression/dequant tester rather than benchmarking "
156 + "(default: False)")
157 +
158 + # Parse the arguments
159 + args = parser.parse_args()
160 +
161 + # Extract values
162 + nbooks = args.nbooks
163 + bits = args.bits
164 +
165 + if args.test:
166 + dequant_test(4096, torch.tensor((4096, )), nbooks, bits)
167 + return
168 +
169 + # Otherwise, benchmark.
170 + methods = [
171 + ops.aqlm_gemm,
172 + dequant_out_scale,
173 + generic_dequantize_gemm,
174 + optimized_dequantize_gemm,
175 + dequant_weight_scale,
176 + torch_mult,
177 + dequant_no_scale,
178 + ]
179 +
180 + filename = f"./aqlm_benchmark_{nbooks}x{bits}.csv"
181 + print(f"writing benchmarks to file {filename}")
182 + with open(filename, "w") as f:
183 + sys.stdout = f
184 +
185 + print('m | k | n | n parts', end='')
186 + for method in methods:
187 + print(f" | {method.__name__.replace('_', ' ')} (µs)", end='')
188 + print('')
189 +
190 + # These are reasonable prefill sizes.
191 + ksandpartions = ((4096, (4096, 4096, 4096)), (4096, (4096, )),
192 + (4096, (11008, 11008)), (11008, (4096, )))
193 +
194 + # reasonable ranges for m.
195 + for m in [
196 + 1, 2, 4, 8, 10, 12, 14, 16, 24, 32, 48, 52, 56, 64, 96, 112,
197 + 128, 256, 512, 1024, 1536, 2048, 3072, 4096
198 + ]:
199 + print(f'{m}', file=sys.__stdout__)
200 + for ksp in ksandpartions:
201 + run_grid(m, ksp[0], torch.tensor(ksp[1]), nbooks, bits,
202 + methods)
203 +
204 + sys.stdout = sys.__stdout__
205 +
206 +
207 +def run_grid(m: int, k: int, parts: torch.tensor, nbooks: int, bits: int,
208 + methods):
209 +
210 + # I didn't see visible improvements from increasing these, but feel free :)
211 + num_warmup_trials = 1
212 + num_trials = 1
213 +
214 + num_calls = 100
215 +
216 + # warmup.
217 + for method in methods:
218 + for _ in range(num_warmup_trials):
219 + run_timing(
220 + num_calls=num_calls,
221 + m=m,
222 + k=k,
223 + parts=parts,
224 + nbooks=nbooks,
225 + bits=bits,
226 + method=method,
227 + )
228 +
229 + n = parts.sum().item()
230 + print(f'{m} | {k} | {n} | {parts.tolist()}', end='')
231 +
232 + for method in methods:
233 + best_time_us = 1e20
234 + for _ in range(num_trials):
235 + kernel_dur_ms = run_timing(
236 + num_calls=num_calls,
237 + m=m,
238 + k=k,
239 + parts=parts,
240 + nbooks=nbooks,
241 + bits=bits,
242 + method=method,
243 + )
244 +
245 + kernel_dur_us = 1000 * kernel_dur_ms
246 +
247 + if kernel_dur_us < best_time_us:
248 + best_time_us = kernel_dur_us
249 +
250 + print(f' | {kernel_dur_us:.0f}', end='')
251 +
252 + print('')
253 +
254 +
255 +def run_timing(num_calls: int, m: int, k: int, parts: torch.tensor,
256 + nbooks: int, bits: int, method) -> float:
257 +
258 + n = parts.sum().item()
259 +
260 + device = torch.device('cuda:0')
261 +
262 + input = torch.randn((1, m, k), dtype=torch.float16, device=device)
263 +
264 + code_range = (1 << bits) // 2
265 + ingroups = 8
266 +
267 + codes = torch.randint(-code_range,
268 + code_range,
269 + size=(n, k // ingroups, nbooks),
270 + dtype=get_int_dtype(bits),
271 + device=device)
272 +
273 + codebooks = torch.randn(size=(parts.shape[0] * nbooks, 1 << bits, 1, 8),
274 + dtype=torch.float16,
275 + device=device)
276 +
277 + scales = torch.randn(size=(n, 1, 1, 1), dtype=torch.float16, device=device)
278 +
279 + # for comparison to just a pytorch mult.
280 + weights = torch.randn((n, k), dtype=torch.float16, device=device)
281 +
282 + start_event = torch.cuda.Event(enable_timing=True)
283 + end_event = torch.cuda.Event(enable_timing=True)
284 +
285 + start_event.record()
286 +
287 + if method is torch_mult:
288 + for i in range(num_calls):
289 + torch_mult(input, weights, scales)
290 + else:
291 + for i in range(num_calls):
292 + method(input, codes, codebooks, scales, parts, None)
293 +
294 + end_event.record()
295 + end_event.synchronize()
296 +
297 + dur_ms = start_event.elapsed_time(end_event) / num_calls
298 + return dur_ms
299 +
300 +
301 +if __name__ == "__main__":
302 + sys.exit(main())
csrc/ops.h
+15
@@ -86,6 +86,21 @@ void gelu_fast(
86 torch::Tensor& input);
87
88 #ifndef USE_ROCM
89 +torch::Tensor aqlm_gemm(
90 + const torch::Tensor& input,
91 + const torch::Tensor& codes,
92 + const torch::Tensor& codebooks,
93 + const torch::Tensor& scales,
94 + const torch::Tensor& codebook_partition_sizes,
95 + const std::optional<torch::Tensor>& bias
96 +);
97 +
98 +torch::Tensor aqlm_dequant(
99 + const torch::Tensor& codes,
100 + const torch::Tensor& codebooks,
101 + const torch::Tensor& codebook_partition_sizes
102 +);
103 +
104 torch::Tensor awq_gemm(
105 torch::Tensor _in_feats,
106 torch::Tensor _kernel,
csrc/pybind.cpp
+2
@@ -63,6 +63,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
63
64 // Quantization ops
65 #ifndef USE_ROCM
66 + ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
67 + ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
68 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69 ops.def("marlin_gemm", &marlin_gemm, "Marlin Optimized Quantized GEMM for GPTQ");
70 ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
csrc/quantization/aqlm/gemm_kernels.cu new
+712
@@ -0,0 +1,712 @@
1 +/*
2 + * Modified by Neural Magic
3 + * Adapted from https://github.com/Vahe1994/AQLM
4 + *
5 + * Licensed under the Apache License, Version 2.0 (the "License");
6 + * you may not use this file except in compliance with the License.
7 + * You may obtain a copy of the License at
8 + *
9 + * http://www.apache.org/licenses/LICENSE-2.0
10 + *
11 + * Unless required by applicable law or agreed to in writing, software
12 + * distributed under the License is distributed on an "AS IS" BASIS,
13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 + * See the License for the specific language governing permissions and
15 + * limitations under the License.
16 + */
17 +
18 +#include <cuda.h>
19 +#include <cuda_fp16.h>
20 +#include <cuda_runtime.h>
21 +#include <torch/extension.h>
22 +#include <c10/cuda/CUDAStream.h>
23 +#include <c10/cuda/CUDAGuard.h>
24 +
25 +#include <iostream>
26 +#include <cstdlib>
27 +
28 +
29 +namespace vllm {
30 +namespace aqlm {
31 +
32 +__global__ void Code1x16MatVec(
33 + const int4* __restrict__ A,
34 + const int4* __restrict__ B,
35 + int4* __restrict__ C,
36 + const int4* __restrict__ codebook,
37 + const int prob_m,
38 + const int prob_k,
39 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
40 + const int codebook_stride // as int4.
41 +) {
42 + int a_gl_stride = prob_k / 8 / 8;
43 + int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
44 + bool pred = a_gl_rd < prob_m;
45 +
46 + if (pred)
47 + {
48 + // advance to the correct codebook, this easy because we only multiply one column of the codebook.
49 + auto codebook_size = &codebook_a_sizes.x;
50 + while (a_gl_rd >= *codebook_size)
51 + {
52 + codebook += codebook_stride;
53 + ++codebook_size;
54 + }
55 + }
56 +
57 + int b_gl_rd = 0;
58 + int c_gl_wr = a_gl_rd;
59 + a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
60 + int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
61 +
62 + __shared__ int4 sh_b[32 * 9];
63 + float res = 0;
64 +
65 + int iters = (prob_k / 8 + 8 * 32 - 1) / (8 * 32);
66 + while (iters--) {
67 + // We pad shared memory to avoid bank conflicts during reads
68 + __syncthreads();
69 + for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
70 + if (b_gl_rd + i < prob_k / 8)
71 + sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
72 + }
73 + __syncthreads();
74 + b_gl_rd += 32 * 8;
75 +
76 + int b_sh_rd = 9 * (threadIdx.x % 32);
77 + if (pred && a_gl_rd < a_gl_end) {
78 + const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
79 + #pragma unroll
80 + for (int i = 0; i < 8; i++) {
81 + uint32_t dec[4];
82 + // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
83 + // actually help us; this brings > 2x speedup.
84 + asm volatile (
85 + "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
86 + : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
87 + : "l"((void*) &codebook[enc[i]])
88 + );
89 + half2* a = reinterpret_cast<half2*>(&dec);
90 + half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
91 + half2 res2 = {};
92 + #pragma unroll
93 + for (int j = 0; j < 4; j++)
94 + res2 = __hfma2(a[j], b[j], res2);
95 + res += __half2float(res2.x) + __half2float(res2.y);
96 + b_sh_rd++;
97 + }
98 + a_gl_rd += 32;
99 + }
100 + }
101 +
102 + if (pred) {
103 + #pragma unroll
104 + for (int i = 16; i > 0; i /= 2)
105 + res += __shfl_down_sync(0xffffffff, res, i);
106 + if (threadIdx.x % 32 == 0)
107 + reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
108 + }
109 +}
110 +
111 +__global__ void Code2x8MatVec(
112 + const int4* __restrict__ A,
113 + const int4* __restrict__ B,
114 + int4* __restrict__ C,
115 + const int4* __restrict__ codebook,
116 + int prob_m,
117 + int prob_k,
118 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
119 + const int codebook_stride // as int4.
120 +
121 +) {
122 + int a_gl_stride = prob_k / 8 / 8;
123 + int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
124 + bool pred = a_gl_rd < prob_m;
125 +
126 + if (pred)
127 + {
128 + // advance to the correct codebook, this easy because we only multiply one column of the codebook.
129 + auto codebook_size = &codebook_a_sizes.x;
130 + while (a_gl_rd >= *codebook_size)
131 + {
132 + codebook += codebook_stride;
133 + ++codebook_size;
134 + }
135 + }
136 +
137 + int b_gl_rd = 0;
138 + int c_gl_wr = a_gl_rd;
139 + a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
140 + int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
141 + int lane = threadIdx.x % 8;
142 +
143 + extern __shared__ int4 sh[];
144 + int4* sh_b = sh;
145 + int4* sh_code = sh_b + 32 * 9;
146 + int4* sh_code0 = sh_code;
147 + int4* sh_code1 = sh_code + 256 * 8;
148 +
149 + for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
150 + int4 dec = codebook[i];
151 + #pragma unroll
152 + for (int j = 0; j < 8; j++)
153 + sh_code[8 * i + (j + lane) % 8] = dec;
154 + }
155 + __syncthreads();
156 +
157 + float res = 0;
158 +
159 + int iters = (prob_k / 8 + 8 * 32 - 1) / (8 * 32);
160 + while (iters--) {
161 + // We pad shared memory to avoid bank conflicts during reads
162 + __syncthreads();
163 + for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
164 + if (b_gl_rd + i < prob_k / 8)
165 + sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
166 + }
167 + __syncthreads();
168 + b_gl_rd += 32 * 8;
169 +
170 + int b_sh_rd = 9 * (threadIdx.x % 32);
171 + if (pred && a_gl_rd < a_gl_end) {
172 + const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
173 + #pragma unroll
174 + for (int i = 0; i < 8; i++) {
175 + half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
176 + half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
177 + half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
178 + half2 res2 = {};
179 + #pragma unroll
180 + for (int j = 0; j < 4; j++)
181 + res2 = __hfma2(__hadd2(a0[j], a1[j]), b[j], res2);
182 + res += __half2float(res2.x) + __half2float(res2.y);
183 + b_sh_rd++;
184 + }
185 + a_gl_rd += 32;
186 + }
187 + }
188 +
189 + if (pred) {
190 + #pragma unroll
191 + for (int i = 16; i > 0; i /= 2)
192 + res += __shfl_down_sync(0xffffffff, res, i);
193 + if (threadIdx.x % 32 == 0)
194 + reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
195 + }
196 +}
197 +
198 +
199 +__global__ void Code1x16Dequant(
200 + const int4* __restrict__ A,
201 + int4* __restrict__ C,
202 + const int4* __restrict__ codebook,
203 + int prob_m,
204 + int prob_k,
205 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, sums to m.
206 + const int codebook_stride // as int4
207 +) {
208 + int a_gl_stride = prob_k / 8 / 8;
209 + int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
210 + bool pred = a_gl_rd < prob_m;
211 +
212 + if (pred)
213 + {
214 + // advance to the correct codebook, this easy because we only multiply one column of the codebook.
215 + auto codebook_size = &codebook_a_sizes.x;
216 + while (a_gl_rd >= *codebook_size)
217 + {
218 + codebook += codebook_stride;
219 + ++codebook_size;
220 + }
221 + }
222 +
223 + a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
224 + int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
225 +
226 + int c_gl_stride = prob_k / 8;
227 + int c_gl_wr = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
228 + c_gl_wr = c_gl_stride * c_gl_wr + (threadIdx.x % 32) * 8;
229 +
230 + int iters = (prob_k / 8 - 1) / (8 * 32) + 1;
231 + while (iters--) {
232 + if (pred && a_gl_rd < a_gl_end) {
233 + const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
234 + #pragma unroll
235 + for (int i = 0; i < 8; i++) {
236 + int4 chunk;
237 + auto dec = reinterpret_cast<uint32_t*>(&chunk);
238 + // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
239 + // actually help us; this brings > 2x speedup.
240 + asm volatile (
241 + "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
242 + : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
243 + : "l"((void*) &codebook[enc[i]])
244 + );
245 +
246 + C[a_gl_rd * 8 + i] = chunk;
247 + }
248 + }
249 + a_gl_rd += 32;
250 + }
251 +}
252 +
253 +
254 +__global__ void Code2x8Dequant(
255 + const int4* __restrict__ A,
256 + int4* __restrict__ C,
257 + const int4* __restrict__ codebook,
258 + int prob_m,
259 + int prob_k,
260 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
261 + const int codebook_stride // as int4
262 +) {
263 + int a_gl_stride = prob_k / 8 / 8;
264 + int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
265 + bool pred = a_gl_rd < prob_m;
266 +
267 + if (pred)
268 + {
269 + // advance to the correct codebook, this easy because we only multiply one column of the codebook.
270 + auto codebook_size = &codebook_a_sizes.x;
271 + while (a_gl_rd >= *codebook_size)
272 + {
273 + codebook += codebook_stride;
274 + ++codebook_size;
275 + }
276 + }
277 +
278 + a_gl_rd = a_gl_stride * a_gl_rd + threadIdx.x % 32;
279 + int a_gl_end = a_gl_rd + a_gl_stride - threadIdx.x % 32;
280 + int lane = threadIdx.x % 8;
281 +
282 + int c_gl_stride = prob_k / 8;
283 + int c_gl_wr = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
284 + c_gl_wr = c_gl_stride * c_gl_wr + (threadIdx.x % 32) * 8;
285 +
286 + extern __shared__ int4 sh[];
287 + int4* sh_code = sh;
288 + int4* sh_code0 = sh_code;
289 + int4* sh_code1 = sh_code + 256 * 8;
290 +
291 + for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
292 + int4 dec = codebook[i];
293 + #pragma unroll
294 + for (int j = 0; j < 8; j++)
295 + sh_code[8 * i + (j + lane) % 8] = dec;
296 + }
297 + __syncthreads();
298 +
299 + float res = 0;
300 +
301 + int iters = (prob_k / 8 - 1) / (8 * 32) + 1;
302 + while (iters--) {
303 + if (pred && a_gl_rd < a_gl_end) {
304 + const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
305 + #pragma unroll
306 + for (int i = 0; i < 8; i++) {
307 + int4 chunk;
308 + half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
309 + half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
310 + #pragma unroll
311 + for (int j = 0; j < 4; j++)
312 + reinterpret_cast<half2*>(&chunk)[j] = __hadd2(a0[j], a1[j]);
313 + C[a_gl_rd * 8 + i] = chunk;
314 + }
315 + }
316 + a_gl_rd += 32;
317 + }
318 +}
319 +
320 +inline int ceildiv(int a, int b) {
321 + return (a + b - 1) / b;
322 +}
323 +
324 +const int THREAD_M = 16;
325 +
326 +void code1x16_matvec_cuda(
327 + const void* __restrict__ A,
328 + const void* __restrict__ B,
329 + void* __restrict__ C,
330 + const void* __restrict__ codebook,
331 + int prob_m,
332 + int prob_k,
333 + const int4 codebook_a_sizes,
334 + const int codebook_stride
335 +) {
336 + int sms;
337 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
338 + int waves = 0;
339 + int thread_m;
340 + do {
341 + waves++;
342 + thread_m = ceildiv(prob_m, waves * sms);
343 + } while (thread_m > THREAD_M);
344 +
345 + int blocks = ceildiv(prob_m, thread_m);
346 + int threads = 32 * thread_m;
347 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
348 + Code1x16MatVec<<<blocks, threads, 16*32*9, stream>>>(
349 + (const int4*) A,
350 + (const int4*) B,
351 + (int4*) C,
352 + (const int4*) codebook,
353 + prob_m,
354 + prob_k,
355 + codebook_a_sizes,
356 + codebook_stride
357 + );
358 +}
359 +
360 +void code2x8_matvec_cuda(
361 + const void* __restrict__ A,
362 + const void* __restrict__ B,
363 + void* __restrict__ C,
364 + const void* __restrict__ codebook,
365 + int prob_m,
366 + int prob_k,
367 + const int4 codebook_a_sizes,
368 + const int codebook_stride
369 +) {
370 + int sms;
371 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
372 + int waves = 0;
373 + int thread_m;
374 + do {
375 + waves++;
376 + thread_m = ceildiv(prob_m, waves * sms);
377 + } while (thread_m > THREAD_M);
378 +
379 + int blocks = ceildiv(prob_m, thread_m);
380 + int threads = 32 * thread_m;
381 + int shared = 16 * (2 * 256 * 8 + 32 * 9);
382 + cudaFuncSetAttribute(
383 + Code2x8MatVec, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
384 + );
385 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
386 + Code2x8MatVec<<<blocks, threads, shared, stream>>>(
387 + (const int4*) A,
388 + (const int4*) B,
389 + (int4*) C,
390 + (const int4*) codebook,
391 + prob_m,
392 + prob_k,
393 + codebook_a_sizes,
394 + codebook_stride
395 + );
396 +}
397 +
398 +void code1x16_dequant_cuda(
399 + const void* __restrict__ A,
400 + void* __restrict__ C,
401 + const void* __restrict__ codebook,
402 + int prob_m,
403 + int prob_k,
404 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
405 + const int codebook_stride // as int4.
406 +) {
407 + int sms;
408 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
409 + int waves = 0;
410 + int thread_m;
411 + do {
412 + waves++;
413 + thread_m = ceildiv(prob_m, waves * sms);
414 + } while (thread_m > THREAD_M);
415 +
416 + int blocks = ceildiv(prob_m, thread_m);
417 + int threads = 32 * thread_m;
418 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
419 + Code1x16Dequant<<<blocks, threads, 0, stream>>>(
420 + (const int4*) A,
421 + (int4*) C,
422 + (const int4*) codebook,
423 + prob_m,
424 + prob_k,
425 + codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
426 + codebook_stride // as int4.
427 + );
428 +}
429 +
430 +// Dequantizes the code and codebook into weights.
431 +void code2x8_dequant_cuda(
432 + const void* __restrict__ A,
433 + void* __restrict__ C,
434 + const void* __restrict__ codebook,
435 + int prob_m,
436 + int prob_k,
437 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
438 + const int codebook_stride // as int4
439 +) {
440 + int sms;
441 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
442 + int waves = 0;
443 + int thread_m;
444 + do {
445 + waves++;
446 + thread_m = ceildiv(prob_m, waves * sms);
447 + } while (thread_m > THREAD_M);
448 +
449 + int blocks = ceildiv(prob_m, thread_m);
450 + int threads = 32 * thread_m;
451 + int shared = 16 * (2 * 256 * 8 + 32 * 9);
452 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
453 +
454 + cudaFuncSetAttribute(
455 + Code2x8Dequant, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
456 + );
457 + Code2x8Dequant<<<blocks, threads, shared, stream>>>(
458 + (const int4*) A,
459 + (int4*) C,
460 + (const int4*) codebook,
461 + prob_m,
462 + prob_k,
463 + codebook_a_sizes,
464 + codebook_stride
465 + );
466 +}
467 +
468 +int codebook_stride(const torch::Tensor& codebooks)
469 +{
470 + return codebooks.stride(0) * codebooks.element_size() / sizeof(int4);
471 +}
472 +
473 +void code1x16_matvec(
474 + const torch::Tensor& A,
475 + const torch::Tensor& B,
476 + torch::Tensor& C,
477 + const torch::Tensor& codebook,
478 + const int4 codebook_a_sizes // cumulative sizes of A spanning each codebook, at most 3 long.
479 +) {
480 + const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
481 + int prob_m = C.size(0);
482 + int prob_k = B.size(0);
483 +
484 + code1x16_matvec_cuda(
485 + A.data_ptr(),
486 + B.data_ptr(),
487 + C.data_ptr(),
488 + codebook.data_ptr(),
489 + prob_m,
490 + prob_k,
491 + codebook_a_sizes,
492 + codebook_stride(codebook)
493 + );
494 +}
495 +
496 +torch::Tensor code1x16_matmat(
497 + const torch::Tensor& input,
498 + const torch::Tensor& codes,
499 + const torch::Tensor& codebooks,
500 + const torch::Tensor& scales,
501 + const int4 codebook_a_sizes,
502 + const std::optional<torch::Tensor>& bias) {
503 + auto input_sizes = input.sizes();
504 + auto out_features = codes.size(0) * codebooks.size(2);
505 + auto flat_input = input.reshape({-1, input.size(-1)});
506 + auto flat_output = torch::empty({flat_input.size(0), out_features},
507 + torch::TensorOptions()
508 + .dtype(input.dtype())
509 + .device(input.device())
510 + );
511 +
512 + for (int i = 0; i < flat_input.size(0); ++i) {
513 + auto input_vec = flat_input.index({i});
514 + auto output_vec = flat_output.index({i});
515 + code1x16_matvec(
516 + codes.squeeze(2),
517 + input_vec,
518 + output_vec,
519 + codebooks,
520 + codebook_a_sizes
521 + );
522 + }
523 + flat_output *= scales.flatten().unsqueeze(0);
524 +
525 + if (bias.has_value()) {
526 + flat_output += bias->unsqueeze(0);
527 + }
528 +
529 + auto output_sizes = input_sizes.vec();
530 + output_sizes.pop_back();
531 + output_sizes.push_back(-1);
532 + auto output = flat_output.reshape(output_sizes);
533 + return output;
534 +}
535 +
536 +void code2x8_matvec(
537 + const torch::Tensor& A,
538 + const torch::Tensor& B,
539 + torch::Tensor& C,
540 + const torch::Tensor& codebook,
541 + const int4 codebook_a_sizes
542 +) {
543 + const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
544 + int prob_m = C.size(0);
545 + int prob_k = B.size(0);
546 + code2x8_matvec_cuda(
547 + A.data_ptr(),
548 + B.data_ptr(),
549 + C.data_ptr(),
550 + codebook.data_ptr(),
551 + prob_m,
552 + prob_k,
553 + codebook_a_sizes,
554 + 2 * codebook_stride(codebook)
555 + );
556 +}
557 +
558 +torch::Tensor code2x8_matmat(
559 + const torch::Tensor& input,
560 + const torch::Tensor& codes,
561 + const torch::Tensor& codebooks,
562 + const torch::Tensor& scales,
563 + const int4 codebook_a_sizes,
564 + const std::optional<torch::Tensor>& bias
565 +) {
566 + auto input_sizes = input.sizes();
567 + auto out_features = codes.size(0) * codebooks.size(2);
568 + auto flat_input = input.reshape({-1, input.size(-1)});
569 + auto flat_output = torch::empty({flat_input.size(0), out_features},
570 + torch::TensorOptions()
571 + .dtype(input.dtype())
572 + .device(input.device())
573 + );
574 +
575 + for (int i = 0; i < flat_input.size(0); ++i) {
576 + auto input_vec = flat_input.index({i});
577 + auto output_vec = flat_output.index({i});
578 + code2x8_matvec(
579 + codes.squeeze(2),
580 + input_vec,
581 + output_vec,
582 + codebooks,
583 + codebook_a_sizes
584 + );
585 + }
586 + flat_output *= scales.flatten().unsqueeze(0);
587 + if (bias.has_value()) {
588 + flat_output += bias->unsqueeze(0);
589 + }
590 +
591 + auto output_sizes = input_sizes.vec();
592 + output_sizes.pop_back();
593 + output_sizes.push_back(-1);
594 + auto output = flat_output.reshape(output_sizes);
595 + return output;
596 +}
597 +
598 +// Accumulate the partition sizes.
599 +int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes)
600 +{
601 + int4 cumulative_sizes;
602 + auto cumulative_size = &cumulative_sizes.x;
603 + int i = 0;
604 + int last = 0;
605 + assert(codebook_partition_sizes.size(0) <= 4);
606 + for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size)
607 + {
608 + *cumulative_size = codebook_partition_sizes[i].item<int>() + last;
609 + last = *cumulative_size;
610 + }
611 + // fill in the rest with unreachable.
612 + for (; i < 4; ++i, ++cumulative_size)
613 + {
614 + *cumulative_size = last*10;
615 + }
616 + return cumulative_sizes;
617 +}
618 +
619 +} // namespace aqlm
620 +} // namespace vllm
621 +
622 +
623 +torch::Tensor aqlm_gemm(
624 + const torch::Tensor& input,
625 + const torch::Tensor& codes,
626 + const torch::Tensor& codebooks,
627 + const torch::Tensor& scales,
628 + const torch::Tensor& codebook_partition_sizes,
629 + const std::optional<torch::Tensor>& bias
630 +)
631 +{
632 + int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
633 +
634 + int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
635 + int const entries = codebooks.size(1);
636 +
637 + if (nbooks == 1 && entries == (1 << 16))
638 + {
639 + return vllm::aqlm::code1x16_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
640 + }
641 + if (nbooks == 2 && entries == (1 << 8))
642 + {
643 + return vllm::aqlm::code2x8_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
644 + }
645 +
646 + TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
647 + return {};
648 +}
649 +
650 +torch::Tensor aqlm_dequant(
651 + const torch::Tensor& codes,
652 + const torch::Tensor& codebooks,
653 + const torch::Tensor& codebook_partition_sizes
654 +)
655 +{
656 + int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
657 +
658 + int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
659 + int const entries = codebooks.size(1);
660 +
661 + const at::cuda::OptionalCUDAGuard device_guard(device_of(codes));
662 + int rows = codes.size(1);
663 + int cols = codes.size(0);
664 +
665 + auto in_features = codes.size(1) * 8;
666 + auto out_features = codes.size(0);
667 +
668 + assert(out_features = codebook_partition_sizes.sum().item<int>());
669 +
670 + auto weights = torch::empty({out_features, in_features},
671 + torch::TensorOptions()
672 + .dtype(codebooks.dtype())
673 + .device(codebooks.device())
674 + );
675 +
676 + if (nbooks == 1 && entries == (1 << 16))
677 + {
678 + vllm::aqlm::code1x16_dequant_cuda(
679 + codes.data_ptr(),
680 + weights.data_ptr(),
681 + codebooks.data_ptr(),
682 + out_features,
683 + in_features,
684 + cumulative_sizes,
685 + vllm::aqlm::codebook_stride(codebooks));
686 +
687 + // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation.)
688 + // weights *= scales.index({"...", 0, 0});
689 +
690 + return weights;
691 + }
692 +
693 + if (nbooks == 2 && entries == (1 << 8))
694 + {
695 + vllm::aqlm::code2x8_dequant_cuda(
696 + codes.data_ptr(),
697 + weights.data_ptr(),
698 + codebooks.data_ptr(),
699 + out_features,
700 + in_features,
701 + cumulative_sizes,
702 + vllm::aqlm::codebook_stride(codebooks));
703 +
704 + // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation)
705 + // weights *= scales.index({"...", 0, 0});
706 +
707 + return weights;
708 + }
709 +
710 + TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
711 + return {};
712 +}
examples/aqlm_example.py new
+46
@@ -0,0 +1,46 @@
1 +import argparse
2 +
3 +from vllm import LLM, SamplingParams
4 +
5 +
6 +def main():
7 +
8 + parser = argparse.ArgumentParser(description='AQLM examples')
9 +
10 + parser.add_argument('--model',
11 + '-m',
12 + type=str,
13 + default=None,
14 + help='model path, as for HF')
15 + parser.add_argument('--choice',
16 + '-c',
17 + type=int,
18 + default=0,
19 + help='known good models by index, [0-4]')
20 + parser.add_argument('--tensor_parallel_size',
21 + '-t',
22 + type=int,
23 + default=1,
24 + help='tensor parallel size')
25 +
26 + args = parser.parse_args()
27 +
28 + models = [
29 + "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf",
30 + "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-2x8-hf",
31 + "ISTA-DASLab/Llama-2-13b-AQLM-2Bit-1x16-hf",
32 + "ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf",
33 + "BlackSamorez/TinyLlama-1_1B-Chat-v1_0-AQLM-2Bit-1x16-hf",
34 + ]
35 +
36 + model = LLM(args.model if args.model is not None else models[args.choice],
37 + tensor_parallel_size=args.tensor_parallel_size)
38 +
39 + sampling_params = SamplingParams(max_tokens=100, temperature=0)
40 + outputs = model.generate("Hello my name is",
41 + sampling_params=sampling_params)
42 + print(outputs[0].outputs[0].text)
43 +
44 +
45 +if __name__ == '__main__':
46 + main()
tests/models/test_aqlm.py new
+95
@@ -0,0 +1,95 @@
1 +"""Compare the outputs of a AQLM model between vLLM and HF Transformers
2 +
3 +Run `pytest tests/models/test_aqlm.py`.
4 +"""
5 +
6 +import pytest
7 +import torch
8 +
9 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10 +
11 +capability = torch.cuda.get_device_capability()
12 +capability = capability[0] * 10 + capability[1]
13 +aqlm_not_supported = (capability <
14 + QUANTIZATION_METHODS["aqlm"].get_min_capability())
15 +
16 +# In this test we hardcode prompts and generations for the model so we don't
17 +# need to require the AQLM package as a dependency
18 +example_prompts = [
19 + 'vLLM is a high-throughput and memory-efficient inference and serving '
20 + 'engine for LLMs.\n',
21 + 'Briefly describe the major milestones in the development of artificial '
22 + 'intelligence from 1950 to 2020.\n',
23 + 'Compare and contrast artificial intelligence with human intelligence in '
24 + 'terms of processing information.\n',
25 + 'Describe the basic components of a neural network and how it can be '
26 + 'trained.\n',
27 + 'Write a short story about a robot that dreams for the first time.\n',
28 + 'Analyze the impact of the COVID-19 pandemic on global economic structures '
29 + 'and future business models.\n',
30 + 'Explain the cultural significance of the Mona Lisa painting, and how its '
31 + 'perception might vary in Western versus Eastern societies.\n',
32 + "Translate the following English sentence into Japanese, French, and "
33 + "Swahili: 'The early bird catches the worm.'\n"
34 +]
35 +
36 +# These ground truth generations were generated using `transformers==4.38.1
37 +# aqlm==1.1.0 torch==2.2.0`
38 +# and the below code:
39 +# ```python
40 +# from transformers import AutoTokenizer, AutoModelForCausalLM
41 +# model_id = "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"
42 +# quantized_model = AutoModelForCausalLM.from_pretrained(model_id,
43 +# torch_dtype="auto", device_map="cuda").cuda()
44 +# tokenizer = AutoTokenizer.from_pretrained(model_id)
45 +# outputs = []
46 +# for prompt in example_prompts:
47 +# input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"].to("cuda")
48 +# hf_outputs = quantized_model.generate(input_ids, max_new_tokens=32)
49 +# outputs.append(tokenizer.decode(hf_outputs[0][input_ids.shape[1]:]))
50 +# print(outputs)
51 +# ```
52 +ground_truth_generations = [
53 + '\n### Features\n\n- **High-throughput**: v',
54 + 'The major milestones in the development of artificial intelligence from '
55 + '195',
56 + 'Compare and contrast artificial intelligence with human intelligence in '
57 + 'terms of processing information. The',
58 + 'Explain the difference between supervised and unsupervised learning.'
59 + '\nExplain',
60 + 'Write a short story about a robot that dreams for the first time. The',
61 + 'Analyze the impact of the COVID-19 pandemic on global economic',
62 + 'The Mona Lisa is a painting by Leonardo da Vinci, and it',
63 + 'The early bird catches the worm.\nThe early bird catches the'
64 +]
65 +
66 +
67 +@pytest.mark.skipif(aqlm_not_supported,
68 + reason="AQLM is not supported on this GPU type.")
69 +@pytest.mark.parametrize("model", ["ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"])
70 +@pytest.mark.parametrize("dtype", ["half"])
71 +@pytest.mark.parametrize("max_tokens", [16])
72 +@pytest.mark.parametrize("num_logprobs", [1])
73 +def test_models(
74 + vllm_runner,
75 + example_prompts,
76 + model: str,
77 + dtype: str,
78 + max_tokens: int,
79 + num_logprobs: int,
80 +) -> None:
81 +
82 + vllm_model = vllm_runner(model, dtype=dtype)
83 + vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
84 + max_tokens,
85 + num_logprobs)
86 +
87 + # loop through the prompts to compare against the ground truth generations
88 + for prompt_idx in range(len(example_prompts)):
89 + vllm_output_ids, vllm_output_str, vllm_logprobs = vllm_outputs[
90 + prompt_idx]
91 +
92 + print("Prompt: ", repr(example_prompts[prompt_idx]))
93 + print("Reference output:", repr(ground_truth_generations[prompt_idx]))
94 + print("Output output: ", repr(vllm_output_str))
95 + assert vllm_output_str == ground_truth_generations[prompt_idx]
vllm/model_executor/layers/linear.py
+34 -7
@@ -31,7 +31,7 @@ class LinearMethodBase(ABC):
31 @abstractmethod
32 def create_weights(self, layer: torch.nn.Module,
33 input_size_per_partition: int,
34 - output_size_per_partition: int, input_size: int,
34 + output_partition_sizes: List[int], input_size: int,
35 output_size: int, params_dtype: torch.dtype,
36 **extra_weight_attrs):
37 """Create weights for a linear layer.
@@ -70,9 +70,10 @@ class UnquantizedLinearMethod(LinearMethodBase):
70
71 def create_weights(self, layer: torch.nn.Module,
72 input_size_per_partition: int,
73 - output_size_per_partition: int, input_size: int,
73 + output_partition_sizes: List[int], input_size: int,
74 output_size: int, params_dtype: torch.dtype,
75 **extra_weight_attrs):
76 + output_size_per_partition = sum(output_partition_sizes)
77 weight = Parameter(torch.empty(output_size_per_partition,
78 input_size_per_partition,
79 dtype=params_dtype),
@@ -127,7 +128,7 @@ class ReplicatedLinear(torch.nn.Module):
128 linear_method = UnquantizedLinearMethod()
129 self.linear_method = linear_method
130 self.linear_method.create_weights(self, self.input_size,
130 - self.output_size, self.input_size,
131 + [self.output_size], self.input_size,
132 self.output_size, self.params_dtype)
133 if bias:
134 self.bias = Parameter(
@@ -161,6 +162,8 @@ class ColumnParallelLinear(torch.nn.Module):
162 skip adding bias but instead return it.
163 params_dtype: Data type for the parameters.
164 linear_method: (Maybe quantized) linear method.
165 + output_sizes: list of output sizes packed into one output, like for QKV
166 + the list would be size 3.
167 """
168
169 def __init__(
@@ -172,6 +175,7 @@ class ColumnParallelLinear(torch.nn.Module):
175 skip_bias_add: bool = False,
176 params_dtype: Optional[torch.dtype] = None,
177 linear_method: Optional[LinearMethodBase] = None,
178 + output_sizes: Optional[List[int]] = None,
179 ):
180 super().__init__()
181
@@ -188,10 +192,12 @@ class ColumnParallelLinear(torch.nn.Module):
192 self.params_dtype = params_dtype
193 if linear_method is None:
194 linear_method = UnquantizedLinearMethod()
195 + if output_sizes is None:
196 + output_sizes = [output_size]
197 self.linear_method = linear_method
198 self.linear_method.create_weights(self,
199 self.input_size,
194 - self.output_size_per_partition,
200 + [x // tp_size for x in output_sizes],
201 self.input_size,
202 self.output_size,
203 self.params_dtype,
@@ -268,14 +274,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
274 tp_size = get_tensor_model_parallel_world_size()
275 assert all(output_size % tp_size == 0 for output_size in output_sizes)
276 super().__init__(input_size, sum(output_sizes), bias, gather_output,
271 - skip_bias_add, params_dtype, linear_method)
277 + skip_bias_add, params_dtype, linear_method,
278 + self.output_sizes)
279
280 def weight_loader(self,
281 param: Parameter,
282 loaded_weight: torch.Tensor,
283 loaded_shard_id: Optional[int] = None):
284 +
285 param_data = param.data
286 output_dim = getattr(param, "output_dim", None)
287 + is_metadata = getattr(param, "is_metadata", False)
288 if loaded_shard_id is None:
289 # Loaded weight is already packed.
290 if output_dim is None:
@@ -328,6 +337,11 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
337 start_idx = tp_rank * shard_size
338 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
339 shard_size)
340 + elif is_metadata:
341 + # metadata indicates fixed size concatenated along dim 0
342 + shard_size = loaded_weight.shape[0]
343 + shard_offset = loaded_shard_id * shard_size
344 + param_data = param_data.narrow(0, shard_offset, shard_size)
345 else:
346 ignore_warning = getattr(param, "ignore_warning", False)
347 if not ignore_warning:
@@ -393,8 +407,14 @@ class QKVParallelLinear(ColumnParallelLinear):
407 input_size = self.hidden_size
408 output_size = (self.num_heads +
409 2 * self.num_kv_heads) * tp_size * self.head_size
410 + output_sizes = [
411 + self.num_heads * tp_size * self.head_size,
412 + self.num_kv_heads * tp_size * self.head_size,
413 + self.num_kv_heads * tp_size * self.head_size
414 + ]
415 +
416 super().__init__(input_size, output_size, bias, False, skip_bias_add,
397 - params_dtype, linear_method)
417 + params_dtype, linear_method, output_sizes)
418
419 def weight_loader(self,
420 param: Parameter,
@@ -402,6 +422,7 @@ class QKVParallelLinear(ColumnParallelLinear):
422 loaded_shard_id: Optional[str] = None):
423 param_data = param.data
424 output_dim = getattr(param, "output_dim", None)
425 + is_metadata = getattr(param, "is_metadata", False)
426
427 if loaded_shard_id is None:
428 # Loaded weight is already packed.
@@ -469,6 +490,12 @@ class QKVParallelLinear(ColumnParallelLinear):
490 start_idx = shard_id * shard_size
491 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
492 shard_size)
493 + elif is_metadata:
494 + # metadata indicates fixed size concatenated along dim 0
495 + shard_size = loaded_weight.shape[0]
496 + shard_index = ["q", "k", "v"].index(loaded_shard_id)
497 + param_data = param_data.narrow(0, shard_index * shard_size,
498 + shard_size)
499 else:
500 ignore_warning = getattr(param, "ignore_warning", False)
501 if not ignore_warning:
@@ -536,7 +563,7 @@ class RowParallelLinear(torch.nn.Module):
563 self.linear_method = linear_method
564 self.linear_method.create_weights(self,
565 self.input_size_per_partition,
539 - self.output_size,
566 + [self.output_size],
567 self.input_size,
568 self.output_size,
569 self.params_dtype,
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -1,5 +1,6 @@
1 from typing import Type
2
3 +from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 from vllm.model_executor.layers.quantization.base_config import (
6 QuantizationConfig)
@@ -9,6 +10,7 @@ from vllm.model_executor.layers.quantization.marlin import MarlinConfig
10 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
11
12 QUANTIZATION_METHODS = {
13 + "aqlm": AQLMConfig,
14 "awq": AWQConfig,
15 "fp8": FP8Config,
16 "gptq": GPTQConfig,
vllm/model_executor/layers/quantization/aqlm.py new
+373
@@ -0,0 +1,373 @@
1 +# Supports AQLM compression, see https://github.com/Vahe1994/AQLM
2 +# and https://arxiv.org/pdf/2401.06118.pdf
3 +
4 +import math
5 +from typing import Any, Dict, List, Optional
6 +
7 +import torch
8 +import torch.nn.functional as F
9 +from torch.nn.parameter import Parameter
10 +
11 +from vllm._C import ops
12 +from vllm.model_executor.layers.linear import (LinearMethodBase,
13 + set_weight_attrs)
14 +from vllm.model_executor.layers.quantization.base_config import (
15 + QuantizationConfig)
16 +
17 +
18 +def get_int_dtype(nbits: int) -> torch.dtype:
19 + if nbits <= 8:
20 + return torch.int8
21 + if nbits <= 16:
22 + return torch.int16
23 + if nbits <= 32:
24 + return torch.int32
25 + if nbits <= 64:
26 + return torch.int64
27 + raise ValueError(f"No dtype available for {nbits}-bit codebooks")
28 +
29 +
30 +@torch.inference_mode()
31 +def unpack_int_data(data: torch.IntTensor, nbits: int) -> torch.IntTensor:
32 + return data.to(torch.int64) % (2**nbits)
33 +
34 +
35 +def dequantize_weight(codes: torch.Tensor,
36 + codebooks: torch.Tensor,
37 + scales: Optional[torch.Tensor] = None) -> torch.Tensor:
38 + """
39 + Decode float weights from quantization codes. Differentiable.
40 + :param codes: tensor of integer quantization codes, shape
41 + [*dims, num_out_groups, num_in_groups, num_codebooks]
42 + :param codebooks: tensor of vectors for each quantization code,
43 + [num_codebooks, codebook_size, out_group_size, in_group_size]
44 + :param scales: weight will be multiplied by this factor, must be
45 + broadcastble with
46 + [*dims, out_groups, num_in_groups, out_group_size, in_group_size]
47 + :return: reconstructed weight tensor of shape
48 + [*dims, num_in_groups*group_size]
49 + """
50 + num_out_groups, num_in_groups, num_codebooks = codes.shape[-3:]
51 + num_codebooks, codebook_size, out_group_size, in_group_size = \
52 + codebooks.shape
53 + out_features = num_out_groups * out_group_size
54 + in_features = num_in_groups * in_group_size
55 + codebook_offsets = torch.arange(
56 + 0, num_codebooks * codebook_size, codebook_size,
57 + device=codes.device) # shape: [num_codebooks]
58 + reconstructed_weight_flat = F.embedding_bag(
59 + codes.flatten(0, -2) + codebook_offsets,
60 + codebooks.flatten(0, 1).flatten(-2, -1),
61 + mode="sum"
62 + ) # [prod(dims) * num_out_groups * num_in_groups, out_group_size
63 + # * in_group_size]
64 +
65 + reconstructed_weight_groupwise = reconstructed_weight_flat.view(
66 + list(codes.shape[:-3]) +
67 + [num_out_groups, num_in_groups, out_group_size, in_group_size])
68 + if scales is not None:
69 + reconstructed_weight_groupwise = reconstructed_weight_groupwise.mul(
70 + scales)
71 + return reconstructed_weight_groupwise.swapaxes(
72 + -3, -2).reshape(list(codes.shape[:-3]) + [out_features, in_features])
73 +
74 +
75 +def dequantize_gemm(
76 + input: torch.Tensor, # [..., in_features]
77 + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
78 + codebooks: torch.
79 + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
80 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
81 + bias: Optional[torch.Tensor],
82 +) -> torch.Tensor:
83 + dequantized_weight = dequantize_weight(
84 + unpack_int_data(codes, codebooks.shape[1].bit_length() - 1),
85 + codebooks,
86 + scales,
87 + )
88 + return F.linear(input, dequantized_weight, bias)
89 +
90 +
91 +# Generic dequantization, slow but flexible.
92 +def generic_dequantize_gemm(
93 + input: torch.Tensor, # [..., in_features]
94 + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
95 + codebooks: torch.
96 + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
97 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
98 + output_partition_sizes: torch.IntTensor,
99 + bias: Optional[torch.Tensor],
100 +) -> torch.Tensor:
101 + output_shape = input.shape[:-1] + (scales.shape[0], )
102 + output = torch.empty(output_shape, dtype=input.dtype, device=input.device)
103 + num_outputs = len(output_partition_sizes)
104 +
105 + # break the inputs and codebooks apart then combine the outputs.
106 + # Surprisingly (to me) this is faster than doing 3 de-quants and 1 big
107 + # multiply at the end.
108 + num_codebooks = codebooks.shape[0] // num_outputs
109 + assert (scales.shape[0] == codes.shape[0])
110 + assert (sum(output_partition_sizes) == scales.shape[0])
111 + output_offset = 0
112 + codebooks_offset = 0
113 + for output_size in output_partition_sizes:
114 + shard_output = dequantize_gemm(
115 + input, codes.narrow(0, output_offset, output_size),
116 + codebooks.narrow(0, codebooks_offset, num_codebooks),
117 + scales.narrow(0, output_offset, output_size), None
118 + if bias is None else bias.narrow(0, output_offset, output_size))
119 +
120 + output_slice = output.narrow(-1, output_offset, output_size)
121 + assert (output_slice.shape == shard_output.shape)
122 + output_slice.copy_(shard_output)
123 + output_offset += output_size
124 + codebooks_offset += num_codebooks
125 + return output
126 +
127 +
128 +# Optimized dequnantize/decompression kernels, supports 1x16 and 2x8
129 +# at 6 and 9 times faster than the generic version above, respectively.
130 +def optimized_dequantize_gemm(
131 + input: torch.Tensor, # [..., in_features]
132 + codes: torch.IntTensor, # [num_out_groups, num_in_groups, num_codebooks]
133 + codebooks: torch.
134 + Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
135 + scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
136 + output_partition_sizes: torch.IntTensor,
137 + bias: Optional[torch.Tensor],
138 +) -> torch.Tensor:
139 + weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
140 +
141 + if bias is None:
142 + # scaling the output is fastest, so we do that when possible.
143 + output = F.linear(input, weights, bias)
144 + orig_shape = output.shape
145 + flattened_output = output.view(-1, output.size(-1))
146 + f_scales = scales.view(-1, scales.shape[0])
147 + b_scales = f_scales.expand(flattened_output.shape[0], -1)
148 + flattened_output *= b_scales
149 + return output.view(orig_shape)
150 + else:
151 + b_scales = scales.view(scales.shape[:-3] + (-1, )).expand(
152 + -1, weights.shape[1])
153 + weights *= b_scales
154 + return F.linear(input, weights, bias)
155 +
156 +
157 +class AQLMConfig(QuantizationConfig):
158 + """Config class for AQLM.
159 +
160 + Reference: https://github.com/Vahe1994/AQLM
161 + """
162 +
163 + def __init__(
164 + self,
165 + in_group_size: int,
166 + nbits_per_codebook: int,
167 + num_codebooks: int,
168 + out_group_size: int,
169 + ) -> None:
170 + self.in_group_size = in_group_size
171 + self.nbits_per_codebook = nbits_per_codebook
172 + self.num_codebooks = num_codebooks
173 + self.out_group_size = out_group_size
174 +
175 + # out_group_size > 1 is untested, and probably won't work as-is.
176 + assert (self.out_group_size == 1)
177 + self.pack_factor = (self.in_group_size * self.out_group_size)
178 +
179 + def __repr__(self) -> str:
180 + return (f"AQLMConfig(in_group_size={self.in_group_size}, "
181 + f"nbits_per_codebook={self.nbits_per_codebook}, "
182 + f"num_codebooks={self.num_codebooks}, "
183 + f"out_group_size={self.out_group_size})")
184 +
185 + @classmethod
186 + def get_name(cls) -> str:
187 + return "aqlm"
188 +
189 + @classmethod
190 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
191 + return [torch.half]
192 +
193 + @classmethod
194 + def get_min_capability(cls) -> int:
195 + return 70
196 +
197 + @classmethod
198 + def get_config_filenames(cls) -> List[str]:
199 + return [] # no extra configs.
200 +
201 + @classmethod
202 + def from_config(cls, config: Dict[str, Any]) -> "AQLMConfig":
203 + in_group_size = cls.get_from_keys(config, ["in_group_size"])
204 + nbits_per_codebook = cls.get_from_keys(config, ["nbits_per_codebook"])
205 + num_code_books = cls.get_from_keys(config, ["num_codebooks"])
206 + out_group_size = cls.get_from_keys(config, ["out_group_size"])
207 + return cls(in_group_size, nbits_per_codebook, num_code_books,
208 + out_group_size)
209 +
210 + def get_linear_method(self) -> "AQLMLinearMethod":
211 + return AQLMLinearMethod(self)
212 +
213 + def get_scaled_act_names(self) -> List[str]:
214 + return []
215 +
216 +
217 +class AQLMLinearMethod(LinearMethodBase):
218 + """Linear method for AQLM.
219 +
220 + Args:
221 + quant_config: The AQLM quantization config.
222 + """
223 +
224 + def __init__(self, quant_config: AQLMConfig):
225 + self.quant_config = quant_config
226 +
227 + def create_weights(self, layer: torch.nn.Module,
228 + input_size_per_partition: int,
229 + output_partition_sizes: List[int], input_size: int,
230 + output_size: int, params_dtype: torch.dtype,
231 + **extra_weight_attrs):
232 + del output_size # Unused.
233 + del input_size # Unused.
234 +
235 + if params_dtype != torch.half:
236 + raise ValueError("Only half is currently supported by aqlm")
237 + if input_size_per_partition % self.quant_config.in_group_size != 0:
238 + raise ValueError(
239 + "The input size is not aligned with the quantized "
240 + "weight shape. This can be caused by too large "
241 + "tensor parallel size.")
242 +
243 + output_size_per_partition = sum(output_partition_sizes)
244 + if output_size_per_partition % self.quant_config.out_group_size != 0:
245 + raise ValueError(
246 + "The output size is not aligned with the quantized "
247 + "weight shape. This can be caused by too large "
248 + "tensor parallel size.")
249 +
250 + codes = Parameter(
251 + torch.empty(
252 + # There could actually be two pack factors, one along input and
253 + # one along output, but we don't currently support
254 + # out_group_size, and only the one along output needs to be
255 + # marked with "packed_dim" in order for QKVLinear to work.
256 + output_size_per_partition,
257 + input_size_per_partition // self.quant_config.pack_factor,
258 + self.quant_config.num_codebooks,
259 + dtype=get_int_dtype(self.quant_config.nbits_per_codebook),
260 + ),
261 + requires_grad=False,
262 + )
263 +
264 + set_weight_attrs(
265 + codes,
266 + {
267 + "input_dim": 1,
268 + "output_dim": 0,
269 + "packed_dim": 1,
270 + "pack_factor": self.quant_config.pack_factor,
271 + },
272 + )
273 +
274 + codebooks = Parameter(
275 + torch.empty(
276 + self.quant_config.num_codebooks * len(output_partition_sizes),
277 + 2**self.quant_config.nbits_per_codebook,
278 + self.quant_config.out_group_size,
279 + self.quant_config.in_group_size,
280 + dtype=params_dtype,
281 + ),
282 + requires_grad=False,
283 + )
284 + set_weight_attrs(
285 + codebooks,
286 + {
287 + # metadata indicates fixed size concatenated along dim 0
288 + "is_metadata":
289 + True,
290 + "output_partition_sizes":
291 + torch.tensor(output_partition_sizes, device='cpu'),
292 + },
293 + )
294 +
295 + scales = Parameter(
296 + torch.empty(
297 + (
298 + output_size_per_partition //
299 + self.quant_config.out_group_size,
300 + 1,
301 + 1,
302 + 1,
303 + ),
304 + dtype=params_dtype,
305 + ),
306 + requires_grad=False,
307 + )
308 + set_weight_attrs(
309 + scales,
310 + {
311 + "output_dim": 0,
312 + "packed_dim": 0,
313 + "pack_factor": self.quant_config.out_group_size
314 + },
315 + )
316 +
317 + layer.register_parameter("codes", codes)
318 + set_weight_attrs(codes, extra_weight_attrs)
319 + layer.register_parameter("codebooks", codebooks)
320 + set_weight_attrs(codebooks, extra_weight_attrs)
321 + layer.register_parameter("scales", scales)
322 + set_weight_attrs(scales, extra_weight_attrs)
323 +
324 + def apply_weights(
325 + self,
326 + layer: torch.nn.Module,
327 + x: torch.Tensor,
328 + bias: Optional[torch.Tensor] = None,
329 + ) -> torch.Tensor:
330 + codebooks = layer.codebooks
331 + codes = layer.codes
332 + scales = layer.scales
333 + output_partition_sizes = getattr(codebooks, "output_partition_sizes",
334 + None)
335 +
336 + nbooks = codes.shape[2]
337 + ingroups = codebooks.shape[3]
338 + outgroups = codebooks.shape[2]
339 + bits = codebooks.shape[1]
340 +
341 + # We support these formats with dedicated gemm and decompression
342 + # kernels.
343 + if ingroups == 8 and outgroups == 1 and (
344 + (bits == 256 and nbooks == 2) or (bits == 65536 and nbooks == 1)):
345 +
346 + # thresholds determined by timings on an A6000, one GPU
347 + use_gemv = math.prod(x.shape[:-1]) <= 6
348 +
349 + return ops.aqlm_gemm(
350 + x,
351 + codes,
352 + codebooks,
353 + scales,
354 + output_partition_sizes,
355 + bias,
356 + ) if use_gemv else optimized_dequantize_gemm(
357 + x,
358 + codes,
359 + codebooks,
360 + scales,
361 + output_partition_sizes,
362 + bias,
363 + )
364 +
365 + # fall back all unoptimized formats
366 + return generic_dequantize_gemm(
367 + x,
368 + codes,
369 + codebooks,
370 + scales,
371 + output_partition_sizes,
372 + bias,
373 + )
vllm/model_executor/layers/quantization/awq.py
+3 -1
@@ -81,7 +81,7 @@ class AWQLinearMethod(LinearMethodBase):
81
82 def create_weights(self, layer: torch.nn.Module,
83 input_size_per_partition: int,
84 - output_size_per_partition: int, input_size: int,
84 + output_partition_sizes: List[int], input_size: int,
85 output_size: int, params_dtype: torch.dtype,
86 **extra_weight_attrs):
87 if input_size_per_partition % self.quant_config.group_size != 0:
@@ -89,6 +89,8 @@ class AWQLinearMethod(LinearMethodBase):
89 "The input size is not aligned with the quantized "
90 "weight shape. This can be caused by too large "
91 "tensor parallel size.")
92 +
93 + output_size_per_partition = sum(output_partition_sizes)
94 if output_size_per_partition % self.quant_config.pack_factor != 0:
95 raise ValueError(
96 "The output size is not aligned with the quantized "
vllm/model_executor/layers/quantization/gptq.py
+2 -1
@@ -91,7 +91,7 @@ class GPTQLinearMethod(LinearMethodBase):
91 self,
92 layer: torch.nn.Module,
93 input_size_per_partition: int,
94 - output_size_per_partition: int,
94 + output_partition_sizes: List[int],
95 input_size: int,
96 output_size: int,
97 params_dtype: torch.dtype,
@@ -103,6 +103,7 @@ class GPTQLinearMethod(LinearMethodBase):
103 "The input size is not aligned with the quantized "
104 "weight shape. This can be caused by too large "
105 "tensor parallel size.")
106 + output_size_per_partition = sum(output_partition_sizes)
107 if (output_size_per_partition % self.quant_config.pack_factor.numerator
108 != 0):
109 raise ValueError(
vllm/model_executor/layers/quantization/marlin.py
+2 -1
@@ -93,7 +93,7 @@ class MarlinLinearMethod(LinearMethodBase):
93 self,
94 layer: torch.nn.Module,
95 input_size_per_partition: int,
96 - output_size_per_partition: int,
96 + output_partition_sizes: List[int],
97 input_size: int,
98 output_size: int,
99 params_dtype: torch.dtype,
@@ -106,6 +106,7 @@ class MarlinLinearMethod(LinearMethodBase):
106 f"The params dtype must be float16, but got {params_dtype}")
107
108 # Validate output_size_per_partition
109 + output_size_per_partition = sum(output_partition_sizes)
110 if output_size_per_partition % self.quant_config.min_n_threads != 0:
111 raise ValueError(
112 f"Weight output_size_per_partition = "
vllm/model_executor/layers/quantization/squeezellm.py
+3 -1
@@ -70,7 +70,7 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
70
71 def create_weights(self, layer: torch.nn.Module,
72 input_size_per_partition: int,
73 - output_size_per_partition: int, input_size: int,
73 + output_partition_sizes: List[int], input_size: int,
74 output_size: int, params_dtype: torch.dtype,
75 **extra_weight_attrs):
76 if input_size_per_partition % self.quant_config.pack_factor != 0:
@@ -78,6 +78,8 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
78 "The input size is not aligned with the quantized "
79 "weight shape. This can be caused by too large "
80 "tensor parallel size.")
81 +
82 + output_size_per_partition = sum(output_partition_sizes)
83 qweight = Parameter(
84 torch.empty(
85 input_size_per_partition // self.quant_config.pack_factor,