@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0fbfc4b81b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add GPTQ support (#916)

CHU Tianxiang committed Dec 15, 2023 at 19:04 UTC 0fbfc4b81b9208f13ceb82d1ea92ff14a6e56088
35 files changed +1781 -81
benchmarks/benchmark_latency.py
+1 -1
@@ -84,7 +84,7 @@ if __name__ == '__main__':
84 parser.add_argument('--tokenizer', type=str, default=None)
85 parser.add_argument('--quantization',
86 '-q',
87 - choices=['awq', 'squeezellm', None],
87 + choices=['awq', 'gptq', 'squeezellm', None],
88 default=None)
89 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
90 parser.add_argument('--input-len', type=int, default=32)
benchmarks/benchmark_throughput.py
+1 -1
@@ -244,7 +244,7 @@ if __name__ == "__main__":
244 parser.add_argument("--tokenizer", type=str, default=None)
245 parser.add_argument('--quantization',
246 '-q',
247 - choices=['awq', 'squeezellm', None],
247 + choices=['awq', 'gptq', 'squeezellm', None],
248 default=None)
249 parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
250 parser.add_argument("--n",
csrc/ops.h
+12
@@ -77,3 +77,15 @@ void squeezellm_gemm(
77 torch::Tensor mat,
78 torch::Tensor mul,
79 torch::Tensor lookup_table);
80 +
81 +torch::Tensor gptq_gemm(
82 + torch::Tensor a,
83 + torch::Tensor b_q_weight,
84 + torch::Tensor b_gptq_qzeros,
85 + torch::Tensor b_gptq_scales,
86 + torch::Tensor b_g_idx,
87 + bool use_exllama);
88 +
89 +void gptq_shuffle(
90 + torch::Tensor q_weight,
91 + torch::Tensor q_perm);
csrc/pybind.cpp
+2 -2
@@ -52,8 +52,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
52 // Quantization ops
53 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
54 #endif
55 -
56 -
55 + ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
56 + ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
57 ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
58
59 // Cache ops
csrc/quantization/gptq/compat.cuh new
+64
@@ -0,0 +1,64 @@
1 +/*
2 +Copied from https://github.com/turboderp/exllamav2
3 +*/
4 +
5 +#ifndef _compat_cuh
6 +#define _compat_cuh
7 +
8 +namespace vllm {
9 +namespace gptq {
10 +// atomicAdd for half types, to support CC < 7.x
11 +
12 +__device__ __forceinline__ void atomicAdd_half(half* address, half val)
13 +{
14 + unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2));
15 + unsigned int old = *address_as_ui;
16 + unsigned int assumed;
17 +
18 + do
19 + {
20 + assumed = old;
21 + __half_raw hsum;
22 + hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
23 + half tmpres = __hadd(hsum, val);
24 + hsum = __half_raw(tmpres);
25 + old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x;
26 + old = atomicCAS(address_as_ui, assumed, old);
27 + }
28 + while (assumed != old);
29 +}
30 +
31 +// atomicAdd for half2 types
32 +
33 +__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val)
34 +{
35 + unsigned int* address_as_ui = (unsigned int*)address;
36 + unsigned int old = *address_as_ui;
37 + unsigned int assumed;
38 + do
39 + {
40 + assumed = old;
41 + half2 old_val = *((half2*)&old);
42 + half2 new_val = __hadd2(old_val, val);
43 + old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
44 + }
45 + while (assumed != old);
46 +}
47 +
48 +//
49 +
50 +#if defined(__CUDA_ARCH__) || defined(USE_ROCM)
51 +#if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
52 +
53 +__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); }
54 +
55 +#if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
56 +__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); }
57 +#endif
58 +
59 +#endif
60 +#endif
61 +
62 +} // namespace gptq
63 +} // namespace vllm
64 +#endif
csrc/quantization/gptq/matrix_view.cuh new
+151
@@ -0,0 +1,151 @@
1 +/*
2 +Adapted from https://github.com/turboderp/exllamav2 and https://github.com/turboderp/exllama
3 +*/
4 +
5 +#ifndef _matrix_view_cuh
6 +#define _matrix_view_cuh
7 +
8 +#include <cuda_runtime.h>
9 +#include <cuda_fp16.h>
10 +
11 +#include "qdq_util.cuh"
12 +
13 +namespace vllm {
14 +namespace gptq {
15 +
16 +class MatrixView_half
17 +{
18 +public:
19 + const half* data;
20 + const int height;
21 + const int width;
22 +
23 + __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width)
24 + : data(data), height(height), width(width)
25 + { }
26 +
27 + __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
28 + __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
29 + __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); }
30 + __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; }
31 +
32 + __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const
33 + {
34 + half2* ptr = (half2*) item_ptr(row, column);
35 + half2 i01 = ptr[0];
36 + half2 i23 = ptr[1];
37 + items[0] = __low2half(i01);
38 + items[1] = __high2half(i01);
39 + items[2] = __low2half(i23);
40 + items[3] = __high2half(i23);
41 + }
42 + __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const
43 + {
44 + half2* ptr = (half2*)item_ptr(row, column);
45 + half2 i01 = ptr[0];
46 + half2 i23 = ptr[1];
47 + items[0] = __half2float(__low2half(i01));
48 + items[1] = __half2float(__high2half(i01));
49 + items[2] = __half2float(__low2half(i23));
50 + items[3] = __half2float(__high2half(i23));
51 + }
52 +
53 + __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const
54 + {
55 + half2* ptr = (half2*)item_ptr(row, column);
56 + half2 i01 = ptr[0];
57 + half2 i23 = ptr[1];
58 + items[0] = __half2half2(__low2half(i01));
59 + items[1] = __half2half2(__high2half(i01));
60 + items[2] = __half2half2(__low2half(i23));
61 + items[3] = __half2half2(__high2half(i23));
62 + }
63 +};
64 +
65 +class MatrixView_half_rw
66 +{
67 +public:
68 + half* data;
69 + const int height;
70 + const int width;
71 +
72 + __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width)
73 + : data(data), height(height), width(width)
74 + { }
75 +
76 + __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
77 + __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
78 + __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; }
79 + __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; }
80 + __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; }
81 +
82 + __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3)
83 + {
84 + half2 v01 = __halves2half2(v0, v1);
85 + half2 v23 = __halves2half2(v2, v3);
86 + half2* ptr = (half2*) item_ptr(row, column);
87 + ptr[0] = v01;
88 + ptr[1] = v23;
89 + }
90 +};
91 +
92 +class MatrixView_q4_row
93 +{
94 +public:
95 + const uint32_t* data;
96 + const int height;
97 + const int width;
98 +
99 + __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width)
100 + : data(data), height(height), width(width)
101 + { }
102 +
103 + __device__ __forceinline__ int item(int row, int column) const
104 + {
105 + int shift = (column & 0x07) * 4;
106 + return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
107 + }
108 +
109 + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
110 + {
111 + int shift = (column & 0x07) * 4;
112 + uint32_t d = data[row * width / 8 + column / 8] >> shift;
113 + items[0] = d & 0x0f;
114 + items[1] = (d >> 4) & 0x0f;
115 + }
116 +
117 + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
118 + {
119 + int shift = (column & 0x07) * 4;
120 + uint32_t d = data[row * width / 8 + column / 8] >> shift;
121 + items[0] = d & 0x0f;
122 + items[1] = (d >> 4) & 0x0f;
123 + items[2] = (d >> 8) & 0x0f;
124 + items[3] = (d >> 12) & 0x0f;
125 + }
126 +};
127 +
128 +class MatrixView_q4_column
129 +{
130 +public:
131 + const uint32_t* data;
132 + const int height;
133 + const int width;
134 +
135 + __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width)
136 + : data(data), height(height), width(width)
137 + { }
138 +
139 + __device__ __forceinline__ int item(int row, int column) const
140 + {
141 + int shift = (row & 0x07) * 4;
142 + return (data[row / 8 * width + column] >> shift) & 0x0f;
143 + }
144 +
145 + __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { return data[row / 8 * width + column]; }
146 + __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; }
147 +};
148 +
149 +} // namespace gptq
150 +} // namespace vllm
151 +#endif
csrc/quantization/gptq/q_gemm.cu new
+859
@@ -0,0 +1,859 @@
1 +/*
2 +Adapted from https://github.com/turboderp/exllamav2 and https://github.com/qwopqwop200/GPTQ-for-LLaMa
3 +*/
4 +
5 +#include <cstdint>
6 +#include <cstdio>
7 +
8 +#include <torch/extension.h>
9 +#include <c10/cuda/CUDAGuard.h>
10 +#include <ATen/cuda/CUDAContext.h>
11 +#include <cuda_runtime.h>
12 +#include <cuda_fp16.h>
13 +
14 +#include "compat.cuh"
15 +#include "matrix_view.cuh"
16 +#include "qdq_4.cuh"
17 +
18 +namespace vllm {
19 +namespace gptq {
20 +
21 +#define BLOCK_KN_SIZE 128
22 +#define BLOCK_M_SIZE_MAX 8
23 +#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)
24 +#define MAX_Q_GEMM_ROWS 50
25 +#define MAX_ALT_GEMM_ROWS 8
26 +#define THREADS_X 32
27 +#define THREADS_Y 32
28 +#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
29 +
30 +#if defined(USE_ROCM)
31 +__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
32 + hipblasOperation_t transA,
33 + hipblasOperation_t transB,
34 + int m,
35 + int n,
36 + int k,
37 + const half* alpha,
38 + const half* AP,
39 + int lda,
40 + const half* BP,
41 + int ldb,
42 + const half* beta,
43 + half* CP,
44 + int ldc) {
45 + return hipblasHgemm(handle, transA, transB, m, n, k,
46 + reinterpret_cast<const hipblasHalf *>(alpha),
47 + reinterpret_cast<const hipblasHalf *>(AP), lda,
48 + reinterpret_cast<const hipblasHalf *>(BP), ldb,
49 + reinterpret_cast<const hipblasHalf *>(beta),
50 + reinterpret_cast<hipblasHalf *>(CP), ldc);
51 +}
52 +#define hipblasHgemm __compat_hipblasHgemm
53 +
54 +// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
55 +#define rocblas_operation_none HIPBLAS_OP_N
56 +#define rocblas_hgemm __compat_hipblasHgemm
57 +#endif
58 +
59 +__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result)
60 +{
61 + half2 result = {};
62 + const half2* a2_ptr = (const half2*)a_ptr;
63 + #pragma unroll
64 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
65 + return __hadd2(result, g_result);
66 +}
67 +
68 +__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
69 +{
70 + half2 result = {};
71 + const half2* a2_ptr = (const half2*)a_ptr;
72 + #pragma unroll
73 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
74 + return __half2float(__low2half(result)) + __half2float(__high2half(result));
75 +}
76 +
77 +typedef void (*fp_gemm_half_q_half_gptq_kernel)
78 +(
79 + const half*,
80 + const uint32_t*,
81 + const uint32_t*,
82 + const half*,
83 + half*,
84 + const int,
85 + const int,
86 + const int,
87 + const int,
88 + const int*
89 +);
90 +
91 +template <bool first_block, int m_count>
92 +__global__ void gemm_half_q_half_gptq_kernel
93 +(
94 + const half* __restrict__ a,
95 + const uint32_t* __restrict__ b_q_weight,
96 + const uint32_t* __restrict__ b_gptq_qzeros,
97 + const half* __restrict__ b_gptq_scales,
98 + half* __restrict__ c,
99 + const int size_m,
100 + const int size_n,
101 + const int size_k,
102 + const int groups,
103 + const int* __restrict__ b_q_perm
104 +)
105 +{
106 + MatrixView_half a_(a, size_m, size_k);
107 + MatrixView_half_rw c_(c, size_m, size_n);
108 + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
109 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
110 +
111 + int t = threadIdx.x;
112 +
113 + // Block
114 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
115 + int offset_m = blockIdx.y * m_count;
116 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
117 +
118 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
119 + int end_m = min(offset_m + m_count, size_m);
120 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
121 +
122 + int n = offset_n + t * 4;
123 +
124 + // Preload block_a
125 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
126 +
127 + if (offset_k + t < end_k)
128 + {
129 + for (int m = 0; m < m_count; ++m)
130 + {
131 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
132 + half* block_a_ptr = block_a[m];
133 +
134 + half a0;
135 + if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
136 + else a0 = a_ptr[offset_k + t];
137 + block_a_ptr[t] = a0;
138 + }
139 + }
140 +
141 + // Zero output
142 + if (n >= size_n) return;
143 +
144 + if (blockIdx.z == 0)
145 + {
146 + for (int m = 0; m < m_count; m++)
147 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
148 + }
149 +
150 + __syncthreads();
151 +
152 + // Find initial group
153 + int groupsize = size_k / groups;
154 + int group = offset_k / groupsize;
155 + int nextgroup = offset_k + groupsize;
156 +
157 + // a, b offset
158 + int qk = offset_k / (32 / 4);
159 +
160 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
161 + const half* a_ptr = &block_a[0][0];
162 + int a_stride = BLOCK_KN_SIZE;
163 +
164 + // Initial group
165 + int zeros[4];
166 + float scales[4];
167 + half2 z1z16[4][2];
168 + half2 y1y16[4][2];
169 + b_gptq_qzeros_.item4(zeros, group, n);
170 + b_gptq_scales_.item4_f(scales, group, n);
171 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
172 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
173 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
174 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
175 +
176 + // Column result
177 + float block_c[m_count][4] = {};
178 +
179 + // Dequantize and multiply
180 + int k = offset_k;
181 + while (k < end_k)
182 + {
183 + if (k == nextgroup)
184 + {
185 + group++;
186 + nextgroup += groupsize;
187 + b_gptq_qzeros_.item4(zeros, group, n);
188 + b_gptq_scales_.item4_f(scales, group, n);
189 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
190 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
191 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
192 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
193 + }
194 +
195 + #pragma unroll
196 + for (int j = 0; j < 4; j++)
197 + {
198 + const int4* b_ptr4 = (int4*) b_ptr;
199 + int4 load_int4 = *b_ptr4;
200 +
201 + half2 dq[4][4];
202 + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
203 + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
204 + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
205 + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
206 +
207 + #pragma unroll
208 + for (int m = 0; m < m_count; m++)
209 + {
210 + block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]);
211 + block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]);
212 + block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]);
213 + block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]);
214 + }
215 +
216 + b_ptr += size_n;
217 + a_ptr += 8;
218 + }
219 +
220 + k += 32;
221 + }
222 +
223 + for (int m = 0; m < m_count; m++)
224 + {
225 + half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
226 + half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));
227 + half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));
228 + atomicAdd(out , result01);
229 + atomicAdd(out + 1, result23);
230 + }
231 +}
232 +
233 +
234 +fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count)
235 +{
236 + #if BLOCK_M_SIZE_MAX >= 1
237 + if (m_count == 1) return gemm_half_q_half_gptq_kernel<true, 1>;
238 + #endif
239 + #if BLOCK_M_SIZE_MAX >= 2
240 + if (m_count == 2) return gemm_half_q_half_gptq_kernel<true, 2>;
241 + #endif
242 + #if BLOCK_M_SIZE_MAX >= 3
243 + if (m_count == 3) return gemm_half_q_half_gptq_kernel<true, 3>;
244 + #endif
245 + #if BLOCK_M_SIZE_MAX >= 4
246 + if (m_count == 4) return gemm_half_q_half_gptq_kernel<true, 4>;
247 + #endif
248 + #if BLOCK_M_SIZE_MAX >= 5
249 + if (m_count == 5) return gemm_half_q_half_gptq_kernel<true, 5>;
250 + #endif
251 + #if BLOCK_M_SIZE_MAX >= 6
252 + if (m_count == 6) return gemm_half_q_half_gptq_kernel<true, 6>;
253 + #endif
254 + #if BLOCK_M_SIZE_MAX >= 7
255 + if (m_count == 7) return gemm_half_q_half_gptq_kernel<true, 7>;
256 + #endif
257 + #if BLOCK_M_SIZE_MAX >= 8
258 + if (m_count == 8) return gemm_half_q_half_gptq_kernel<true, 8>;
259 + #endif
260 + return NULL;
261 +}
262 +
263 +
264 +void gemm_half_q_half_cuda_part
265 +(
266 + const half* a,
267 + const uint32_t* b_q_weight,
268 + const uint32_t* b_gptq_qzeros,
269 + const half* b_gptq_scales,
270 + const int* b_q_perm,
271 + half* c,
272 + int size_m,
273 + int size_n,
274 + int size_k,
275 + int m_count,
276 + int groups
277 +)
278 +{
279 + dim3 blockDim, gridDim;
280 + blockDim.x = BLOCK_KN_SIZE;
281 + blockDim.y = 1;
282 + blockDim.z = 1;
283 + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
284 + gridDim.y = DIVIDE(size_m, m_count);
285 + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
286 +
287 + fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
288 +
289 + kernel<<<gridDim, blockDim>>>
290 + (
291 + a,
292 + b_q_weight,
293 + b_gptq_qzeros,
294 + b_gptq_scales,
295 + c,
296 + size_m,
297 + size_n,
298 + size_k,
299 + groups,
300 + b_q_perm
301 + );
302 +}
303 +
304 +
305 +__global__ void reconstruct_exllama_kernel
306 +(
307 + const uint32_t* __restrict__ b_q_weight,
308 + const int* __restrict__ b_q_perm,
309 + const uint32_t* __restrict__ b_gptq_qzeros,
310 + const half* __restrict__ b_gptq_scales,
311 + const int size_k,
312 + const int size_n,
313 + const int groups,
314 + half* __restrict__ b
315 +)
316 +{
317 + MatrixView_half_rw b_(b, size_k, size_n);
318 + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
319 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
320 +
321 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
322 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
323 +
324 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
325 +
326 + // Preload remapping table
327 + __shared__ int perm[BLOCK_KN_SIZE];
328 + int t = threadIdx.x;
329 +
330 + if (b_q_perm)
331 + {
332 + if (offset_k + t < size_k)
333 + perm[t] = b_q_perm[offset_k + t];
334 + }
335 +
336 + // Column
337 + int n = offset_n + t * 4;
338 + if (n >= size_n) return;
339 +
340 + // Find initial group
341 + int groupsize = size_k / groups;
342 + int group = offset_k / groupsize;
343 + int nextgroup = offset_k + groupsize;
344 +
345 + // b offset
346 + int qk = offset_k / (32 / 4);
347 +
348 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
349 +
350 + // Initial zeros/scale
351 + int zeros[4];
352 + half2 scales[4];
353 + half2 z1z16[4][2];
354 + half2 y1y16[4][2];
355 + b_gptq_qzeros_.item4(zeros, group, n);
356 + b_gptq_scales_.item4_h2(scales, group, n);
357 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
358 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
359 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
360 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
361 +
362 + __syncthreads();
363 +
364 + int k = offset_k;
365 + int lk = 0;
366 +
367 + while (k < end_k)
368 + {
369 + if (k == nextgroup)
370 + {
371 + group++;
372 + nextgroup += groupsize;
373 + b_gptq_qzeros_.item4(zeros, group, n);
374 + b_gptq_scales_.item4_h2(scales, group, n);
375 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
376 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
377 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
378 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
379 + }
380 +
381 + for (int p = 0; p < 4; p++)
382 + {
383 + half2 dq[4][4];
384 + const int4* b_ptr4 = (int4*) b_ptr;
385 + int4 load_int4 = *b_ptr4;
386 +
387 + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
388 + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
389 + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
390 + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
391 +
392 + b_ptr += size_n;
393 + //half* dqh = (half*)dq;
394 + if (b_q_perm)
395 + {
396 + for (int j = 0; j < 4; j++)
397 + {
398 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
399 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
400 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
401 + }
402 + }
403 + else
404 + {
405 + for (int j = 0; j < 4; j++)
406 + {
407 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
408 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
409 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
410 + }
411 + }
412 + }
413 + k += 32;
414 + }
415 +}
416 +
417 +
418 +void reconstruct_exllama
419 +(
420 + const uint32_t* b_q_weight,
421 + const uint32_t* b_gptq_qzeros,
422 + const half* b_gptq_scales,
423 + const int* b_q_perm,
424 + half* out,
425 + int height,
426 + int width,
427 + int groups
428 +)
429 +{
430 + dim3 blockDim, gridDim;
431 + blockDim.x = BLOCK_KN_SIZE;
432 + blockDim.y = 1;
433 + gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
434 + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
435 +
436 + reconstruct_exllama_kernel<<<gridDim, blockDim>>>
437 + (
438 + b_q_weight,
439 + b_q_perm,
440 + b_gptq_qzeros,
441 + b_gptq_scales,
442 + height,
443 + width,
444 + groups,
445 + out
446 + );
447 +}
448 +
449 +
450 +__global__ void gemm_half_q_half_alt_kernel(
451 + const half2* __restrict__ vec,
452 + const uint32_t* __restrict__ mat,
453 + half* __restrict__ mul,
454 + const half* __restrict__ scales,
455 + const uint32_t* __restrict__ zeros,
456 + const int* __restrict__ g_idx,
457 + int batch,
458 + int height,
459 + int width
460 +)
461 +{
462 + int zero_width = width / 8;
463 + int vec_height = height * 4;
464 + const int blockwidth2 = BLOCK_KN_SIZE / 2;
465 + int b = blockIdx.y * BLOCK_M_SIZE_MAX;
466 + int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
467 + int h = BLOCK_KN_SIZE * blockIdx.z / 8;
468 + int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;
469 + int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
470 +
471 + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
472 + if (threadIdx.x < h_end) {
473 + for (int m = 0; m < b_end; ++m) {
474 + blockvec[m][threadIdx.x] =
475 + vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
476 + threadIdx.x];
477 + }
478 + }
479 +
480 + __shared__ half2 deq2[256][8];
481 + int val = threadIdx.x / 8;
482 + int off = threadIdx.x % 8;
483 + for (; val < 256; val += BLOCK_KN_SIZE / 8) {
484 + deq2[val][off] = __halves2half2(
485 + __int2half_rn(val & 0xF), __int2half_rn(val >> 4)
486 + );
487 + }
488 +
489 + if (blockIdx.z == 0)
490 + {
491 + for (int m = 0; m < b_end; m++)
492 + mul[(b + m) * width + w] = __int2half_rn(0);
493 + }
494 + __syncthreads();
495 +
496 + int i = width * h + w;
497 + int g_h = h * 8;
498 + int k = 0;
499 + int z_w = w / 8;
500 + int z_mod = (w % 8) * 4;
501 + half2 res2;
502 + half res[BLOCK_M_SIZE_MAX] = {};
503 +
504 + unsigned int tmp;
505 + while (k < h_end) {
506 + tmp = mat[i];
507 + half2 scales_tmp[4];
508 + half2 zeros_tmp[4];
509 + for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
510 + int g = g_idx[g_h + (k + tmp_k) * 2];
511 + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
512 + half scale_f = scales[g * width + w];
513 + half scale_f2 = scales[g2 * width + w];
514 + half2 scale = __halves2half2(scale_f, scale_f2);
515 + half2 zero = __halves2half2(
516 + __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) - 1)),
517 + __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1))
518 + );
519 + scales_tmp[tmp_k] = scale;
520 + zeros_tmp[tmp_k] = zero;
521 + }
522 + for (int m = 0; m < b_end; m++) {
523 + res2 = {};
524 + res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
525 + res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
526 + res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2);
527 + res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2);
528 + res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
529 + }
530 + i += width;
531 + k += 4;
532 + }
533 + for (int m = 0; m < b_end; m++) {
534 + atomicAdd(&mul[(b + m) * width + w], res[m]);
535 + }
536 +}
537 +
538 +
539 +void gemm_half_q_half_alt
540 +(
541 + const half* a,
542 + const uint32_t* b_q_weight,
543 + const uint32_t* b_gptq_qzeros,
544 + const half* b_gptq_scales,
545 + const int* b_g_idx,
546 + half* c,
547 + int size_m,
548 + int size_n,
549 + int size_k
550 +)
551 +{
552 + dim3 blockDim, gridDim;
553 + blockDim.x = BLOCK_KN_SIZE;
554 + blockDim.y = 1;
555 + blockDim.z = 1;
556 + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
557 + gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
558 + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
559 +
560 + gemm_half_q_half_alt_kernel<<<gridDim, blockDim>>>
561 + (
562 + (const half2*) a,
563 + b_q_weight,
564 + c,
565 + b_gptq_scales,
566 + b_gptq_qzeros,
567 + b_g_idx,
568 + size_m,
569 + size_k / 8,
570 + size_n
571 + );
572 +}
573 +
574 +
575 +__global__ void reconstruct_gptq_kernel
576 +(
577 + const uint32_t* __restrict__ w,
578 + const half* __restrict__ w_scales,
579 + const uint32_t* __restrict__ w_zeros,
580 + const int* __restrict__ g_idx,
581 + const int height,
582 + const int width,
583 + const int group,
584 + half* __restrict__ out
585 +)
586 +{
587 + // Start of block
588 +
589 + int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
590 + int row = blockIdx.y * 8;
591 + if (column >= width) return;
592 +
593 + // Views
594 +
595 + MatrixView_q4_column w_(w, height, width);
596 + MatrixView_half_rw out_(out, height, width);
597 + MatrixView_half w_scales_(w_scales, group, width);
598 + MatrixView_q4_row w_zeros_(w_zeros, group, width);
599 +
600 + uint32_t w_read = w_.item_uint32_t(row, column);
601 + half* out_ptr = out_.item_ptr(row, column);
602 +
603 + #pragma unroll
604 + for (int s = 0; s < 32; s += 4)
605 + {
606 + int group = g_idx[row + s / 4];
607 + half w_scale = w_scales_.item(group, column);
608 + uint32_t w_zero = w_zeros_.item(group, column) + 1;
609 + half w_item = __hmul(__int2half_rn((int)((w_read >> s) & 0x0f) - w_zero), w_scale);
610 + *out_ptr = w_item; out_ptr += out_.width;
611 + }
612 +}
613 +
614 +
615 +void reconstruct_gptq
616 +(
617 + const uint32_t* b_q_weight,
618 + const uint32_t* b_gptq_qzeros,
619 + const half* b_gptq_scales,
620 + const int* b_g_idx,
621 + half* out,
622 + int height,
623 + int width,
624 + int groups
625 +)
626 +{
627 + dim3 blockDim, gridDim;
628 + blockDim.x = BLOCK_KN_SIZE;
629 + blockDim.y = 1;
630 + gridDim.y = DIVIDE(height, 8);
631 + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
632 + reconstruct_gptq_kernel<<<gridDim, blockDim>>>
633 + (
634 + b_q_weight,
635 + b_gptq_scales,
636 + b_gptq_qzeros,
637 + b_g_idx,
638 + height,
639 + width,
640 + groups,
641 + out
642 + );
643 +}
644 +
645 +
646 +void gemm_half_q_half_cuda
647 +(
648 + cublasHandle_t cublas_handle,
649 + const half* a,
650 + const uint32_t* b_q_weight,
651 + const uint32_t* b_gptq_qzeros,
652 + const half* b_gptq_scales,
653 + const int* b_g_idx,
654 + half* c,
655 + half* temp_dq,
656 + int size_m,
657 + int size_n,
658 + int size_k,
659 + int groups,
660 + bool use_exllama
661 +)
662 +{
663 + if ((use_exllama && size_m > MAX_Q_GEMM_ROWS) || (!use_exllama && size_m > MAX_ALT_GEMM_ROWS)) {
664 + // Reconstruct FP16 matrix, then cuBLAS
665 + if (use_exllama) {
666 + reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq,
667 + size_k, size_n, groups);
668 + }
669 + else
670 + {
671 + reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
672 + temp_dq, size_k, size_n, groups);
673 + }
674 +
675 + const half alpha = __float2half(1.0f);
676 + const half beta = __float2half(0.0f);
677 + cublasHgemm(cublas_handle,
678 + CUBLAS_OP_N,
679 + CUBLAS_OP_N,
680 + size_n, size_m, size_k,
681 + &alpha, temp_dq, size_n,
682 + a, size_k,
683 + &beta, c, size_n);
684 + }
685 + else if (use_exllama)
686 + {
687 + // Quantized matmul
688 + int max_chunks = size_m / BLOCK_M_SIZE_MAX;
689 + int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
690 + int last_chunk_size = size_m - last_chunk;
691 +
692 + if (max_chunks)
693 + {
694 + gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
695 + c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
696 + groups);
697 + }
698 +
699 + if (last_chunk_size)
700 + {
701 + gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight, b_gptq_qzeros,
702 + b_gptq_scales, b_g_idx, c + last_chunk * size_n,
703 + last_chunk_size, size_n, size_k, last_chunk_size,
704 + groups);
705 + }
706 + }
707 + else
708 + {
709 + gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
710 + c, size_m, size_n, size_k);
711 + }
712 +}
713 +
714 +
715 +__global__ void shuffle_kernel
716 +(
717 + uint32_t* __restrict__ b_q_weight,
718 + const int size_k,
719 + const int size_n
720 +)
721 +{
722 + int n = blockIdx.x * THREADS_X + threadIdx.x;
723 + if (n >= size_n) return;
724 + int k = 0;
725 + uint32_t* b_ptr = b_q_weight + n;
726 + while (k < size_k) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
727 +}
728 +
729 +
730 +__global__ void make_sequential_kernel
731 +(
732 + const uint32_t* __restrict__ w,
733 + uint32_t* __restrict__ w_new,
734 + const int* __restrict__ q_perm,
735 + const int w_height,
736 + const int w_width
737 +)
738 +{
739 + const uint64_t* w2 = (uint64_t*) w;
740 + uint64_t* w_new2 = (uint64_t*) w_new;
741 + int w2_stride = w_width >> 1;
742 + int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
743 + if (w2_column >= w2_stride) return;
744 + int w_new2_row = blockIdx.y;
745 + int q_perm_idx = w_new2_row << 3;
746 + uint64_t dst = 0;
747 +
748 + #pragma unroll
749 + for (int i = 0; i < 8; i++)
750 + {
751 + int source_row = q_perm[q_perm_idx++];
752 +
753 + int w2_row = source_row >> 3;
754 + int w2_subrow = source_row & 0x07;
755 + int w2_row_shift = w2_subrow << 2;
756 + int wnew2_row_shift = i << 2;
757 +
758 + uint64_t src = w2[w2_row * w2_stride + w2_column];
759 + src >>= w2_row_shift;
760 + src &= 0x0000000f0000000f;
761 + src <<= wnew2_row_shift;
762 + dst |= src;
763 + }
764 + w_new2[w_new2_row * w2_stride + w2_column] = dst;
765 +}
766 +
767 +
768 +void shuffle_exllama_weight
769 +(
770 + uint32_t* q_weight,
771 + int* q_perm,
772 + int height,
773 + int width
774 +)
775 +{
776 + if (q_perm)
777 + {
778 + uint32_t* new_qweight = NULL;
779 + cudaMalloc(&new_qweight, height / 8 * width * sizeof(uint32_t));
780 +
781 + dim3 blockDim, gridDim;
782 + blockDim.x = THREADS_X;
783 + blockDim.y = 1;
784 + gridDim.x = DIVIDE(width, THREADS_X);
785 + gridDim.y = height / 8;
786 +
787 + make_sequential_kernel<<<gridDim, blockDim>>>
788 + (
789 + q_weight,
790 + new_qweight,
791 + q_perm,
792 + height / 8,
793 + width
794 + );
795 + // Replace qweights
796 + cudaMemcpyAsync(q_weight, new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
797 + // Cleanup
798 + cudaDeviceSynchronize();
799 + cudaFree(new_qweight);
800 + }
801 + dim3 blockDim, gridDim;
802 + blockDim.x = THREADS_X;
803 + blockDim.y = 1;
804 + gridDim.x = DIVIDE(width, THREADS_X);
805 + gridDim.y = 1;
806 + shuffle_kernel<<<gridDim, blockDim>>>(q_weight, height, width);
807 +}
808 +
809 +} // namespace gptq
810 +} // namespace vllm
811 +
812 +torch::Tensor gptq_gemm
813 +(
814 + torch::Tensor a,
815 + torch::Tensor b_q_weight,
816 + torch::Tensor b_gptq_qzeros,
817 + torch::Tensor b_gptq_scales,
818 + torch::Tensor b_g_idx,
819 + bool use_exllama
820 +)
821 +{
822 + const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
823 + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
824 + at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
825 + at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 8, b_q_weight.size(1)}, options);
826 +
827 + vllm::gptq::gemm_half_q_half_cuda
828 + (
829 + at::cuda::getCurrentCUDABlasHandle(),
830 + (const half*) a.data_ptr(),
831 + (const uint32_t*) b_q_weight.data_ptr(),
832 + (const uint32_t*)b_gptq_qzeros.data_ptr(),
833 + (const half*) b_gptq_scales.data_ptr(),
834 + b_g_idx.device().is_meta() ? NULL : (const int*) b_g_idx.data_ptr(),
835 + (half*) c.data_ptr(),
836 + (half*) temp_dq.data_ptr(),
837 + c.size(0), // m
838 + c.size(1), // n
839 + a.size(1), // k
840 + b_gptq_qzeros.size(0), // group number
841 + use_exllama
842 + );
843 + return c;
844 +}
845 +
846 +void gptq_shuffle
847 +(
848 + torch::Tensor q_weight,
849 + torch::Tensor q_perm
850 +)
851 +{
852 + const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
853 + vllm::gptq::shuffle_exllama_weight(
854 + (uint32_t*) q_weight.data_ptr(),
855 + q_perm.device().is_meta() ? NULL : (int*) q_perm.data_ptr(),
856 + q_weight.size(0) * 8,
857 + q_weight.size(1)
858 + );
859 +}
csrc/quantization/gptq/qdq_4.cuh new
+235
@@ -0,0 +1,235 @@
1 +/*
2 +Copied from https://github.com/turboderp/exllamav2
3 +*/
4 +
5 +#ifndef _qdq_4_cuh
6 +#define _qdq_4_cuh
7 +
8 +#include "qdq_util.cuh"
9 +
10 +namespace vllm {
11 +namespace gptq {
12 +// Permutation:
13 +//
14 +// 77775555 33331111 66664444 22220000
15 +
16 +__forceinline__ __device__ void shuffle_4bit_8
17 +(
18 + uint32_t* q,
19 + int stride
20 +)
21 +{
22 + uint32_t qa = q[0];
23 + uint32_t qb = 0;
24 +
25 + #pragma unroll
26 + for (int i = 0; i < 4; i++)
27 + {
28 + uint32_t qa0 = qa & 0x0f;
29 + uint32_t qa1 = (qa & 0xf0) >> 4;
30 + qa >>= 8;
31 + qb |= (qa1 << (i * 4 + 16));
32 + qb |= (qa0 << (i * 4));
33 + }
34 + q[0] = qb;
35 +}
36 +
37 +__forceinline__ __device__ void dequant_4bit_8
38 +(
39 + const uint32_t q_0,
40 + half2 (&dq)[4],
41 + int stride
42 +)
43 +{
44 + const uint32_t c0 = 0x64006400;
45 + const half y16_ = __float2half_rn(1.0f / 16.0f);
46 + const half2 y16 = __halves2half2(y16_, y16_);
47 + const half z1_ = __float2half_rn(-1024.0f - 8.0f);
48 + const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);
49 + const half2 z1 = __halves2half2(z1_, z1_);
50 + const half2 z16 = __halves2half2(z16_, z16_);
51 +
52 + uint32_t qa = q_0;
53 + half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
54 + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
55 + qa >>= 8;
56 + half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
57 + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
58 +
59 + dq[0] = __hadd2(q0.as_half2, z1);
60 + dq[1] = __hfma2(q1.as_half2, y16, z16);
61 + dq[2] = __hadd2(q2.as_half2, z1);
62 + dq[3] = __hfma2(q3.as_half2, y16, z16);
63 +}
64 +
65 +__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
66 +(
67 + const uint32_t zero,
68 + const half scale,
69 + half2 (&z1z16)[2],
70 + half2 (&y1y16)[2]
71 +)
72 +{
73 + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
74 + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
75 +
76 + half2 scale2 = __half2half2(scale);
77 +
78 + z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
79 + z1z16[1] = __hmul2(scale2, __half2half2(z16));
80 +
81 + const half y1 = __float2half_rn(1.0f);
82 + const half y16 = __float2half_rn(1.0f / 16.0f);
83 +
84 + y1y16[0] = __hmul2(scale2, __half2half2(y1));
85 + y1y16[1] = __hmul2(scale2, __half2half2(y16));
86 +}
87 +
88 +__forceinline__ __device__ void dequant_4bit_8_prep_zero
89 +(
90 + const uint32_t zero,
91 + half2(&z1z16)[2],
92 + half2(&y1y16)[2]
93 +)
94 +{
95 + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
96 + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
97 +
98 + z1z16[0] = __half2half2(z1.as_half);
99 + z1z16[1] = __half2half2(z16);
100 +
101 + const half y1 = __float2half_rn(1.0f);
102 + const half y16 = __float2half_rn(1.0f / 16.0f);
103 +
104 + y1y16[0] = __half2half2(y1);
105 + y1y16[1] = __half2half2(y16);
106 +}
107 +
108 +
109 +__forceinline__ __device__ void dequant_4bit_8_gptq
110 +(
111 + const uint32_t q_0,
112 + half2 (&dq)[4],
113 + half2 (&z1z16)[2],
114 + half2 (&y1y16)[2],
115 + int stride,
116 + bool scaled
117 +)
118 +{
119 + const uint32_t c0 = 0x64006400;
120 +
121 + uint32_t qa = q_0;
122 + half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 )
123 + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
124 + qa >>= 8;
125 + half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 )
126 + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
127 +
128 + if (scaled)
129 + {
130 + dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
131 + dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
132 + dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
133 + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
134 + }
135 + else
136 + {
137 + dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
138 + dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z )
139 + dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
140 + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z )
141 + }
142 +}
143 +} // namespace gptq
144 +} // namespace vllm
145 +
146 +#else
147 +
148 +namespace vllm {
149 +namespace gptq {
150 +__forceinline__ __device__ void shuffle_4bit_8
151 +(
152 + uint32_t* q,
153 + int stride
154 +)
155 +{
156 +}
157 +
158 +__forceinline__ __device__ void dequant_4bit_8
159 +(
160 + const uint32_t q_0,
161 + half2 (&dq)[4],
162 + int stride
163 +)
164 +{
165 + half dqh[8];
166 + for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);
167 +
168 + for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
169 +}
170 +
171 +__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
172 +(
173 + const uint32_t zero,
174 + const half scale,
175 + half2 (&z1)[2],
176 + half2 (&y1)[2]
177 +)
178 +{
179 + half z = __int2half_rn(-((int)zero));
180 + z = __hmul(z, scale);
181 + z1[0] = __half2half2(z);
182 + y1[0] = __half2half2(scale);
183 +}
184 +
185 +__forceinline__ __device__ void dequant_4bit_8_prep_zero
186 +(
187 + const uint32_t zero,
188 + half2(&z1)[2],
189 + half2(&y1)[2]
190 +)
191 +{
192 + half z = __int2half_rn(-((int)zero));
193 + z1[0] = __half2half2(z);
194 +}
195 +
196 +__forceinline__ __device__ void dequant_4bit_8_gptq
197 +(
198 + const uint32_t q_0,
199 + half2 (&dq)[4],
200 + half2 (&z1)[2],
201 + half2 (&y1)[2],
202 + int stride,
203 + bool scaled
204 +)
205 +{
206 + half2 dqh2[8];
207 +
208 + uint32_t qa = q_0;
209 + for (int i = 0; i < 4; i++)
210 + {
211 + half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;
212 + half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;
213 + dqh2[i] = __halves2half2(d0, d1);
214 + }
215 +
216 + if (scaled)
217 + {
218 + dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);
219 + dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);
220 + dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);
221 + dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);
222 + }
223 + else
224 + {
225 + dq[0] = __hadd2(dqh2[0], z1[0]);
226 + dq[1] = __hadd2(dqh2[1], z1[0]);
227 + dq[2] = __hadd2(dqh2[2], z1[0]);
228 + dq[3] = __hadd2(dqh2[3], z1[0]);
229 + }
230 +}
231 +
232 +} // namespace gptq
233 +} // namespace vllm
234 +
235 +#endif
csrc/quantization/gptq/qdq_util.cuh new
+60
@@ -0,0 +1,60 @@
1 +/*
2 +Copied from https://github.com/turboderp/exllamav2
3 +*/
4 +
5 +#ifndef _qdq_util_cuh
6 +#define _qdq_util_cuh
7 +
8 +namespace vllm {
9 +namespace gptq {
10 +
11 +union half2_uint32
12 +{
13 + uint32_t as_uint32;
14 + half2 as_half2;
15 + __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
16 + __device__ half2_uint32(half2 val) : as_half2(val) {}
17 +};
18 +
19 +union half_uint16
20 +{
21 + uint16_t as_uint16;
22 + half as_half;
23 + __device__ half_uint16(uint16_t val) : as_uint16(val) {}
24 + __device__ half_uint16(half val) : as_half(val) {}
25 +};
26 +
27 +// Max_scale premultiplied by 1/256
28 +
29 +__forceinline__ __device__ half dq_scale(const int qs, const half max_scale)
30 +{
31 + int qs_i = qs + 1;
32 + half qs_h = __int2half_rn(qs_i * qs_i);
33 + qs_h = __hmul(qs_h, max_scale);
34 + return qs_h;
35 +}
36 +
37 +__forceinline__ __device__ half dq(const int q, const int qzero, const half scale)
38 +{
39 + return __hmul(__int2half_rn(q - qzero), scale);
40 +}
41 +
42 +__forceinline__ __device__ half dq_ns(const int q, const int qzero)
43 +{
44 + //return __hsub(__int2half_rn(q), __int2half_rn(qzero));
45 + return __int2half_rn(q - qzero);
46 +}
47 +
48 +__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask)
49 +{
50 + return (int)((q >> shift) & mask);
51 +}
52 +
53 +__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask)
54 +{
55 + return (int)(__funnelshift_rc(q0, q1, shift) & mask);
56 +}
57 +
58 +} // namespace gptq
59 +} // namespace vllm
60 +#endif
setup.py
+1
@@ -219,6 +219,7 @@ vllm_extension_sources = [
219 "csrc/activation_kernels.cu",
220 "csrc/layernorm_kernels.cu",
221 "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222 + "csrc/quantization/gptq/q_gemm.cu",
223 "csrc/cuda_utils_kernels.cu",
224 "csrc/pybind.cpp",
225 ]
vllm/config.py
+1 -1
@@ -142,7 +142,7 @@ class ModelConfig:
142 self.tokenizer_mode = tokenizer_mode
143
144 def _verify_quantization(self) -> None:
145 - supported_quantization = ["awq", "squeezellm"]
145 + supported_quantization = ["awq", "gptq", "squeezellm"]
146 rocm_not_supported_quantization = ["awq"]
147 if self.quantization is not None:
148 self.quantization = self.quantization.lower()
vllm/engine/arg_utils.py
+1 -1
@@ -179,7 +179,7 @@ class EngineArgs:
179 parser.add_argument('--quantization',
180 '-q',
181 type=str,
182 - choices=['awq', 'squeezellm', None],
182 + choices=['awq', 'gptq', 'squeezellm', None],
183 default=None,
184 help='Method used to quantize the weights')
185 return parser
vllm/entrypoints/llm.py
+3 -2
@@ -38,8 +38,9 @@ class LLM:
38 However, if the `torch_dtype` in the config is `float32`, we will
39 use `float16` instead.
40 quantization: The method used to quantize the model weights. Currently,
41 - we support "awq". If None, we assume the model weights are not
42 - quantized and use `dtype` to determine the data type of the weights.
41 + we support "awq", "gptq" and "squeezellm". If None, we assume the
42 + model weights are not quantized and use `dtype` to determine the
43 + data type of the weights.
44 revision: The specific model version to use. It can be a branch name,
45 a tag name, or a commit id.
46 tokenizer_revision: The specific tokenizer version to use. It can be a
vllm/model_executor/layers/linear.py
+37 -23
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, List, Optional
2 +from typing import Any, Dict, List, Optional
3
4 import torch
5 import torch.nn.functional as F
@@ -21,8 +21,10 @@ class LinearMethodBase(ABC):
21 """Base class for different (maybe quantized) linear methods."""
22
23 @abstractmethod
24 - def create_weights(self, input_size: int, output_size: int,
25 - params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
24 + def create_weights(self, input_size_per_partition: int,
25 + output_size_per_partition: int, input_size: int,
26 + output_size: int,
27 + params_dtype: torch.dtype) -> Dict[str, Any]:
28 """Create weights for a linear layer."""
29 raise NotImplementedError
30
@@ -46,10 +48,12 @@ class UnquantizedLinearMethod(LinearMethodBase):
48 def __init__(self, separate_bias_add: bool = False):
49 self.separate_bias_add = separate_bias_add
50
49 - def create_weights(self, input_size: int, output_size: int,
50 - params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
51 - weight = Parameter(torch.empty(output_size,
52 - input_size,
51 + def create_weights(self, input_size_per_partition: int,
52 + output_size_per_partition: int, input_size: int,
53 + output_size: int,
54 + params_dtype: torch.dtype) -> Dict[str, Any]:
55 + weight = Parameter(torch.empty(output_size_per_partition,
56 + input_size_per_partition,
57 device=torch.cuda.current_device(),
58 dtype=params_dtype),
59 requires_grad=False)
@@ -102,9 +106,11 @@ class ReplicatedLinear(torch.nn.Module):
106 linear_method = UnquantizedLinearMethod()
107 self.linear_method = linear_method
108 self.linear_weights = self.linear_method.create_weights(
105 - self.input_size, self.output_size, self.params_dtype)
109 + self.input_size, self.output_size, self.input_size,
110 + self.output_size, self.params_dtype)
111 for name, weight in self.linear_weights.items():
107 - self.register_parameter(name, weight)
112 + if isinstance(weight, torch.Tensor):
113 + self.register_parameter(name, weight)
114 if bias:
115 self.bias = Parameter(
116 torch.empty(self.output_size,
@@ -168,10 +174,12 @@ class ColumnParallelLinear(torch.nn.Module):
174 linear_method = UnquantizedLinearMethod()
175 self.linear_method = linear_method
176 self.linear_weights = self.linear_method.create_weights(
171 - self.input_size, self.output_size_per_partition, self.params_dtype)
177 + self.input_size, self.output_size_per_partition, self.input_size,
178 + self.output_size, self.params_dtype)
179 for name, weight in self.linear_weights.items():
173 - self.register_parameter(name, weight)
174 - set_weight_attrs(weight, {"weight_loader": self.weight_loader})
180 + if isinstance(weight, torch.Tensor):
181 + self.register_parameter(name, weight)
182 + set_weight_attrs(weight, {"weight_loader": self.weight_loader})
183 if bias:
184 self.bias = Parameter(
185 torch.empty(self.output_size_per_partition,
@@ -295,10 +303,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
303 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
304 shard_size)
305 else:
298 - logger.warning(
299 - "Loading a weight without `output_dim` attribute in "
300 - "MergedColumnParallelLinear, assume the weight is "
301 - "the same for all partitions.")
306 + ignore_warning = getattr(param, "ignore_warning", False)
307 + if not ignore_warning:
308 + logger.warning(
309 + "Loading a weight without `output_dim` attribute in "
310 + "MergedColumnParallelLinear, assume the weight is "
311 + "the same for all partitions.")
312 assert param_data.shape == loaded_weight.shape
313 param_data.copy_(loaded_weight)
314
@@ -418,10 +428,12 @@ class QKVParallelLinear(ColumnParallelLinear):
428 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
429 shard_size)
430 else:
421 - logger.warning(
422 - "Loading a weight without `output_dim` attribute in "
423 - "QKVParallelLinear, assume the weight is the same "
424 - "for all partitions.")
431 + ignore_warning = getattr(param, "ignore_warning", False)
432 + if not ignore_warning:
433 + logger.warning(
434 + "Loading a weight without `output_dim` attribute in "
435 + "QKVParallelLinear, assume the weight is the same "
436 + "for all partitions.")
437 assert param_data.shape == loaded_weight.shape
438 param_data.copy_(loaded_weight)
439
@@ -481,10 +493,12 @@ class RowParallelLinear(torch.nn.Module):
493 linear_method = UnquantizedLinearMethod()
494 self.linear_method = linear_method
495 self.linear_weights = self.linear_method.create_weights(
484 - self.input_size_per_partition, self.output_size, self.params_dtype)
496 + self.input_size_per_partition, self.output_size, self.input_size,
497 + self.output_size, self.params_dtype)
498 for name, weight in self.linear_weights.items():
486 - self.register_parameter(name, weight)
487 - set_weight_attrs(weight, {"weight_loader": self.weight_loader})
499 + if isinstance(weight, torch.Tensor):
500 + self.register_parameter(name, weight)
501 + set_weight_attrs(weight, {"weight_loader": self.weight_loader})
502
503 if not reduce_results and (bias and not skip_bias_add):
504 raise ValueError("When not reduce the results, adding bias to the "
vllm/model_executor/layers/quantization/__init__.py
+3 -1
@@ -1,11 +1,13 @@
1 from typing import Type
2
3 +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig
6 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
5 -from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
7
8 _QUANTIZATION_CONFIG_REGISTRY = {
9 "awq": AWQConfig,
10 + "gptq": GPTQConfig,
11 "squeezellm": SqueezeLLMConfig,
12 }
13
vllm/model_executor/layers/quantization/awq.py
+13 -11
@@ -77,14 +77,16 @@ class AWQLinearMethod(LinearMethodBase):
77 def __init__(self, quant_config: AWQConfig):
78 self.quant_config = quant_config
79
80 - def create_weights(self, input_size: int, output_size: int,
81 - params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
82 - if input_size % self.quant_config.group_size != 0:
80 + def create_weights(self, input_size_per_partition: int,
81 + output_size_per_partition: int, input_size: int,
82 + output_size: int,
83 + params_dtype: torch.dtype) -> Dict[str, Any]:
84 + if input_size_per_partition % self.quant_config.group_size != 0:
85 raise ValueError(
86 "The input size is not aligned with the quantized "
87 "weight shape. This can be caused by too large "
88 "tensor parallel size.")
87 - if output_size % self.quant_config.pack_factor != 0:
89 + if output_size_per_partition % self.quant_config.pack_factor != 0:
90 raise ValueError(
91 "The output size is not aligned with the quantized "
92 "weight shape. This can be caused by too large "
@@ -92,8 +94,8 @@ class AWQLinearMethod(LinearMethodBase):
94
95 qweight = Parameter(
96 torch.empty(
95 - input_size,
96 - output_size // self.quant_config.pack_factor,
97 + input_size_per_partition,
98 + output_size_per_partition // self.quant_config.pack_factor,
99 device="cuda",
100 dtype=torch.int32,
101 ),
@@ -108,8 +110,8 @@ class AWQLinearMethod(LinearMethodBase):
110 })
111 qzeros = Parameter(
112 torch.empty(
111 - input_size // self.quant_config.group_size,
112 - output_size // self.quant_config.pack_factor,
113 + input_size_per_partition // self.quant_config.group_size,
114 + output_size_per_partition // self.quant_config.pack_factor,
115 device="cuda",
116 dtype=torch.int32,
117 ),
@@ -124,8 +126,8 @@ class AWQLinearMethod(LinearMethodBase):
126 })
127 scales = Parameter(
128 torch.empty(
127 - input_size // self.quant_config.group_size,
128 - output_size,
129 + input_size_per_partition // self.quant_config.group_size,
130 + output_size_per_partition,
131 device="cuda",
132 dtype=params_dtype,
133 ),
@@ -142,7 +144,7 @@ class AWQLinearMethod(LinearMethodBase):
144 }
145
146 def apply_weights(self,
145 - weights: Dict[str, torch.Tensor],
147 + weights: Dict[str, Any],
148 x: torch.Tensor,
149 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
150 qweight = weights["qweight"]
vllm/model_executor/layers/quantization/gptq.py new
+215
@@ -0,0 +1,215 @@
1 +import enum
2 +from enum import Enum
3 +from typing import Any, Dict, List, Optional
4 +
5 +import torch
6 +from torch.nn.parameter import Parameter
7 +
8 +from vllm._C import ops
9 +from vllm.model_executor.layers.linear import (LinearMethodBase,
10 + set_weight_attrs)
11 +from vllm.model_executor.layers.quantization.base_config import (
12 + QuantizationConfig)
13 +
14 +
15 +class GPTQConfig(QuantizationConfig):
16 + """Config class for GPTQ.
17 +
18 + Reference: https://arxiv.org/abs/2210.17323
19 + """
20 +
21 + def __init__(
22 + self,
23 + weight_bits: int,
24 + group_size: int,
25 + desc_act: bool,
26 + ) -> None:
27 + self.weight_bits = weight_bits
28 + self.group_size = group_size
29 + self.desc_act = desc_act
30 + self.pack_factor = 32 // self.weight_bits
31 + # exllama kernel v1 only supports 4 bit
32 + if self.weight_bits != 4:
33 + raise ValueError(
34 + "Currently, only 4-bit weight quantization is supported for "
35 + f"GPTQ, but got {self.weight_bits} bits.")
36 +
37 + def __repr__(self) -> str:
38 + return (f"GPTQConfig(weight_bits={self.weight_bits}, "
39 + f"group_size={self.group_size}, "
40 + f"desc_act={self.desc_act})")
41 +
42 + @classmethod
43 + def get_name(cls) -> str:
44 + return "gptq"
45 +
46 + @classmethod
47 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
48 + return [torch.half]
49 +
50 + @classmethod
51 + # Need to figure it out
52 + def get_min_capability(cls) -> int:
53 + return 60
54 +
55 + @classmethod
56 + def get_config_filenames(cls) -> List[str]:
57 + return ["quantize_config.json"]
58 +
59 + @classmethod
60 + def from_config(cls, config: Dict[str, Any]) -> "GPTQConfig":
61 + weight_bits = cls.get_from_keys(config, ["bits"])
62 + group_size = cls.get_from_keys(config, ["group_size"])
63 + desc_act = cls.get_from_keys(config, ["desc_act"])
64 + return cls(weight_bits, group_size, desc_act)
65 +
66 + def get_linear_method(self) -> "GPTQLinearMethod":
67 + return GPTQLinearMethod(self)
68 +
69 + def get_scaled_act_names(self) -> List[str]:
70 + return []
71 +
72 +
73 +class ExllamaState(Enum):
74 +
75 + UNUSED = enum.auto()
76 + UNINITIALIZED = enum.auto()
77 + READY = enum.auto()
78 +
79 +
80 +class GPTQLinearMethod(LinearMethodBase):
81 + """Linear method for GPTQ.
82 +
83 + Args:
84 + quant_config: The GPTQ quantization config.
85 + """
86 +
87 + def __init__(self, quant_config: GPTQConfig):
88 + self.quant_config = quant_config
89 +
90 + def create_weights(
91 + self,
92 + input_size_per_partition: int,
93 + output_size_per_partition: int,
94 + input_size: int,
95 + output_size: int,
96 + params_dtype: torch.dtype,
97 + ) -> Dict[str, Any]:
98 + del output_size # Unused.
99 + if input_size_per_partition % self.quant_config.group_size != 0:
100 + raise ValueError(
101 + "The input size is not aligned with the quantized "
102 + "weight shape. This can be caused by too large "
103 + "tensor parallel size.")
104 + if output_size_per_partition % self.quant_config.pack_factor != 0:
105 + raise ValueError(
106 + "The output size is not aligned with the quantized "
107 + "weight shape. This can be caused by too large "
108 + "tensor parallel size.")
109 +
110 + if self.quant_config.group_size != -1:
111 + group_size = self.quant_config.group_size
112 + else:
113 + group_size = input_size
114 + exllama_state = ExllamaState.UNINITIALIZED
115 + scale_and_zero_size = input_size // group_size
116 + scale_and_zero_input_dim = None
117 + if input_size != input_size_per_partition and self.quant_config.group_size != -1:
118 + # For act-order models, we cannot use Exllama for row parallel layer
119 + if self.quant_config.desc_act:
120 + exllama_state = ExllamaState.UNUSED
121 + else:
122 + # we need to partition qzeros and scales for exllama kernel
123 + scale_and_zero_size = input_size_per_partition // group_size
124 + scale_and_zero_input_dim = 0
125 +
126 + qweight = Parameter(
127 + torch.empty(
128 + input_size_per_partition // self.quant_config.pack_factor,
129 + output_size_per_partition,
130 + device="cuda",
131 + dtype=torch.int32,
132 + ),
133 + requires_grad=False,
134 + )
135 + set_weight_attrs(
136 + qweight, {
137 + "input_dim": 0,
138 + "output_dim": 1,
139 + "packed_dim": 0,
140 + "pack_factor": self.quant_config.pack_factor,
141 + })
142 + g_idx = Parameter(
143 + torch.tensor(
144 + [
145 + i // self.quant_config.group_size
146 + for i in range(input_size_per_partition)
147 + ],
148 + device="cuda",
149 + dtype=torch.int32,
150 + ),
151 + requires_grad=False,
152 + )
153 + # Ignore warning from fused linear layers such as QKVParallelLinear.
154 + set_weight_attrs(g_idx, {"input_dim": 0, "ignore_warning": True})
155 + qzeros = Parameter(
156 + torch.empty(
157 + scale_and_zero_size,
158 + output_size_per_partition // self.quant_config.pack_factor,
159 + device="cuda",
160 + dtype=torch.int32,
161 + ),
162 + requires_grad=False,
163 + )
164 + set_weight_attrs(
165 + qzeros, {
166 + "input_dim": scale_and_zero_input_dim,
167 + "output_dim": 1,
168 + "packed_dim": 1,
169 + "pack_factor": self.quant_config.pack_factor,
170 + })
171 + scales = Parameter(
172 + torch.empty(
173 + scale_and_zero_size,
174 + output_size_per_partition,
175 + device="cuda",
176 + dtype=params_dtype,
177 + ),
178 + requires_grad=False,
179 + )
180 + set_weight_attrs(scales, {
181 + "input_dim": scale_and_zero_input_dim,
182 + "output_dim": 1,
183 + })
184 + return {
185 + "qweight": qweight,
186 + "g_idx": g_idx,
187 + "qzeros": qzeros,
188 + "scales": scales,
189 + "exllama_state": exllama_state,
190 + }
191 +
192 + def apply_weights(self,
193 + weights: Dict[str, Any],
194 + x: torch.Tensor,
195 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
196 + qweight = weights["qweight"]
197 + out_shape = x.shape[:-1] + (qweight.shape[-1], )
198 + reshaped_x = x.reshape(-1, x.shape[-1])
199 + # exllama needs to shuffle the weight after the weight is loaded
200 + # here we do the shuffle on first forward pass
201 + if weights["exllama_state"] == ExllamaState.UNINITIALIZED:
202 + if self.quant_config.desc_act:
203 + weights["g_idx"] = torch.argsort(weights["g_idx"]).to(
204 + torch.int)
205 + else:
206 + weights["g_idx"] = torch.empty((1, 1), device="meta")
207 + weights["exllama_state"] = ExllamaState.READY
208 + ops.gptq_shuffle(weights["qweight"], weights["g_idx"])
209 + output = ops.gptq_gemm(reshaped_x, weights["qweight"],
210 + weights["qzeros"], weights["scales"],
211 + weights["g_idx"],
212 + weights["exllama_state"] == ExllamaState.READY)
213 + if bias is not None:
214 + output = output + bias
215 + return output.reshape(out_shape)
vllm/model_executor/layers/quantization/squeezellm.py
+8 -6
@@ -67,17 +67,19 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
67 def __init__(self, quant_config: SqueezeLLMConfig):
68 self.quant_config = quant_config
69
70 - def create_weights(self, input_size: int, output_size: int,
71 - params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
72 - if input_size % self.quant_config.pack_factor != 0:
70 + def create_weights(self, input_size_per_partition: int,
71 + output_size_per_partition: int, input_size: int,
72 + output_size: int,
73 + params_dtype: torch.dtype) -> Dict[str, Any]:
74 + if input_size_per_partition % self.quant_config.pack_factor != 0:
75 raise ValueError(
76 "The input size is not aligned with the quantized "
77 "weight shape. This can be caused by too large "
78 "tensor parallel size.")
79 qweight = Parameter(
80 torch.empty(
79 - input_size // self.quant_config.pack_factor,
80 - output_size,
81 + input_size_per_partition // self.quant_config.pack_factor,
82 + output_size_per_partition,
83 device="cuda",
84 dtype=torch.int32,
85 ),
@@ -108,7 +110,7 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
110 }
111
112 def apply_weights(self,
111 - weights: Dict[str, torch.Tensor],
113 + weights: Dict[str, Any],
114 x: torch.Tensor,
115 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
116 qweight = weights["qweight"]
vllm/model_executor/models/aquila.py
+8 -1
@@ -332,11 +332,18 @@ class AquilaForCausalLM(nn.Module):
332 for (param_name, weight_name, shard_id) in stacked_params_mapping:
333 if weight_name not in name:
334 continue
335 - param = params_dict[name.replace(weight_name, param_name)]
335 + name = name.replace(weight_name, param_name)
336 + # Skip loading extra bias for GPTQ models.
337 + if name.endswith(".bias") and name not in params_dict:
338 + continue
339 + param = params_dict[name]
340 weight_loader = param.weight_loader
341 weight_loader(param, loaded_weight, shard_id)
342 break
343 else:
344 + # Skip loading extra bias for GPTQ models.
345 + if name.endswith(".bias") and name not in params_dict:
346 + continue
347 param = params_dict[name]
348 weight_loader = getattr(param, "weight_loader",
349 default_weight_loader)
vllm/model_executor/models/baichuan.py
+8 -1
@@ -355,11 +355,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
355 for (param_name, weight_name, shard_id) in stacked_params_mapping:
356 if weight_name not in name:
357 continue
358 - param = params_dict[name.replace(weight_name, param_name)]
358 + name = name.replace(weight_name, param_name)
359 + # Skip loading extra bias for GPTQ models.
360 + if name.endswith(".bias") and name not in params_dict:
361 + continue
362 + param = params_dict[name]
363 weight_loader = param.weight_loader
364 weight_loader(param, loaded_weight, shard_id)
365 break
366 else:
367 + # Skip loading extra bias for GPTQ models.
368 + if name.endswith(".bias") and name not in params_dict:
369 + continue
370 param = params_dict[name]
371 weight_loader = getattr(param, "weight_loader",
372 default_weight_loader)
vllm/model_executor/models/chatglm.py
+3
@@ -377,6 +377,9 @@ class ChatGLMForCausalLM(nn.Module):
377 continue
378 if "word_embeddings" in name:
379 name = name.replace(".word_embeddings", "")
380 + # Skip loading extra bias for GPTQ models.
381 + if name.endswith(".bias") and name not in params_dict:
382 + continue
383 param = params_dict[name]
384 weight_loader = getattr(param, "weight_loader",
385 default_weight_loader)
vllm/model_executor/models/falcon.py
+22 -17
@@ -425,27 +425,32 @@ class FalconForCausalLM(nn.Module):
425 params_dict = dict(self.named_parameters())
426 for name, loaded_weight in hf_model_weights_iterator(
427 model_name_or_path, cache_dir, load_format, revision):
428 + # Skip loading extra bias for GPTQ models.
429 + if name.endswith(".bias") and name not in params_dict:
430 + continue
431 param = params_dict[name]
432 if "query_key_value" in name:
433 output_dim = getattr(param, "output_dim", None)
434 loaded_weight_shape = loaded_weight.shape
432 - loaded_weight = loaded_weight.view(
433 - loaded_weight_shape[:output_dim] +
434 - (total_num_kv_heads, num_query_heads_per_kv_head + 2, -1) +
435 - loaded_weight_shape[output_dim + 1:])
436 - wq = loaded_weight.narrow(
437 - output_dim + 1, 0, num_query_heads_per_kv_head).reshape(
438 - *loaded_weight_shape[:output_dim], -1,
439 - *loaded_weight_shape[output_dim + 1:])
440 - wk = loaded_weight.narrow(
441 - output_dim + 1, num_query_heads_per_kv_head,
442 - 1).reshape(*loaded_weight_shape[:output_dim], -1,
443 - *loaded_weight_shape[output_dim + 1:])
444 - wv = loaded_weight.narrow(
445 - output_dim + 1, num_query_heads_per_kv_head + 1,
446 - 1).reshape(*loaded_weight_shape[:output_dim], -1,
447 - *loaded_weight_shape[output_dim + 1:])
448 - loaded_weight = torch.cat([wq, wk, wv], dim=output_dim)
435 + if output_dim is not None:
436 + loaded_weight = loaded_weight.view(
437 + loaded_weight_shape[:output_dim] +
438 + (total_num_kv_heads, num_query_heads_per_kv_head + 2,
439 + -1) + loaded_weight_shape[output_dim + 1:])
440 + wq = loaded_weight.narrow(
441 + output_dim + 1, 0,
442 + num_query_heads_per_kv_head).reshape(
443 + *loaded_weight_shape[:output_dim], -1,
444 + *loaded_weight_shape[output_dim + 1:])
445 + wk = loaded_weight.narrow(
446 + output_dim + 1, num_query_heads_per_kv_head,
447 + 1).reshape(*loaded_weight_shape[:output_dim], -1,
448 + *loaded_weight_shape[output_dim + 1:])
449 + wv = loaded_weight.narrow(
450 + output_dim + 1, num_query_heads_per_kv_head + 1,
451 + 1).reshape(*loaded_weight_shape[:output_dim], -1,
452 + *loaded_weight_shape[output_dim + 1:])
453 + loaded_weight = torch.cat([wq, wk, wv], dim=output_dim)
454
455 weight_loader = getattr(param, "weight_loader",
456 default_weight_loader)
vllm/model_executor/models/gpt2.py
-1
@@ -275,7 +275,6 @@ class GPT2LMHeadModel(nn.Module):
275 if not name.endswith(".weight"):
276 continue
277 loaded_weight = loaded_weight.t()
278 -
278 weight_loader = getattr(param, "weight_loader",
279 default_weight_loader)
280 weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_j.py
+8 -1
@@ -274,11 +274,18 @@ class GPTJForCausalLM(nn.Module):
274 for (param_name, weight_name, shard_id) in stacked_params_mapping:
275 if weight_name not in name:
276 continue
277 - param = params_dict[name.replace(weight_name, param_name)]
277 + name = name.replace(weight_name, param_name)
278 + # Skip loading extra bias for GPTQ models.
279 + if name.endswith(".bias") and name not in params_dict:
280 + continue
281 + param = params_dict[name]
282 weight_loader = param.weight_loader
283 weight_loader(param, loaded_weight, shard_id)
284 break
285 else:
286 + # Skip loading extra bias for GPTQ models.
287 + if name.endswith(".bias") and name not in params_dict:
288 + continue
289 param = params_dict[name]
290 weight_loader = getattr(param, "weight_loader",
291 default_weight_loader)
vllm/model_executor/models/gpt_neox.py
-1
@@ -72,7 +72,6 @@ class GPTNeoXAttention(nn.Module):
72 config.hidden_size,
73 linear_method=linear_method,
74 )
75 -
75 scaling = self.head_size**-0.5
76 rotary_dim = int(self.head_size * config.rotary_pct)
77 assert rotary_dim % 2 == 0
vllm/model_executor/models/internlm.py
+8 -1
@@ -289,11 +289,18 @@ class InternLMForCausalLM(nn.Module):
289 for (param_name, weight_name, shard_id) in stacked_params_mapping:
290 if weight_name not in name:
291 continue
292 - param = params_dict[name.replace(weight_name, param_name)]
292 + name = name.replace(weight_name, param_name)
293 + # Skip loading extra bias for GPTQ models.
294 + if name.endswith(".bias") and name not in params_dict:
295 + continue
296 + param = params_dict[name]
297 weight_loader = param.weight_loader
298 weight_loader(param, loaded_weight, shard_id)
299 break
300 else:
301 + # Skip loading extra bias for GPTQ models.
302 + if name.endswith(".bias") and name not in params_dict:
303 + continue
304 param = params_dict[name]
305 weight_loader = getattr(param, "weight_loader",
306 default_weight_loader)
vllm/model_executor/models/llama.py
+8 -1
@@ -330,11 +330,18 @@ class LlamaForCausalLM(nn.Module):
330 for (param_name, weight_name, shard_id) in stacked_params_mapping:
331 if weight_name not in name:
332 continue
333 - param = params_dict[name.replace(weight_name, param_name)]
333 + name = name.replace(weight_name, param_name)
334 + # Skip loading extra bias for GPTQ models.
335 + if name.endswith(".bias") and name not in params_dict:
336 + continue
337 + param = params_dict[name]
338 weight_loader = param.weight_loader
339 weight_loader(param, loaded_weight, shard_id)
340 break
341 else:
342 + # Skip loading extra bias for GPTQ models.
343 + if name.endswith(".bias") and name not in params_dict:
344 + continue
345 param = params_dict[name]
346 weight_loader = getattr(param, "weight_loader",
347 default_weight_loader)
vllm/model_executor/models/mistral.py
+8 -1
@@ -321,11 +321,18 @@ class MistralForCausalLM(nn.Module):
321 for (param_name, weight_name, shard_id) in stacked_params_mapping:
322 if weight_name not in name:
323 continue
324 - param = params_dict[name.replace(weight_name, param_name)]
324 + name = name.replace(weight_name, param_name)
325 + # Skip loading extra bias for GPTQ models.
326 + if name.endswith(".bias") and name not in params_dict:
327 + continue
328 + param = params_dict[name]
329 weight_loader = param.weight_loader
330 weight_loader(param, loaded_weight, shard_id)
331 break
332 else:
333 + # Skip loading extra bias for GPTQ models.
334 + if name.endswith(".bias") and name not in params_dict:
335 + continue
336 param = params_dict[name]
337 weight_loader = getattr(param, "weight_loader",
338 default_weight_loader)
vllm/model_executor/models/mixtral.py
+9 -2
@@ -153,7 +153,7 @@ class MixtralMoE(nn.Module):
153 self.gate = ReplicatedLinear(config.hidden_size,
154 self.num_total_experts,
155 bias=False,
156 - linear_method=linear_method)
156 + linear_method=None)
157
158 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
159 batch_size, sequence_length, hidden_dim = hidden_states.shape
@@ -418,11 +418,18 @@ class MixtralForCausalLM(nn.Module):
418 for (param_name, weight_name, shard_id) in stacked_params_mapping:
419 if weight_name not in name:
420 continue
421 - param = params_dict[name.replace(weight_name, param_name)]
421 + name = name.replace(weight_name, param_name)
422 + # Skip loading extra bias for GPTQ models.
423 + if name.endswith(".bias") and name not in params_dict:
424 + continue
425 + param = params_dict[name]
426 weight_loader = param.weight_loader
427 weight_loader(param, loaded_weight, shard_id)
428 break
429 else:
430 + # Skip loading extra bias for GPTQ models.
431 + if name.endswith(".bias") and name not in params_dict:
432 + continue
433 param = params_dict[name]
434 weight_loader = getattr(param, "weight_loader",
435 default_weight_loader)
vllm/model_executor/models/mpt.py
+3
@@ -297,6 +297,9 @@ class MPTForCausalLM(nn.Module):
297 params_dict = dict(self.named_parameters(remove_duplicate=False))
298 for name, loaded_weight in hf_model_weights_iterator(
299 model_name_or_path, cache_dir, load_format, revision):
300 + # Skip loading extra bias for GPTQ models.
301 + if name.endswith(".bias") and name not in params_dict:
302 + continue
303 param = params_dict[name]
304 weight_loader = getattr(param, "weight_loader",
305 default_weight_loader)
vllm/model_executor/models/opt.py
+8 -1
@@ -345,11 +345,18 @@ class OPTForCausalLM(nn.Module):
345 for (param_name, weight_name, shard_id) in stacked_params_mapping:
346 if weight_name not in name:
347 continue
348 - param = params_dict[name.replace(weight_name, param_name)]
348 + name = name.replace(weight_name, param_name)
349 + # Skip loading extra bias for GPTQ models.
350 + if name.endswith(".bias") and name not in params_dict:
351 + continue
352 + param = params_dict[name]
353 weight_loader = param.weight_loader
354 weight_loader(param, loaded_weight, shard_id)
355 break
356 else:
357 + # Skip loading extra bias for GPTQ models.
358 + if name.endswith(".bias") and name not in params_dict:
359 + continue
360 param = params_dict[name]
361 weight_loader = getattr(param, "weight_loader",
362 default_weight_loader)
vllm/model_executor/models/phi_1_5.py
+3
@@ -305,6 +305,9 @@ class PhiForCausalLM(nn.Module):
305 if "rotary_emb.inv_freq" in name:
306 continue
307
308 + # Skip loading extra bias for GPTQ models.
309 + if name.endswith(".bias") and name not in params_dict:
310 + continue
311 # pylint: disable=E1136
312 param = params_dict[name]
313 weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/qwen.py
+8 -2
@@ -82,7 +82,6 @@ class QWenAttention(nn.Module):
82 self.num_heads = (self.total_num_heads //
83 tensor_model_parallel_world_size)
84 self.head_dim = hidden_size // self.total_num_heads
85 -
85 self.c_attn = QKVParallelLinear(
86 hidden_size,
87 self.head_dim,
@@ -279,11 +278,18 @@ class QWenLMHeadModel(nn.Module):
278 for (param_name, weight_name, shard_id) in stacked_params_mapping:
279 if weight_name not in name:
280 continue
282 - param = params_dict[name.replace(weight_name, param_name)]
281 + name = name.replace(weight_name, param_name)
282 + # Skip loading extra bias for GPTQ models.
283 + if name.endswith(".bias") and name not in params_dict:
284 + continue
285 + param = params_dict[name]
286 weight_loader = param.weight_loader
287 weight_loader(param, loaded_weight, shard_id)
288 break
289 else:
290 + # Skip loading extra bias for GPTQ models.
291 + if name.endswith(".bias") and name not in params_dict:
292 + continue
293 param = params_dict[name]
294 weight_loader = getattr(param, "weight_loader",
295 default_weight_loader)
vllm/model_executor/models/yi.py
+8 -1
@@ -320,11 +320,18 @@ class YiForCausalLM(nn.Module):
320 for (param_name, weight_name, shard_id) in stacked_params_mapping:
321 if weight_name not in name:
322 continue
323 - param = params_dict[name.replace(weight_name, param_name)]
323 + name = name.replace(weight_name, param_name)
324 + # Skip loading extra bias for GPTQ models.
325 + if name.endswith(".bias") and name not in params_dict:
326 + continue
327 + param = params_dict[name]
328 weight_loader = param.weight_loader
329 weight_loader(param, loaded_weight, shard_id)
330 break
331 else:
332 + # Skip loading extra bias for GPTQ models.
333 + if name.endswith(".bias") and name not in params_dict:
334 + continue
335 param = params_dict[name]
336 weight_loader = getattr(param, "weight_loader",
337 default_weight_loader)
vllm/model_executor/weight_utils.py
+2 -1
@@ -287,4 +287,5 @@ def initialize_dummy_weights(
287 values between -1e-3 and 1e-3 works well for most models.
288 """
289 for param in model.state_dict().values():
290 - param.data.uniform_(low, high)
290 + if torch.is_floating_point(param):
291 + param.data.uniform_(low, high)