A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add GPTQ support (#916)
CHU Tianxiang committed
Dec 15, 2023 at 19:04 UTC
0fbfc4b81b9208f13ceb82d1ea92ff14a6e56088
35 files changed
+1781
-81
benchmarks/benchmark_latency.py
+1
-1
@@ -84,7 +84,7 @@ if __name__ == '__main__':
84
parser.add_argument('--tokenizer', type=str, default=None)
85
parser.add_argument('--quantization',
86
'-q',
87
- choices=['awq', 'squeezellm', None],
87
+ choices=['awq', 'gptq', 'squeezellm', None],
88
default=None)
89
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
90
parser.add_argument('--input-len', type=int, default=32)
benchmarks/benchmark_throughput.py
+1
-1
@@ -244,7 +244,7 @@ if __name__ == "__main__":
244
parser.add_argument("--tokenizer", type=str, default=None)
245
parser.add_argument('--quantization',
246
'-q',
247
- choices=['awq', 'squeezellm', None],
247
+ choices=['awq', 'gptq', 'squeezellm', None],
248
default=None)
249
parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
250
parser.add_argument("--n",
csrc/ops.h
+12
@@ -77,3 +77,15 @@ void squeezellm_gemm(
77
torch::Tensor mat,
78
torch::Tensor mul,
79
torch::Tensor lookup_table);
80
+
81
+torch::Tensor gptq_gemm(
82
+ torch::Tensor a,
83
+ torch::Tensor b_q_weight,
84
+ torch::Tensor b_gptq_qzeros,
85
+ torch::Tensor b_gptq_scales,
86
+ torch::Tensor b_g_idx,
87
+ bool use_exllama);
88
+
89
+void gptq_shuffle(
90
+ torch::Tensor q_weight,
91
+ torch::Tensor q_perm);
csrc/pybind.cpp
+2
-2
@@ -52,8 +52,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
52
// Quantization ops
53
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
54
#endif
55
-
56
-
55
+ ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
56
+ ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
57
ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
58
59
// Cache ops
csrc/quantization/gptq/compat.cuh
new
+64
@@ -0,0 +1,64 @@
1
+/*
2
+Copied from https://github.com/turboderp/exllamav2
3
+*/
4
+
5
+#ifndef _compat_cuh
6
+#define _compat_cuh
7
+
8
+namespace vllm {
9
+namespace gptq {
10
+// atomicAdd for half types, to support CC < 7.x
11
+
12
+__device__ __forceinline__ void atomicAdd_half(half* address, half val)
13
+{
14
+ unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2));
15
+ unsigned int old = *address_as_ui;
16
+ unsigned int assumed;
17
+
18
+ do
19
+ {
20
+ assumed = old;
21
+ __half_raw hsum;
22
+ hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
23
+ half tmpres = __hadd(hsum, val);
24
+ hsum = __half_raw(tmpres);
25
+ old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x;
26
+ old = atomicCAS(address_as_ui, assumed, old);
27
+ }
28
+ while (assumed != old);
29
+}
30
+
31
+// atomicAdd for half2 types
32
+
33
+__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val)
34
+{
35
+ unsigned int* address_as_ui = (unsigned int*)address;
36
+ unsigned int old = *address_as_ui;
37
+ unsigned int assumed;
38
+ do
39
+ {
40
+ assumed = old;
41
+ half2 old_val = *((half2*)&old);
42
+ half2 new_val = __hadd2(old_val, val);
43
+ old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
44
+ }
45
+ while (assumed != old);
46
+}
47
+
48
+//
49
+
50
+#if defined(__CUDA_ARCH__) || defined(USE_ROCM)
51
+#if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
52
+
53
+__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); }
54
+
55
+#if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
56
+__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); }
57
+#endif
58
+
59
+#endif
60
+#endif
61
+
62
+} // namespace gptq
63
+} // namespace vllm
64
+#endif
csrc/quantization/gptq/matrix_view.cuh
new
+151
@@ -0,0 +1,151 @@
1
+/*
2
+Adapted from https://github.com/turboderp/exllamav2 and https://github.com/turboderp/exllama
3
+*/
4
+
5
+#ifndef _matrix_view_cuh
6
+#define _matrix_view_cuh
7
+
8
+#include <cuda_runtime.h>
9
+#include <cuda_fp16.h>
10
+
11
+#include "qdq_util.cuh"
12
+
13
+namespace vllm {
14
+namespace gptq {
15
+
16
+class MatrixView_half
17
+{
18
+public:
19
+ const half* data;
20
+ const int height;
21
+ const int width;
22
+
23
+ __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width)
24
+ : data(data), height(height), width(width)
25
+ { }
26
+
27
+ __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
28
+ __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
29
+ __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); }
30
+ __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; }
31
+
32
+ __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const
33
+ {
34
+ half2* ptr = (half2*) item_ptr(row, column);
35
+ half2 i01 = ptr[0];
36
+ half2 i23 = ptr[1];
37
+ items[0] = __low2half(i01);
38
+ items[1] = __high2half(i01);
39
+ items[2] = __low2half(i23);
40
+ items[3] = __high2half(i23);
41
+ }
42
+ __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const
43
+ {
44
+ half2* ptr = (half2*)item_ptr(row, column);
45
+ half2 i01 = ptr[0];
46
+ half2 i23 = ptr[1];
47
+ items[0] = __half2float(__low2half(i01));
48
+ items[1] = __half2float(__high2half(i01));
49
+ items[2] = __half2float(__low2half(i23));
50
+ items[3] = __half2float(__high2half(i23));
51
+ }
52
+
53
+ __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const
54
+ {
55
+ half2* ptr = (half2*)item_ptr(row, column);
56
+ half2 i01 = ptr[0];
57
+ half2 i23 = ptr[1];
58
+ items[0] = __half2half2(__low2half(i01));
59
+ items[1] = __half2half2(__high2half(i01));
60
+ items[2] = __half2half2(__low2half(i23));
61
+ items[3] = __half2half2(__high2half(i23));
62
+ }
63
+};
64
+
65
+class MatrixView_half_rw
66
+{
67
+public:
68
+ half* data;
69
+ const int height;
70
+ const int width;
71
+
72
+ __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width)
73
+ : data(data), height(height), width(width)
74
+ { }
75
+
76
+ __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
77
+ __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
78
+ __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; }
79
+ __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; }
80
+ __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; }
81
+
82
+ __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3)
83
+ {
84
+ half2 v01 = __halves2half2(v0, v1);
85
+ half2 v23 = __halves2half2(v2, v3);
86
+ half2* ptr = (half2*) item_ptr(row, column);
87
+ ptr[0] = v01;
88
+ ptr[1] = v23;
89
+ }
90
+};
91
+
92
+class MatrixView_q4_row
93
+{
94
+public:
95
+ const uint32_t* data;
96
+ const int height;
97
+ const int width;
98
+
99
+ __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width)
100
+ : data(data), height(height), width(width)
101
+ { }
102
+
103
+ __device__ __forceinline__ int item(int row, int column) const
104
+ {
105
+ int shift = (column & 0x07) * 4;
106
+ return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
107
+ }
108
+
109
+ __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
110
+ {
111
+ int shift = (column & 0x07) * 4;
112
+ uint32_t d = data[row * width / 8 + column / 8] >> shift;
113
+ items[0] = d & 0x0f;
114
+ items[1] = (d >> 4) & 0x0f;
115
+ }
116
+
117
+ __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
118
+ {
119
+ int shift = (column & 0x07) * 4;
120
+ uint32_t d = data[row * width / 8 + column / 8] >> shift;
121
+ items[0] = d & 0x0f;
122
+ items[1] = (d >> 4) & 0x0f;
123
+ items[2] = (d >> 8) & 0x0f;
124
+ items[3] = (d >> 12) & 0x0f;
125
+ }
126
+};
127
+
128
+class MatrixView_q4_column
129
+{
130
+public:
131
+ const uint32_t* data;
132
+ const int height;
133
+ const int width;
134
+
135
+ __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width)
136
+ : data(data), height(height), width(width)
137
+ { }
138
+
139
+ __device__ __forceinline__ int item(int row, int column) const
140
+ {
141
+ int shift = (row & 0x07) * 4;
142
+ return (data[row / 8 * width + column] >> shift) & 0x0f;
143
+ }
144
+
145
+ __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { return data[row / 8 * width + column]; }
146
+ __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; }
147
+};
148
+
149
+} // namespace gptq
150
+} // namespace vllm
151
+#endif
csrc/quantization/gptq/q_gemm.cu
new
+859
@@ -0,0 +1,859 @@
1
+/*
2
+Adapted from https://github.com/turboderp/exllamav2 and https://github.com/qwopqwop200/GPTQ-for-LLaMa
3
+*/
4
+
5
+#include <cstdint>
6
+#include <cstdio>
7
+
8
+#include <torch/extension.h>
9
+#include <c10/cuda/CUDAGuard.h>
10
+#include <ATen/cuda/CUDAContext.h>
11
+#include <cuda_runtime.h>
12
+#include <cuda_fp16.h>
13
+
14
+#include "compat.cuh"
15
+#include "matrix_view.cuh"
16
+#include "qdq_4.cuh"
17
+
18
+namespace vllm {
19
+namespace gptq {
20
+
21
+#define BLOCK_KN_SIZE 128
22
+#define BLOCK_M_SIZE_MAX 8
23
+#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)
24
+#define MAX_Q_GEMM_ROWS 50
25
+#define MAX_ALT_GEMM_ROWS 8
26
+#define THREADS_X 32
27
+#define THREADS_Y 32
28
+#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
29
+
30
+#if defined(USE_ROCM)
31
+__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
32
+ hipblasOperation_t transA,
33
+ hipblasOperation_t transB,
34
+ int m,
35
+ int n,
36
+ int k,
37
+ const half* alpha,
38
+ const half* AP,
39
+ int lda,
40
+ const half* BP,
41
+ int ldb,
42
+ const half* beta,
43
+ half* CP,
44
+ int ldc) {
45
+ return hipblasHgemm(handle, transA, transB, m, n, k,
46
+ reinterpret_cast<const hipblasHalf *>(alpha),
47
+ reinterpret_cast<const hipblasHalf *>(AP), lda,
48
+ reinterpret_cast<const hipblasHalf *>(BP), ldb,
49
+ reinterpret_cast<const hipblasHalf *>(beta),
50
+ reinterpret_cast<hipblasHalf *>(CP), ldc);
51
+}
52
+#define hipblasHgemm __compat_hipblasHgemm
53
+
54
+// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
55
+#define rocblas_operation_none HIPBLAS_OP_N
56
+#define rocblas_hgemm __compat_hipblasHgemm
57
+#endif
58
+
59
+__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result)
60
+{
61
+ half2 result = {};
62
+ const half2* a2_ptr = (const half2*)a_ptr;
63
+ #pragma unroll
64
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
65
+ return __hadd2(result, g_result);
66
+}
67
+
68
+__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
69
+{
70
+ half2 result = {};
71
+ const half2* a2_ptr = (const half2*)a_ptr;
72
+ #pragma unroll
73
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
74
+ return __half2float(__low2half(result)) + __half2float(__high2half(result));
75
+}
76
+
77
+typedef void (*fp_gemm_half_q_half_gptq_kernel)
78
+(
79
+ const half*,
80
+ const uint32_t*,
81
+ const uint32_t*,
82
+ const half*,
83
+ half*,
84
+ const int,
85
+ const int,
86
+ const int,
87
+ const int,
88
+ const int*
89
+);
90
+
91
+template <bool first_block, int m_count>
92
+__global__ void gemm_half_q_half_gptq_kernel
93
+(
94
+ const half* __restrict__ a,
95
+ const uint32_t* __restrict__ b_q_weight,
96
+ const uint32_t* __restrict__ b_gptq_qzeros,
97
+ const half* __restrict__ b_gptq_scales,
98
+ half* __restrict__ c,
99
+ const int size_m,
100
+ const int size_n,
101
+ const int size_k,
102
+ const int groups,
103
+ const int* __restrict__ b_q_perm
104
+)
105
+{
106
+ MatrixView_half a_(a, size_m, size_k);
107
+ MatrixView_half_rw c_(c, size_m, size_n);
108
+ MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
109
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
110
+
111
+ int t = threadIdx.x;
112
+
113
+ // Block
114
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
115
+ int offset_m = blockIdx.y * m_count;
116
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
117
+
118
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
119
+ int end_m = min(offset_m + m_count, size_m);
120
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
121
+
122
+ int n = offset_n + t * 4;
123
+
124
+ // Preload block_a
125
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
126
+
127
+ if (offset_k + t < end_k)
128
+ {
129
+ for (int m = 0; m < m_count; ++m)
130
+ {
131
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
132
+ half* block_a_ptr = block_a[m];
133
+
134
+ half a0;
135
+ if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
136
+ else a0 = a_ptr[offset_k + t];
137
+ block_a_ptr[t] = a0;
138
+ }
139
+ }
140
+
141
+ // Zero output
142
+ if (n >= size_n) return;
143
+
144
+ if (blockIdx.z == 0)
145
+ {
146
+ for (int m = 0; m < m_count; m++)
147
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
148
+ }
149
+
150
+ __syncthreads();
151
+
152
+ // Find initial group
153
+ int groupsize = size_k / groups;
154
+ int group = offset_k / groupsize;
155
+ int nextgroup = offset_k + groupsize;
156
+
157
+ // a, b offset
158
+ int qk = offset_k / (32 / 4);
159
+
160
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
161
+ const half* a_ptr = &block_a[0][0];
162
+ int a_stride = BLOCK_KN_SIZE;
163
+
164
+ // Initial group
165
+ int zeros[4];
166
+ float scales[4];
167
+ half2 z1z16[4][2];
168
+ half2 y1y16[4][2];
169
+ b_gptq_qzeros_.item4(zeros, group, n);
170
+ b_gptq_scales_.item4_f(scales, group, n);
171
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
172
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
173
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
174
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
175
+
176
+ // Column result
177
+ float block_c[m_count][4] = {};
178
+
179
+ // Dequantize and multiply
180
+ int k = offset_k;
181
+ while (k < end_k)
182
+ {
183
+ if (k == nextgroup)
184
+ {
185
+ group++;
186
+ nextgroup += groupsize;
187
+ b_gptq_qzeros_.item4(zeros, group, n);
188
+ b_gptq_scales_.item4_f(scales, group, n);
189
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
190
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
191
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
192
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
193
+ }
194
+
195
+ #pragma unroll
196
+ for (int j = 0; j < 4; j++)
197
+ {
198
+ const int4* b_ptr4 = (int4*) b_ptr;
199
+ int4 load_int4 = *b_ptr4;
200
+
201
+ half2 dq[4][4];
202
+ dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
203
+ dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
204
+ dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
205
+ dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
206
+
207
+ #pragma unroll
208
+ for (int m = 0; m < m_count; m++)
209
+ {
210
+ block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]);
211
+ block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]);
212
+ block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]);
213
+ block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]);
214
+ }
215
+
216
+ b_ptr += size_n;
217
+ a_ptr += 8;
218
+ }
219
+
220
+ k += 32;
221
+ }
222
+
223
+ for (int m = 0; m < m_count; m++)
224
+ {
225
+ half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
226
+ half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));
227
+ half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));
228
+ atomicAdd(out , result01);
229
+ atomicAdd(out + 1, result23);
230
+ }
231
+}
232
+
233
+
234
+fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count)
235
+{
236
+ #if BLOCK_M_SIZE_MAX >= 1
237
+ if (m_count == 1) return gemm_half_q_half_gptq_kernel<true, 1>;
238
+ #endif
239
+ #if BLOCK_M_SIZE_MAX >= 2
240
+ if (m_count == 2) return gemm_half_q_half_gptq_kernel<true, 2>;
241
+ #endif
242
+ #if BLOCK_M_SIZE_MAX >= 3
243
+ if (m_count == 3) return gemm_half_q_half_gptq_kernel<true, 3>;
244
+ #endif
245
+ #if BLOCK_M_SIZE_MAX >= 4
246
+ if (m_count == 4) return gemm_half_q_half_gptq_kernel<true, 4>;
247
+ #endif
248
+ #if BLOCK_M_SIZE_MAX >= 5
249
+ if (m_count == 5) return gemm_half_q_half_gptq_kernel<true, 5>;
250
+ #endif
251
+ #if BLOCK_M_SIZE_MAX >= 6
252
+ if (m_count == 6) return gemm_half_q_half_gptq_kernel<true, 6>;
253
+ #endif
254
+ #if BLOCK_M_SIZE_MAX >= 7
255
+ if (m_count == 7) return gemm_half_q_half_gptq_kernel<true, 7>;
256
+ #endif
257
+ #if BLOCK_M_SIZE_MAX >= 8
258
+ if (m_count == 8) return gemm_half_q_half_gptq_kernel<true, 8>;
259
+ #endif
260
+ return NULL;
261
+}
262
+
263
+
264
+void gemm_half_q_half_cuda_part
265
+(
266
+ const half* a,
267
+ const uint32_t* b_q_weight,
268
+ const uint32_t* b_gptq_qzeros,
269
+ const half* b_gptq_scales,
270
+ const int* b_q_perm,
271
+ half* c,
272
+ int size_m,
273
+ int size_n,
274
+ int size_k,
275
+ int m_count,
276
+ int groups
277
+)
278
+{
279
+ dim3 blockDim, gridDim;
280
+ blockDim.x = BLOCK_KN_SIZE;
281
+ blockDim.y = 1;
282
+ blockDim.z = 1;
283
+ gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
284
+ gridDim.y = DIVIDE(size_m, m_count);
285
+ gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
286
+
287
+ fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
288
+
289
+ kernel<<<gridDim, blockDim>>>
290
+ (
291
+ a,
292
+ b_q_weight,
293
+ b_gptq_qzeros,
294
+ b_gptq_scales,
295
+ c,
296
+ size_m,
297
+ size_n,
298
+ size_k,
299
+ groups,
300
+ b_q_perm
301
+ );
302
+}
303
+
304
+
305
+__global__ void reconstruct_exllama_kernel
306
+(
307
+ const uint32_t* __restrict__ b_q_weight,
308
+ const int* __restrict__ b_q_perm,
309
+ const uint32_t* __restrict__ b_gptq_qzeros,
310
+ const half* __restrict__ b_gptq_scales,
311
+ const int size_k,
312
+ const int size_n,
313
+ const int groups,
314
+ half* __restrict__ b
315
+)
316
+{
317
+ MatrixView_half_rw b_(b, size_k, size_n);
318
+ MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
319
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
320
+
321
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
322
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
323
+
324
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
325
+
326
+ // Preload remapping table
327
+ __shared__ int perm[BLOCK_KN_SIZE];
328
+ int t = threadIdx.x;
329
+
330
+ if (b_q_perm)
331
+ {
332
+ if (offset_k + t < size_k)
333
+ perm[t] = b_q_perm[offset_k + t];
334
+ }
335
+
336
+ // Column
337
+ int n = offset_n + t * 4;
338
+ if (n >= size_n) return;
339
+
340
+ // Find initial group
341
+ int groupsize = size_k / groups;
342
+ int group = offset_k / groupsize;
343
+ int nextgroup = offset_k + groupsize;
344
+
345
+ // b offset
346
+ int qk = offset_k / (32 / 4);
347
+
348
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
349
+
350
+ // Initial zeros/scale
351
+ int zeros[4];
352
+ half2 scales[4];
353
+ half2 z1z16[4][2];
354
+ half2 y1y16[4][2];
355
+ b_gptq_qzeros_.item4(zeros, group, n);
356
+ b_gptq_scales_.item4_h2(scales, group, n);
357
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
358
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
359
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
360
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
361
+
362
+ __syncthreads();
363
+
364
+ int k = offset_k;
365
+ int lk = 0;
366
+
367
+ while (k < end_k)
368
+ {
369
+ if (k == nextgroup)
370
+ {
371
+ group++;
372
+ nextgroup += groupsize;
373
+ b_gptq_qzeros_.item4(zeros, group, n);
374
+ b_gptq_scales_.item4_h2(scales, group, n);
375
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
376
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
377
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
378
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
379
+ }
380
+
381
+ for (int p = 0; p < 4; p++)
382
+ {
383
+ half2 dq[4][4];
384
+ const int4* b_ptr4 = (int4*) b_ptr;
385
+ int4 load_int4 = *b_ptr4;
386
+
387
+ dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
388
+ dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
389
+ dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
390
+ dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
391
+
392
+ b_ptr += size_n;
393
+ //half* dqh = (half*)dq;
394
+ if (b_q_perm)
395
+ {
396
+ for (int j = 0; j < 4; j++)
397
+ {
398
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
399
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
400
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
401
+ }
402
+ }
403
+ else
404
+ {
405
+ for (int j = 0; j < 4; j++)
406
+ {
407
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
408
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
409
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
410
+ }
411
+ }
412
+ }
413
+ k += 32;
414
+ }
415
+}
416
+
417
+
418
+void reconstruct_exllama
419
+(
420
+ const uint32_t* b_q_weight,
421
+ const uint32_t* b_gptq_qzeros,
422
+ const half* b_gptq_scales,
423
+ const int* b_q_perm,
424
+ half* out,
425
+ int height,
426
+ int width,
427
+ int groups
428
+)
429
+{
430
+ dim3 blockDim, gridDim;
431
+ blockDim.x = BLOCK_KN_SIZE;
432
+ blockDim.y = 1;
433
+ gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
434
+ gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
435
+
436
+ reconstruct_exllama_kernel<<<gridDim, blockDim>>>
437
+ (
438
+ b_q_weight,
439
+ b_q_perm,
440
+ b_gptq_qzeros,
441
+ b_gptq_scales,
442
+ height,
443
+ width,
444
+ groups,
445
+ out
446
+ );
447
+}
448
+
449
+
450
+__global__ void gemm_half_q_half_alt_kernel(
451
+ const half2* __restrict__ vec,
452
+ const uint32_t* __restrict__ mat,
453
+ half* __restrict__ mul,
454
+ const half* __restrict__ scales,
455
+ const uint32_t* __restrict__ zeros,
456
+ const int* __restrict__ g_idx,
457
+ int batch,
458
+ int height,
459
+ int width
460
+)
461
+{
462
+ int zero_width = width / 8;
463
+ int vec_height = height * 4;
464
+ const int blockwidth2 = BLOCK_KN_SIZE / 2;
465
+ int b = blockIdx.y * BLOCK_M_SIZE_MAX;
466
+ int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
467
+ int h = BLOCK_KN_SIZE * blockIdx.z / 8;
468
+ int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;
469
+ int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
470
+
471
+ __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
472
+ if (threadIdx.x < h_end) {
473
+ for (int m = 0; m < b_end; ++m) {
474
+ blockvec[m][threadIdx.x] =
475
+ vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
476
+ threadIdx.x];
477
+ }
478
+ }
479
+
480
+ __shared__ half2 deq2[256][8];
481
+ int val = threadIdx.x / 8;
482
+ int off = threadIdx.x % 8;
483
+ for (; val < 256; val += BLOCK_KN_SIZE / 8) {
484
+ deq2[val][off] = __halves2half2(
485
+ __int2half_rn(val & 0xF), __int2half_rn(val >> 4)
486
+ );
487
+ }
488
+
489
+ if (blockIdx.z == 0)
490
+ {
491
+ for (int m = 0; m < b_end; m++)
492
+ mul[(b + m) * width + w] = __int2half_rn(0);
493
+ }
494
+ __syncthreads();
495
+
496
+ int i = width * h + w;
497
+ int g_h = h * 8;
498
+ int k = 0;
499
+ int z_w = w / 8;
500
+ int z_mod = (w % 8) * 4;
501
+ half2 res2;
502
+ half res[BLOCK_M_SIZE_MAX] = {};
503
+
504
+ unsigned int tmp;
505
+ while (k < h_end) {
506
+ tmp = mat[i];
507
+ half2 scales_tmp[4];
508
+ half2 zeros_tmp[4];
509
+ for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
510
+ int g = g_idx[g_h + (k + tmp_k) * 2];
511
+ int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
512
+ half scale_f = scales[g * width + w];
513
+ half scale_f2 = scales[g2 * width + w];
514
+ half2 scale = __halves2half2(scale_f, scale_f2);
515
+ half2 zero = __halves2half2(
516
+ __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) - 1)),
517
+ __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1))
518
+ );
519
+ scales_tmp[tmp_k] = scale;
520
+ zeros_tmp[tmp_k] = zero;
521
+ }
522
+ for (int m = 0; m < b_end; m++) {
523
+ res2 = {};
524
+ res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
525
+ res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
526
+ res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2);
527
+ res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2);
528
+ res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
529
+ }
530
+ i += width;
531
+ k += 4;
532
+ }
533
+ for (int m = 0; m < b_end; m++) {
534
+ atomicAdd(&mul[(b + m) * width + w], res[m]);
535
+ }
536
+}
537
+
538
+
539
+void gemm_half_q_half_alt
540
+(
541
+ const half* a,
542
+ const uint32_t* b_q_weight,
543
+ const uint32_t* b_gptq_qzeros,
544
+ const half* b_gptq_scales,
545
+ const int* b_g_idx,
546
+ half* c,
547
+ int size_m,
548
+ int size_n,
549
+ int size_k
550
+)
551
+{
552
+ dim3 blockDim, gridDim;
553
+ blockDim.x = BLOCK_KN_SIZE;
554
+ blockDim.y = 1;
555
+ blockDim.z = 1;
556
+ gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
557
+ gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
558
+ gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
559
+
560
+ gemm_half_q_half_alt_kernel<<<gridDim, blockDim>>>
561
+ (
562
+ (const half2*) a,
563
+ b_q_weight,
564
+ c,
565
+ b_gptq_scales,
566
+ b_gptq_qzeros,
567
+ b_g_idx,
568
+ size_m,
569
+ size_k / 8,
570
+ size_n
571
+ );
572
+}
573
+
574
+
575
+__global__ void reconstruct_gptq_kernel
576
+(
577
+ const uint32_t* __restrict__ w,
578
+ const half* __restrict__ w_scales,
579
+ const uint32_t* __restrict__ w_zeros,
580
+ const int* __restrict__ g_idx,
581
+ const int height,
582
+ const int width,
583
+ const int group,
584
+ half* __restrict__ out
585
+)
586
+{
587
+ // Start of block
588
+
589
+ int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
590
+ int row = blockIdx.y * 8;
591
+ if (column >= width) return;
592
+
593
+ // Views
594
+
595
+ MatrixView_q4_column w_(w, height, width);
596
+ MatrixView_half_rw out_(out, height, width);
597
+ MatrixView_half w_scales_(w_scales, group, width);
598
+ MatrixView_q4_row w_zeros_(w_zeros, group, width);
599
+
600
+ uint32_t w_read = w_.item_uint32_t(row, column);
601
+ half* out_ptr = out_.item_ptr(row, column);
602
+
603
+ #pragma unroll
604
+ for (int s = 0; s < 32; s += 4)
605
+ {
606
+ int group = g_idx[row + s / 4];
607
+ half w_scale = w_scales_.item(group, column);
608
+ uint32_t w_zero = w_zeros_.item(group, column) + 1;
609
+ half w_item = __hmul(__int2half_rn((int)((w_read >> s) & 0x0f) - w_zero), w_scale);
610
+ *out_ptr = w_item; out_ptr += out_.width;
611
+ }
612
+}
613
+
614
+
615
+void reconstruct_gptq
616
+(
617
+ const uint32_t* b_q_weight,
618
+ const uint32_t* b_gptq_qzeros,
619
+ const half* b_gptq_scales,
620
+ const int* b_g_idx,
621
+ half* out,
622
+ int height,
623
+ int width,
624
+ int groups
625
+)
626
+{
627
+ dim3 blockDim, gridDim;
628
+ blockDim.x = BLOCK_KN_SIZE;
629
+ blockDim.y = 1;
630
+ gridDim.y = DIVIDE(height, 8);
631
+ gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
632
+ reconstruct_gptq_kernel<<<gridDim, blockDim>>>
633
+ (
634
+ b_q_weight,
635
+ b_gptq_scales,
636
+ b_gptq_qzeros,
637
+ b_g_idx,
638
+ height,
639
+ width,
640
+ groups,
641
+ out
642
+ );
643
+}
644
+
645
+
646
+void gemm_half_q_half_cuda
647
+(
648
+ cublasHandle_t cublas_handle,
649
+ const half* a,
650
+ const uint32_t* b_q_weight,
651
+ const uint32_t* b_gptq_qzeros,
652
+ const half* b_gptq_scales,
653
+ const int* b_g_idx,
654
+ half* c,
655
+ half* temp_dq,
656
+ int size_m,
657
+ int size_n,
658
+ int size_k,
659
+ int groups,
660
+ bool use_exllama
661
+)
662
+{
663
+ if ((use_exllama && size_m > MAX_Q_GEMM_ROWS) || (!use_exllama && size_m > MAX_ALT_GEMM_ROWS)) {
664
+ // Reconstruct FP16 matrix, then cuBLAS
665
+ if (use_exllama) {
666
+ reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq,
667
+ size_k, size_n, groups);
668
+ }
669
+ else
670
+ {
671
+ reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
672
+ temp_dq, size_k, size_n, groups);
673
+ }
674
+
675
+ const half alpha = __float2half(1.0f);
676
+ const half beta = __float2half(0.0f);
677
+ cublasHgemm(cublas_handle,
678
+ CUBLAS_OP_N,
679
+ CUBLAS_OP_N,
680
+ size_n, size_m, size_k,
681
+ &alpha, temp_dq, size_n,
682
+ a, size_k,
683
+ &beta, c, size_n);
684
+ }
685
+ else if (use_exllama)
686
+ {
687
+ // Quantized matmul
688
+ int max_chunks = size_m / BLOCK_M_SIZE_MAX;
689
+ int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
690
+ int last_chunk_size = size_m - last_chunk;
691
+
692
+ if (max_chunks)
693
+ {
694
+ gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
695
+ c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
696
+ groups);
697
+ }
698
+
699
+ if (last_chunk_size)
700
+ {
701
+ gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight, b_gptq_qzeros,
702
+ b_gptq_scales, b_g_idx, c + last_chunk * size_n,
703
+ last_chunk_size, size_n, size_k, last_chunk_size,
704
+ groups);
705
+ }
706
+ }
707
+ else
708
+ {
709
+ gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
710
+ c, size_m, size_n, size_k);
711
+ }
712
+}
713
+
714
+
715
+__global__ void shuffle_kernel
716
+(
717
+ uint32_t* __restrict__ b_q_weight,
718
+ const int size_k,
719
+ const int size_n
720
+)
721
+{
722
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
723
+ if (n >= size_n) return;
724
+ int k = 0;
725
+ uint32_t* b_ptr = b_q_weight + n;
726
+ while (k < size_k) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
727
+}
728
+
729
+
730
+__global__ void make_sequential_kernel
731
+(
732
+ const uint32_t* __restrict__ w,
733
+ uint32_t* __restrict__ w_new,
734
+ const int* __restrict__ q_perm,
735
+ const int w_height,
736
+ const int w_width
737
+)
738
+{
739
+ const uint64_t* w2 = (uint64_t*) w;
740
+ uint64_t* w_new2 = (uint64_t*) w_new;
741
+ int w2_stride = w_width >> 1;
742
+ int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
743
+ if (w2_column >= w2_stride) return;
744
+ int w_new2_row = blockIdx.y;
745
+ int q_perm_idx = w_new2_row << 3;
746
+ uint64_t dst = 0;
747
+
748
+ #pragma unroll
749
+ for (int i = 0; i < 8; i++)
750
+ {
751
+ int source_row = q_perm[q_perm_idx++];
752
+
753
+ int w2_row = source_row >> 3;
754
+ int w2_subrow = source_row & 0x07;
755
+ int w2_row_shift = w2_subrow << 2;
756
+ int wnew2_row_shift = i << 2;
757
+
758
+ uint64_t src = w2[w2_row * w2_stride + w2_column];
759
+ src >>= w2_row_shift;
760
+ src &= 0x0000000f0000000f;
761
+ src <<= wnew2_row_shift;
762
+ dst |= src;
763
+ }
764
+ w_new2[w_new2_row * w2_stride + w2_column] = dst;
765
+}
766
+
767
+
768
+void shuffle_exllama_weight
769
+(
770
+ uint32_t* q_weight,
771
+ int* q_perm,
772
+ int height,
773
+ int width
774
+)
775
+{
776
+ if (q_perm)
777
+ {
778
+ uint32_t* new_qweight = NULL;
779
+ cudaMalloc(&new_qweight, height / 8 * width * sizeof(uint32_t));
780
+
781
+ dim3 blockDim, gridDim;
782
+ blockDim.x = THREADS_X;
783
+ blockDim.y = 1;
784
+ gridDim.x = DIVIDE(width, THREADS_X);
785
+ gridDim.y = height / 8;
786
+
787
+ make_sequential_kernel<<<gridDim, blockDim>>>
788
+ (
789
+ q_weight,
790
+ new_qweight,
791
+ q_perm,
792
+ height / 8,
793
+ width
794
+ );
795
+ // Replace qweights
796
+ cudaMemcpyAsync(q_weight, new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
797
+ // Cleanup
798
+ cudaDeviceSynchronize();
799
+ cudaFree(new_qweight);
800
+ }
801
+ dim3 blockDim, gridDim;
802
+ blockDim.x = THREADS_X;
803
+ blockDim.y = 1;
804
+ gridDim.x = DIVIDE(width, THREADS_X);
805
+ gridDim.y = 1;
806
+ shuffle_kernel<<<gridDim, blockDim>>>(q_weight, height, width);
807
+}
808
+
809
+} // namespace gptq
810
+} // namespace vllm
811
+
812
+torch::Tensor gptq_gemm
813
+(
814
+ torch::Tensor a,
815
+ torch::Tensor b_q_weight,
816
+ torch::Tensor b_gptq_qzeros,
817
+ torch::Tensor b_gptq_scales,
818
+ torch::Tensor b_g_idx,
819
+ bool use_exllama
820
+)
821
+{
822
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
823
+ auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
824
+ at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
825
+ at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 8, b_q_weight.size(1)}, options);
826
+
827
+ vllm::gptq::gemm_half_q_half_cuda
828
+ (
829
+ at::cuda::getCurrentCUDABlasHandle(),
830
+ (const half*) a.data_ptr(),
831
+ (const uint32_t*) b_q_weight.data_ptr(),
832
+ (const uint32_t*)b_gptq_qzeros.data_ptr(),
833
+ (const half*) b_gptq_scales.data_ptr(),
834
+ b_g_idx.device().is_meta() ? NULL : (const int*) b_g_idx.data_ptr(),
835
+ (half*) c.data_ptr(),
836
+ (half*) temp_dq.data_ptr(),
837
+ c.size(0), // m
838
+ c.size(1), // n
839
+ a.size(1), // k
840
+ b_gptq_qzeros.size(0), // group number
841
+ use_exllama
842
+ );
843
+ return c;
844
+}
845
+
846
+void gptq_shuffle
847
+(
848
+ torch::Tensor q_weight,
849
+ torch::Tensor q_perm
850
+)
851
+{
852
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
853
+ vllm::gptq::shuffle_exllama_weight(
854
+ (uint32_t*) q_weight.data_ptr(),
855
+ q_perm.device().is_meta() ? NULL : (int*) q_perm.data_ptr(),
856
+ q_weight.size(0) * 8,
857
+ q_weight.size(1)
858
+ );
859
+}
csrc/quantization/gptq/qdq_4.cuh
new
+235
@@ -0,0 +1,235 @@
1
+/*
2
+Copied from https://github.com/turboderp/exllamav2
3
+*/
4
+
5
+#ifndef _qdq_4_cuh
6
+#define _qdq_4_cuh
7
+
8
+#include "qdq_util.cuh"
9
+
10
+namespace vllm {
11
+namespace gptq {
12
+// Permutation:
13
+//
14
+// 77775555 33331111 66664444 22220000
15
+
16
+__forceinline__ __device__ void shuffle_4bit_8
17
+(
18
+ uint32_t* q,
19
+ int stride
20
+)
21
+{
22
+ uint32_t qa = q[0];
23
+ uint32_t qb = 0;
24
+
25
+ #pragma unroll
26
+ for (int i = 0; i < 4; i++)
27
+ {
28
+ uint32_t qa0 = qa & 0x0f;
29
+ uint32_t qa1 = (qa & 0xf0) >> 4;
30
+ qa >>= 8;
31
+ qb |= (qa1 << (i * 4 + 16));
32
+ qb |= (qa0 << (i * 4));
33
+ }
34
+ q[0] = qb;
35
+}
36
+
37
+__forceinline__ __device__ void dequant_4bit_8
38
+(
39
+ const uint32_t q_0,
40
+ half2 (&dq)[4],
41
+ int stride
42
+)
43
+{
44
+ const uint32_t c0 = 0x64006400;
45
+ const half y16_ = __float2half_rn(1.0f / 16.0f);
46
+ const half2 y16 = __halves2half2(y16_, y16_);
47
+ const half z1_ = __float2half_rn(-1024.0f - 8.0f);
48
+ const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);
49
+ const half2 z1 = __halves2half2(z1_, z1_);
50
+ const half2 z16 = __halves2half2(z16_, z16_);
51
+
52
+ uint32_t qa = q_0;
53
+ half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
54
+ half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
55
+ qa >>= 8;
56
+ half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
57
+ half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
58
+
59
+ dq[0] = __hadd2(q0.as_half2, z1);
60
+ dq[1] = __hfma2(q1.as_half2, y16, z16);
61
+ dq[2] = __hadd2(q2.as_half2, z1);
62
+ dq[3] = __hfma2(q3.as_half2, y16, z16);
63
+}
64
+
65
+__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
66
+(
67
+ const uint32_t zero,
68
+ const half scale,
69
+ half2 (&z1z16)[2],
70
+ half2 (&y1y16)[2]
71
+)
72
+{
73
+ half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
74
+ half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
75
+
76
+ half2 scale2 = __half2half2(scale);
77
+
78
+ z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
79
+ z1z16[1] = __hmul2(scale2, __half2half2(z16));
80
+
81
+ const half y1 = __float2half_rn(1.0f);
82
+ const half y16 = __float2half_rn(1.0f / 16.0f);
83
+
84
+ y1y16[0] = __hmul2(scale2, __half2half2(y1));
85
+ y1y16[1] = __hmul2(scale2, __half2half2(y16));
86
+}
87
+
88
+__forceinline__ __device__ void dequant_4bit_8_prep_zero
89
+(
90
+ const uint32_t zero,
91
+ half2(&z1z16)[2],
92
+ half2(&y1y16)[2]
93
+)
94
+{
95
+ half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
96
+ half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
97
+
98
+ z1z16[0] = __half2half2(z1.as_half);
99
+ z1z16[1] = __half2half2(z16);
100
+
101
+ const half y1 = __float2half_rn(1.0f);
102
+ const half y16 = __float2half_rn(1.0f / 16.0f);
103
+
104
+ y1y16[0] = __half2half2(y1);
105
+ y1y16[1] = __half2half2(y16);
106
+}
107
+
108
+
109
+__forceinline__ __device__ void dequant_4bit_8_gptq
110
+(
111
+ const uint32_t q_0,
112
+ half2 (&dq)[4],
113
+ half2 (&z1z16)[2],
114
+ half2 (&y1y16)[2],
115
+ int stride,
116
+ bool scaled
117
+)
118
+{
119
+ const uint32_t c0 = 0x64006400;
120
+
121
+ uint32_t qa = q_0;
122
+ half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 )
123
+ half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
124
+ qa >>= 8;
125
+ half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 )
126
+ half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
127
+
128
+ if (scaled)
129
+ {
130
+ dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
131
+ dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
132
+ dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
133
+ dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
134
+ }
135
+ else
136
+ {
137
+ dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
138
+ dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z )
139
+ dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
140
+ dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z )
141
+ }
142
+}
143
+} // namespace gptq
144
+} // namespace vllm
145
+
146
+#else
147
+
148
+namespace vllm {
149
+namespace gptq {
150
+__forceinline__ __device__ void shuffle_4bit_8
151
+(
152
+ uint32_t* q,
153
+ int stride
154
+)
155
+{
156
+}
157
+
158
+__forceinline__ __device__ void dequant_4bit_8
159
+(
160
+ const uint32_t q_0,
161
+ half2 (&dq)[4],
162
+ int stride
163
+)
164
+{
165
+ half dqh[8];
166
+ for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);
167
+
168
+ for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
169
+}
170
+
171
+__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
172
+(
173
+ const uint32_t zero,
174
+ const half scale,
175
+ half2 (&z1)[2],
176
+ half2 (&y1)[2]
177
+)
178
+{
179
+ half z = __int2half_rn(-((int)zero));
180
+ z = __hmul(z, scale);
181
+ z1[0] = __half2half2(z);
182
+ y1[0] = __half2half2(scale);
183
+}
184
+
185
+__forceinline__ __device__ void dequant_4bit_8_prep_zero
186
+(
187
+ const uint32_t zero,
188
+ half2(&z1)[2],
189
+ half2(&y1)[2]
190
+)
191
+{
192
+ half z = __int2half_rn(-((int)zero));
193
+ z1[0] = __half2half2(z);
194
+}
195
+
196
+__forceinline__ __device__ void dequant_4bit_8_gptq
197
+(
198
+ const uint32_t q_0,
199
+ half2 (&dq)[4],
200
+ half2 (&z1)[2],
201
+ half2 (&y1)[2],
202
+ int stride,
203
+ bool scaled
204
+)
205
+{
206
+ half2 dqh2[8];
207
+
208
+ uint32_t qa = q_0;
209
+ for (int i = 0; i < 4; i++)
210
+ {
211
+ half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;
212
+ half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;
213
+ dqh2[i] = __halves2half2(d0, d1);
214
+ }
215
+
216
+ if (scaled)
217
+ {
218
+ dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);
219
+ dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);
220
+ dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);
221
+ dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);
222
+ }
223
+ else
224
+ {
225
+ dq[0] = __hadd2(dqh2[0], z1[0]);
226
+ dq[1] = __hadd2(dqh2[1], z1[0]);
227
+ dq[2] = __hadd2(dqh2[2], z1[0]);
228
+ dq[3] = __hadd2(dqh2[3], z1[0]);
229
+ }
230
+}
231
+
232
+} // namespace gptq
233
+} // namespace vllm
234
+
235
+#endif
csrc/quantization/gptq/qdq_util.cuh
new
+60
@@ -0,0 +1,60 @@
1
+/*
2
+Copied from https://github.com/turboderp/exllamav2
3
+*/
4
+
5
+#ifndef _qdq_util_cuh
6
+#define _qdq_util_cuh
7
+
8
+namespace vllm {
9
+namespace gptq {
10
+
11
+union half2_uint32
12
+{
13
+ uint32_t as_uint32;
14
+ half2 as_half2;
15
+ __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
16
+ __device__ half2_uint32(half2 val) : as_half2(val) {}
17
+};
18
+
19
+union half_uint16
20
+{
21
+ uint16_t as_uint16;
22
+ half as_half;
23
+ __device__ half_uint16(uint16_t val) : as_uint16(val) {}
24
+ __device__ half_uint16(half val) : as_half(val) {}
25
+};
26
+
27
+// Max_scale premultiplied by 1/256
28
+
29
+__forceinline__ __device__ half dq_scale(const int qs, const half max_scale)
30
+{
31
+ int qs_i = qs + 1;
32
+ half qs_h = __int2half_rn(qs_i * qs_i);
33
+ qs_h = __hmul(qs_h, max_scale);
34
+ return qs_h;
35
+}
36
+
37
+__forceinline__ __device__ half dq(const int q, const int qzero, const half scale)
38
+{
39
+ return __hmul(__int2half_rn(q - qzero), scale);
40
+}
41
+
42
+__forceinline__ __device__ half dq_ns(const int q, const int qzero)
43
+{
44
+ //return __hsub(__int2half_rn(q), __int2half_rn(qzero));
45
+ return __int2half_rn(q - qzero);
46
+}
47
+
48
+__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask)
49
+{
50
+ return (int)((q >> shift) & mask);
51
+}
52
+
53
+__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask)
54
+{
55
+ return (int)(__funnelshift_rc(q0, q1, shift) & mask);
56
+}
57
+
58
+} // namespace gptq
59
+} // namespace vllm
60
+#endif
setup.py
+1
@@ -219,6 +219,7 @@ vllm_extension_sources = [
219
"csrc/activation_kernels.cu",
220
"csrc/layernorm_kernels.cu",
221
"csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222
+ "csrc/quantization/gptq/q_gemm.cu",
223
"csrc/cuda_utils_kernels.cu",
224
"csrc/pybind.cpp",
225
]
vllm/config.py
+1
-1
@@ -142,7 +142,7 @@ class ModelConfig:
142
self.tokenizer_mode = tokenizer_mode
143
144
def _verify_quantization(self) -> None:
145
- supported_quantization = ["awq", "squeezellm"]
145
+ supported_quantization = ["awq", "gptq", "squeezellm"]
146
rocm_not_supported_quantization = ["awq"]
147
if self.quantization is not None:
148
self.quantization = self.quantization.lower()
vllm/engine/arg_utils.py
+1
-1
@@ -179,7 +179,7 @@ class EngineArgs:
179
parser.add_argument('--quantization',
180
'-q',
181
type=str,
182
- choices=['awq', 'squeezellm', None],
182
+ choices=['awq', 'gptq', 'squeezellm', None],
183
default=None,
184
help='Method used to quantize the weights')
185
return parser
vllm/entrypoints/llm.py
+3
-2
@@ -38,8 +38,9 @@ class LLM:
38
However, if the `torch_dtype` in the config is `float32`, we will
39
use `float16` instead.
40
quantization: The method used to quantize the model weights. Currently,
41
- we support "awq". If None, we assume the model weights are not
42
- quantized and use `dtype` to determine the data type of the weights.
41
+ we support "awq", "gptq" and "squeezellm". If None, we assume the
42
+ model weights are not quantized and use `dtype` to determine the
43
+ data type of the weights.
44
revision: The specific model version to use. It can be a branch name,
45
a tag name, or a commit id.
46
tokenizer_revision: The specific tokenizer version to use. It can be a
vllm/model_executor/layers/linear.py
+37
-23
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, List, Optional
2
+from typing import Any, Dict, List, Optional
3
4
import torch
5
import torch.nn.functional as F
@@ -21,8 +21,10 @@ class LinearMethodBase(ABC):
21
"""Base class for different (maybe quantized) linear methods."""
22
23
@abstractmethod
24
- def create_weights(self, input_size: int, output_size: int,
25
- params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
24
+ def create_weights(self, input_size_per_partition: int,
25
+ output_size_per_partition: int, input_size: int,
26
+ output_size: int,
27
+ params_dtype: torch.dtype) -> Dict[str, Any]:
28
"""Create weights for a linear layer."""
29
raise NotImplementedError
30
@@ -46,10 +48,12 @@ class UnquantizedLinearMethod(LinearMethodBase):
48
def __init__(self, separate_bias_add: bool = False):
49
self.separate_bias_add = separate_bias_add
50
49
- def create_weights(self, input_size: int, output_size: int,
50
- params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
51
- weight = Parameter(torch.empty(output_size,
52
- input_size,
51
+ def create_weights(self, input_size_per_partition: int,
52
+ output_size_per_partition: int, input_size: int,
53
+ output_size: int,
54
+ params_dtype: torch.dtype) -> Dict[str, Any]:
55
+ weight = Parameter(torch.empty(output_size_per_partition,
56
+ input_size_per_partition,
57
device=torch.cuda.current_device(),
58
dtype=params_dtype),
59
requires_grad=False)
@@ -102,9 +106,11 @@ class ReplicatedLinear(torch.nn.Module):
106
linear_method = UnquantizedLinearMethod()
107
self.linear_method = linear_method
108
self.linear_weights = self.linear_method.create_weights(
105
- self.input_size, self.output_size, self.params_dtype)
109
+ self.input_size, self.output_size, self.input_size,
110
+ self.output_size, self.params_dtype)
111
for name, weight in self.linear_weights.items():
107
- self.register_parameter(name, weight)
112
+ if isinstance(weight, torch.Tensor):
113
+ self.register_parameter(name, weight)
114
if bias:
115
self.bias = Parameter(
116
torch.empty(self.output_size,
@@ -168,10 +174,12 @@ class ColumnParallelLinear(torch.nn.Module):
174
linear_method = UnquantizedLinearMethod()
175
self.linear_method = linear_method
176
self.linear_weights = self.linear_method.create_weights(
171
- self.input_size, self.output_size_per_partition, self.params_dtype)
177
+ self.input_size, self.output_size_per_partition, self.input_size,
178
+ self.output_size, self.params_dtype)
179
for name, weight in self.linear_weights.items():
173
- self.register_parameter(name, weight)
174
- set_weight_attrs(weight, {"weight_loader": self.weight_loader})
180
+ if isinstance(weight, torch.Tensor):
181
+ self.register_parameter(name, weight)
182
+ set_weight_attrs(weight, {"weight_loader": self.weight_loader})
183
if bias:
184
self.bias = Parameter(
185
torch.empty(self.output_size_per_partition,
@@ -295,10 +303,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
303
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
304
shard_size)
305
else:
298
- logger.warning(
299
- "Loading a weight without `output_dim` attribute in "
300
- "MergedColumnParallelLinear, assume the weight is "
301
- "the same for all partitions.")
306
+ ignore_warning = getattr(param, "ignore_warning", False)
307
+ if not ignore_warning:
308
+ logger.warning(
309
+ "Loading a weight without `output_dim` attribute in "
310
+ "MergedColumnParallelLinear, assume the weight is "
311
+ "the same for all partitions.")
312
assert param_data.shape == loaded_weight.shape
313
param_data.copy_(loaded_weight)
314
@@ -418,10 +428,12 @@ class QKVParallelLinear(ColumnParallelLinear):
428
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
429
shard_size)
430
else:
421
- logger.warning(
422
- "Loading a weight without `output_dim` attribute in "
423
- "QKVParallelLinear, assume the weight is the same "
424
- "for all partitions.")
431
+ ignore_warning = getattr(param, "ignore_warning", False)
432
+ if not ignore_warning:
433
+ logger.warning(
434
+ "Loading a weight without `output_dim` attribute in "
435
+ "QKVParallelLinear, assume the weight is the same "
436
+ "for all partitions.")
437
assert param_data.shape == loaded_weight.shape
438
param_data.copy_(loaded_weight)
439
@@ -481,10 +493,12 @@ class RowParallelLinear(torch.nn.Module):
493
linear_method = UnquantizedLinearMethod()
494
self.linear_method = linear_method
495
self.linear_weights = self.linear_method.create_weights(
484
- self.input_size_per_partition, self.output_size, self.params_dtype)
496
+ self.input_size_per_partition, self.output_size, self.input_size,
497
+ self.output_size, self.params_dtype)
498
for name, weight in self.linear_weights.items():
486
- self.register_parameter(name, weight)
487
- set_weight_attrs(weight, {"weight_loader": self.weight_loader})
499
+ if isinstance(weight, torch.Tensor):
500
+ self.register_parameter(name, weight)
501
+ set_weight_attrs(weight, {"weight_loader": self.weight_loader})
502
503
if not reduce_results and (bias and not skip_bias_add):
504
raise ValueError("When not reduce the results, adding bias to the "
vllm/model_executor/layers/quantization/__init__.py
+3
-1
@@ -1,11 +1,13 @@
1
from typing import Type
2
3
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
+from vllm.model_executor.layers.quantization.gptq import GPTQConfig
6
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
5
-from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
7
8
_QUANTIZATION_CONFIG_REGISTRY = {
9
"awq": AWQConfig,
10
+ "gptq": GPTQConfig,
11
"squeezellm": SqueezeLLMConfig,
12
}
13
vllm/model_executor/layers/quantization/awq.py
+13
-11
@@ -77,14 +77,16 @@ class AWQLinearMethod(LinearMethodBase):
77
def __init__(self, quant_config: AWQConfig):
78
self.quant_config = quant_config
79
80
- def create_weights(self, input_size: int, output_size: int,
81
- params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
82
- if input_size % self.quant_config.group_size != 0:
80
+ def create_weights(self, input_size_per_partition: int,
81
+ output_size_per_partition: int, input_size: int,
82
+ output_size: int,
83
+ params_dtype: torch.dtype) -> Dict[str, Any]:
84
+ if input_size_per_partition % self.quant_config.group_size != 0:
85
raise ValueError(
86
"The input size is not aligned with the quantized "
87
"weight shape. This can be caused by too large "
88
"tensor parallel size.")
87
- if output_size % self.quant_config.pack_factor != 0:
89
+ if output_size_per_partition % self.quant_config.pack_factor != 0:
90
raise ValueError(
91
"The output size is not aligned with the quantized "
92
"weight shape. This can be caused by too large "
@@ -92,8 +94,8 @@ class AWQLinearMethod(LinearMethodBase):
94
95
qweight = Parameter(
96
torch.empty(
95
- input_size,
96
- output_size // self.quant_config.pack_factor,
97
+ input_size_per_partition,
98
+ output_size_per_partition // self.quant_config.pack_factor,
99
device="cuda",
100
dtype=torch.int32,
101
),
@@ -108,8 +110,8 @@ class AWQLinearMethod(LinearMethodBase):
110
})
111
qzeros = Parameter(
112
torch.empty(
111
- input_size // self.quant_config.group_size,
112
- output_size // self.quant_config.pack_factor,
113
+ input_size_per_partition // self.quant_config.group_size,
114
+ output_size_per_partition // self.quant_config.pack_factor,
115
device="cuda",
116
dtype=torch.int32,
117
),
@@ -124,8 +126,8 @@ class AWQLinearMethod(LinearMethodBase):
126
})
127
scales = Parameter(
128
torch.empty(
127
- input_size // self.quant_config.group_size,
128
- output_size,
129
+ input_size_per_partition // self.quant_config.group_size,
130
+ output_size_per_partition,
131
device="cuda",
132
dtype=params_dtype,
133
),
@@ -142,7 +144,7 @@ class AWQLinearMethod(LinearMethodBase):
144
}
145
146
def apply_weights(self,
145
- weights: Dict[str, torch.Tensor],
147
+ weights: Dict[str, Any],
148
x: torch.Tensor,
149
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
150
qweight = weights["qweight"]
vllm/model_executor/layers/quantization/gptq.py
new
+215
@@ -0,0 +1,215 @@
1
+import enum
2
+from enum import Enum
3
+from typing import Any, Dict, List, Optional
4
+
5
+import torch
6
+from torch.nn.parameter import Parameter
7
+
8
+from vllm._C import ops
9
+from vllm.model_executor.layers.linear import (LinearMethodBase,
10
+ set_weight_attrs)
11
+from vllm.model_executor.layers.quantization.base_config import (
12
+ QuantizationConfig)
13
+
14
+
15
+class GPTQConfig(QuantizationConfig):
16
+ """Config class for GPTQ.
17
+
18
+ Reference: https://arxiv.org/abs/2210.17323
19
+ """
20
+
21
+ def __init__(
22
+ self,
23
+ weight_bits: int,
24
+ group_size: int,
25
+ desc_act: bool,
26
+ ) -> None:
27
+ self.weight_bits = weight_bits
28
+ self.group_size = group_size
29
+ self.desc_act = desc_act
30
+ self.pack_factor = 32 // self.weight_bits
31
+ # exllama kernel v1 only supports 4 bit
32
+ if self.weight_bits != 4:
33
+ raise ValueError(
34
+ "Currently, only 4-bit weight quantization is supported for "
35
+ f"GPTQ, but got {self.weight_bits} bits.")
36
+
37
+ def __repr__(self) -> str:
38
+ return (f"GPTQConfig(weight_bits={self.weight_bits}, "
39
+ f"group_size={self.group_size}, "
40
+ f"desc_act={self.desc_act})")
41
+
42
+ @classmethod
43
+ def get_name(cls) -> str:
44
+ return "gptq"
45
+
46
+ @classmethod
47
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
48
+ return [torch.half]
49
+
50
+ @classmethod
51
+ # Need to figure it out
52
+ def get_min_capability(cls) -> int:
53
+ return 60
54
+
55
+ @classmethod
56
+ def get_config_filenames(cls) -> List[str]:
57
+ return ["quantize_config.json"]
58
+
59
+ @classmethod
60
+ def from_config(cls, config: Dict[str, Any]) -> "GPTQConfig":
61
+ weight_bits = cls.get_from_keys(config, ["bits"])
62
+ group_size = cls.get_from_keys(config, ["group_size"])
63
+ desc_act = cls.get_from_keys(config, ["desc_act"])
64
+ return cls(weight_bits, group_size, desc_act)
65
+
66
+ def get_linear_method(self) -> "GPTQLinearMethod":
67
+ return GPTQLinearMethod(self)
68
+
69
+ def get_scaled_act_names(self) -> List[str]:
70
+ return []
71
+
72
+
73
+class ExllamaState(Enum):
74
+
75
+ UNUSED = enum.auto()
76
+ UNINITIALIZED = enum.auto()
77
+ READY = enum.auto()
78
+
79
+
80
+class GPTQLinearMethod(LinearMethodBase):
81
+ """Linear method for GPTQ.
82
+
83
+ Args:
84
+ quant_config: The GPTQ quantization config.
85
+ """
86
+
87
+ def __init__(self, quant_config: GPTQConfig):
88
+ self.quant_config = quant_config
89
+
90
+ def create_weights(
91
+ self,
92
+ input_size_per_partition: int,
93
+ output_size_per_partition: int,
94
+ input_size: int,
95
+ output_size: int,
96
+ params_dtype: torch.dtype,
97
+ ) -> Dict[str, Any]:
98
+ del output_size # Unused.
99
+ if input_size_per_partition % self.quant_config.group_size != 0:
100
+ raise ValueError(
101
+ "The input size is not aligned with the quantized "
102
+ "weight shape. This can be caused by too large "
103
+ "tensor parallel size.")
104
+ if output_size_per_partition % self.quant_config.pack_factor != 0:
105
+ raise ValueError(
106
+ "The output size is not aligned with the quantized "
107
+ "weight shape. This can be caused by too large "
108
+ "tensor parallel size.")
109
+
110
+ if self.quant_config.group_size != -1:
111
+ group_size = self.quant_config.group_size
112
+ else:
113
+ group_size = input_size
114
+ exllama_state = ExllamaState.UNINITIALIZED
115
+ scale_and_zero_size = input_size // group_size
116
+ scale_and_zero_input_dim = None
117
+ if input_size != input_size_per_partition and self.quant_config.group_size != -1:
118
+ # For act-order models, we cannot use Exllama for row parallel layer
119
+ if self.quant_config.desc_act:
120
+ exllama_state = ExllamaState.UNUSED
121
+ else:
122
+ # we need to partition qzeros and scales for exllama kernel
123
+ scale_and_zero_size = input_size_per_partition // group_size
124
+ scale_and_zero_input_dim = 0
125
+
126
+ qweight = Parameter(
127
+ torch.empty(
128
+ input_size_per_partition // self.quant_config.pack_factor,
129
+ output_size_per_partition,
130
+ device="cuda",
131
+ dtype=torch.int32,
132
+ ),
133
+ requires_grad=False,
134
+ )
135
+ set_weight_attrs(
136
+ qweight, {
137
+ "input_dim": 0,
138
+ "output_dim": 1,
139
+ "packed_dim": 0,
140
+ "pack_factor": self.quant_config.pack_factor,
141
+ })
142
+ g_idx = Parameter(
143
+ torch.tensor(
144
+ [
145
+ i // self.quant_config.group_size
146
+ for i in range(input_size_per_partition)
147
+ ],
148
+ device="cuda",
149
+ dtype=torch.int32,
150
+ ),
151
+ requires_grad=False,
152
+ )
153
+ # Ignore warning from fused linear layers such as QKVParallelLinear.
154
+ set_weight_attrs(g_idx, {"input_dim": 0, "ignore_warning": True})
155
+ qzeros = Parameter(
156
+ torch.empty(
157
+ scale_and_zero_size,
158
+ output_size_per_partition // self.quant_config.pack_factor,
159
+ device="cuda",
160
+ dtype=torch.int32,
161
+ ),
162
+ requires_grad=False,
163
+ )
164
+ set_weight_attrs(
165
+ qzeros, {
166
+ "input_dim": scale_and_zero_input_dim,
167
+ "output_dim": 1,
168
+ "packed_dim": 1,
169
+ "pack_factor": self.quant_config.pack_factor,
170
+ })
171
+ scales = Parameter(
172
+ torch.empty(
173
+ scale_and_zero_size,
174
+ output_size_per_partition,
175
+ device="cuda",
176
+ dtype=params_dtype,
177
+ ),
178
+ requires_grad=False,
179
+ )
180
+ set_weight_attrs(scales, {
181
+ "input_dim": scale_and_zero_input_dim,
182
+ "output_dim": 1,
183
+ })
184
+ return {
185
+ "qweight": qweight,
186
+ "g_idx": g_idx,
187
+ "qzeros": qzeros,
188
+ "scales": scales,
189
+ "exllama_state": exllama_state,
190
+ }
191
+
192
+ def apply_weights(self,
193
+ weights: Dict[str, Any],
194
+ x: torch.Tensor,
195
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
196
+ qweight = weights["qweight"]
197
+ out_shape = x.shape[:-1] + (qweight.shape[-1], )
198
+ reshaped_x = x.reshape(-1, x.shape[-1])
199
+ # exllama needs to shuffle the weight after the weight is loaded
200
+ # here we do the shuffle on first forward pass
201
+ if weights["exllama_state"] == ExllamaState.UNINITIALIZED:
202
+ if self.quant_config.desc_act:
203
+ weights["g_idx"] = torch.argsort(weights["g_idx"]).to(
204
+ torch.int)
205
+ else:
206
+ weights["g_idx"] = torch.empty((1, 1), device="meta")
207
+ weights["exllama_state"] = ExllamaState.READY
208
+ ops.gptq_shuffle(weights["qweight"], weights["g_idx"])
209
+ output = ops.gptq_gemm(reshaped_x, weights["qweight"],
210
+ weights["qzeros"], weights["scales"],
211
+ weights["g_idx"],
212
+ weights["exllama_state"] == ExllamaState.READY)
213
+ if bias is not None:
214
+ output = output + bias
215
+ return output.reshape(out_shape)
vllm/model_executor/layers/quantization/squeezellm.py
+8
-6
@@ -67,17 +67,19 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
67
def __init__(self, quant_config: SqueezeLLMConfig):
68
self.quant_config = quant_config
69
70
- def create_weights(self, input_size: int, output_size: int,
71
- params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
72
- if input_size % self.quant_config.pack_factor != 0:
70
+ def create_weights(self, input_size_per_partition: int,
71
+ output_size_per_partition: int, input_size: int,
72
+ output_size: int,
73
+ params_dtype: torch.dtype) -> Dict[str, Any]:
74
+ if input_size_per_partition % self.quant_config.pack_factor != 0:
75
raise ValueError(
76
"The input size is not aligned with the quantized "
77
"weight shape. This can be caused by too large "
78
"tensor parallel size.")
79
qweight = Parameter(
80
torch.empty(
79
- input_size // self.quant_config.pack_factor,
80
- output_size,
81
+ input_size_per_partition // self.quant_config.pack_factor,
82
+ output_size_per_partition,
83
device="cuda",
84
dtype=torch.int32,
85
),
@@ -108,7 +110,7 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
110
}
111
112
def apply_weights(self,
111
- weights: Dict[str, torch.Tensor],
113
+ weights: Dict[str, Any],
114
x: torch.Tensor,
115
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
116
qweight = weights["qweight"]
vllm/model_executor/models/aquila.py
+8
-1
@@ -332,11 +332,18 @@ class AquilaForCausalLM(nn.Module):
332
for (param_name, weight_name, shard_id) in stacked_params_mapping:
333
if weight_name not in name:
334
continue
335
- param = params_dict[name.replace(weight_name, param_name)]
335
+ name = name.replace(weight_name, param_name)
336
+ # Skip loading extra bias for GPTQ models.
337
+ if name.endswith(".bias") and name not in params_dict:
338
+ continue
339
+ param = params_dict[name]
340
weight_loader = param.weight_loader
341
weight_loader(param, loaded_weight, shard_id)
342
break
343
else:
344
+ # Skip loading extra bias for GPTQ models.
345
+ if name.endswith(".bias") and name not in params_dict:
346
+ continue
347
param = params_dict[name]
348
weight_loader = getattr(param, "weight_loader",
349
default_weight_loader)
vllm/model_executor/models/baichuan.py
+8
-1
@@ -355,11 +355,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
355
for (param_name, weight_name, shard_id) in stacked_params_mapping:
356
if weight_name not in name:
357
continue
358
- param = params_dict[name.replace(weight_name, param_name)]
358
+ name = name.replace(weight_name, param_name)
359
+ # Skip loading extra bias for GPTQ models.
360
+ if name.endswith(".bias") and name not in params_dict:
361
+ continue
362
+ param = params_dict[name]
363
weight_loader = param.weight_loader
364
weight_loader(param, loaded_weight, shard_id)
365
break
366
else:
367
+ # Skip loading extra bias for GPTQ models.
368
+ if name.endswith(".bias") and name not in params_dict:
369
+ continue
370
param = params_dict[name]
371
weight_loader = getattr(param, "weight_loader",
372
default_weight_loader)
vllm/model_executor/models/chatglm.py
+3
@@ -377,6 +377,9 @@ class ChatGLMForCausalLM(nn.Module):
377
continue
378
if "word_embeddings" in name:
379
name = name.replace(".word_embeddings", "")
380
+ # Skip loading extra bias for GPTQ models.
381
+ if name.endswith(".bias") and name not in params_dict:
382
+ continue
383
param = params_dict[name]
384
weight_loader = getattr(param, "weight_loader",
385
default_weight_loader)
vllm/model_executor/models/falcon.py
+22
-17
@@ -425,27 +425,32 @@ class FalconForCausalLM(nn.Module):
425
params_dict = dict(self.named_parameters())
426
for name, loaded_weight in hf_model_weights_iterator(
427
model_name_or_path, cache_dir, load_format, revision):
428
+ # Skip loading extra bias for GPTQ models.
429
+ if name.endswith(".bias") and name not in params_dict:
430
+ continue
431
param = params_dict[name]
432
if "query_key_value" in name:
433
output_dim = getattr(param, "output_dim", None)
434
loaded_weight_shape = loaded_weight.shape
432
- loaded_weight = loaded_weight.view(
433
- loaded_weight_shape[:output_dim] +
434
- (total_num_kv_heads, num_query_heads_per_kv_head + 2, -1) +
435
- loaded_weight_shape[output_dim + 1:])
436
- wq = loaded_weight.narrow(
437
- output_dim + 1, 0, num_query_heads_per_kv_head).reshape(
438
- *loaded_weight_shape[:output_dim], -1,
439
- *loaded_weight_shape[output_dim + 1:])
440
- wk = loaded_weight.narrow(
441
- output_dim + 1, num_query_heads_per_kv_head,
442
- 1).reshape(*loaded_weight_shape[:output_dim], -1,
443
- *loaded_weight_shape[output_dim + 1:])
444
- wv = loaded_weight.narrow(
445
- output_dim + 1, num_query_heads_per_kv_head + 1,
446
- 1).reshape(*loaded_weight_shape[:output_dim], -1,
447
- *loaded_weight_shape[output_dim + 1:])
448
- loaded_weight = torch.cat([wq, wk, wv], dim=output_dim)
435
+ if output_dim is not None:
436
+ loaded_weight = loaded_weight.view(
437
+ loaded_weight_shape[:output_dim] +
438
+ (total_num_kv_heads, num_query_heads_per_kv_head + 2,
439
+ -1) + loaded_weight_shape[output_dim + 1:])
440
+ wq = loaded_weight.narrow(
441
+ output_dim + 1, 0,
442
+ num_query_heads_per_kv_head).reshape(
443
+ *loaded_weight_shape[:output_dim], -1,
444
+ *loaded_weight_shape[output_dim + 1:])
445
+ wk = loaded_weight.narrow(
446
+ output_dim + 1, num_query_heads_per_kv_head,
447
+ 1).reshape(*loaded_weight_shape[:output_dim], -1,
448
+ *loaded_weight_shape[output_dim + 1:])
449
+ wv = loaded_weight.narrow(
450
+ output_dim + 1, num_query_heads_per_kv_head + 1,
451
+ 1).reshape(*loaded_weight_shape[:output_dim], -1,
452
+ *loaded_weight_shape[output_dim + 1:])
453
+ loaded_weight = torch.cat([wq, wk, wv], dim=output_dim)
454
455
weight_loader = getattr(param, "weight_loader",
456
default_weight_loader)
vllm/model_executor/models/gpt2.py
-1
@@ -275,7 +275,6 @@ class GPT2LMHeadModel(nn.Module):
275
if not name.endswith(".weight"):
276
continue
277
loaded_weight = loaded_weight.t()
278
-
278
weight_loader = getattr(param, "weight_loader",
279
default_weight_loader)
280
weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_j.py
+8
-1
@@ -274,11 +274,18 @@ class GPTJForCausalLM(nn.Module):
274
for (param_name, weight_name, shard_id) in stacked_params_mapping:
275
if weight_name not in name:
276
continue
277
- param = params_dict[name.replace(weight_name, param_name)]
277
+ name = name.replace(weight_name, param_name)
278
+ # Skip loading extra bias for GPTQ models.
279
+ if name.endswith(".bias") and name not in params_dict:
280
+ continue
281
+ param = params_dict[name]
282
weight_loader = param.weight_loader
283
weight_loader(param, loaded_weight, shard_id)
284
break
285
else:
286
+ # Skip loading extra bias for GPTQ models.
287
+ if name.endswith(".bias") and name not in params_dict:
288
+ continue
289
param = params_dict[name]
290
weight_loader = getattr(param, "weight_loader",
291
default_weight_loader)
vllm/model_executor/models/gpt_neox.py
-1
@@ -72,7 +72,6 @@ class GPTNeoXAttention(nn.Module):
72
config.hidden_size,
73
linear_method=linear_method,
74
)
75
-
75
scaling = self.head_size**-0.5
76
rotary_dim = int(self.head_size * config.rotary_pct)
77
assert rotary_dim % 2 == 0
vllm/model_executor/models/internlm.py
+8
-1
@@ -289,11 +289,18 @@ class InternLMForCausalLM(nn.Module):
289
for (param_name, weight_name, shard_id) in stacked_params_mapping:
290
if weight_name not in name:
291
continue
292
- param = params_dict[name.replace(weight_name, param_name)]
292
+ name = name.replace(weight_name, param_name)
293
+ # Skip loading extra bias for GPTQ models.
294
+ if name.endswith(".bias") and name not in params_dict:
295
+ continue
296
+ param = params_dict[name]
297
weight_loader = param.weight_loader
298
weight_loader(param, loaded_weight, shard_id)
299
break
300
else:
301
+ # Skip loading extra bias for GPTQ models.
302
+ if name.endswith(".bias") and name not in params_dict:
303
+ continue
304
param = params_dict[name]
305
weight_loader = getattr(param, "weight_loader",
306
default_weight_loader)
vllm/model_executor/models/llama.py
+8
-1
@@ -330,11 +330,18 @@ class LlamaForCausalLM(nn.Module):
330
for (param_name, weight_name, shard_id) in stacked_params_mapping:
331
if weight_name not in name:
332
continue
333
- param = params_dict[name.replace(weight_name, param_name)]
333
+ name = name.replace(weight_name, param_name)
334
+ # Skip loading extra bias for GPTQ models.
335
+ if name.endswith(".bias") and name not in params_dict:
336
+ continue
337
+ param = params_dict[name]
338
weight_loader = param.weight_loader
339
weight_loader(param, loaded_weight, shard_id)
340
break
341
else:
342
+ # Skip loading extra bias for GPTQ models.
343
+ if name.endswith(".bias") and name not in params_dict:
344
+ continue
345
param = params_dict[name]
346
weight_loader = getattr(param, "weight_loader",
347
default_weight_loader)
vllm/model_executor/models/mistral.py
+8
-1
@@ -321,11 +321,18 @@ class MistralForCausalLM(nn.Module):
321
for (param_name, weight_name, shard_id) in stacked_params_mapping:
322
if weight_name not in name:
323
continue
324
- param = params_dict[name.replace(weight_name, param_name)]
324
+ name = name.replace(weight_name, param_name)
325
+ # Skip loading extra bias for GPTQ models.
326
+ if name.endswith(".bias") and name not in params_dict:
327
+ continue
328
+ param = params_dict[name]
329
weight_loader = param.weight_loader
330
weight_loader(param, loaded_weight, shard_id)
331
break
332
else:
333
+ # Skip loading extra bias for GPTQ models.
334
+ if name.endswith(".bias") and name not in params_dict:
335
+ continue
336
param = params_dict[name]
337
weight_loader = getattr(param, "weight_loader",
338
default_weight_loader)
vllm/model_executor/models/mixtral.py
+9
-2
@@ -153,7 +153,7 @@ class MixtralMoE(nn.Module):
153
self.gate = ReplicatedLinear(config.hidden_size,
154
self.num_total_experts,
155
bias=False,
156
- linear_method=linear_method)
156
+ linear_method=None)
157
158
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
159
batch_size, sequence_length, hidden_dim = hidden_states.shape
@@ -418,11 +418,18 @@ class MixtralForCausalLM(nn.Module):
418
for (param_name, weight_name, shard_id) in stacked_params_mapping:
419
if weight_name not in name:
420
continue
421
- param = params_dict[name.replace(weight_name, param_name)]
421
+ name = name.replace(weight_name, param_name)
422
+ # Skip loading extra bias for GPTQ models.
423
+ if name.endswith(".bias") and name not in params_dict:
424
+ continue
425
+ param = params_dict[name]
426
weight_loader = param.weight_loader
427
weight_loader(param, loaded_weight, shard_id)
428
break
429
else:
430
+ # Skip loading extra bias for GPTQ models.
431
+ if name.endswith(".bias") and name not in params_dict:
432
+ continue
433
param = params_dict[name]
434
weight_loader = getattr(param, "weight_loader",
435
default_weight_loader)
vllm/model_executor/models/mpt.py
+3
@@ -297,6 +297,9 @@ class MPTForCausalLM(nn.Module):
297
params_dict = dict(self.named_parameters(remove_duplicate=False))
298
for name, loaded_weight in hf_model_weights_iterator(
299
model_name_or_path, cache_dir, load_format, revision):
300
+ # Skip loading extra bias for GPTQ models.
301
+ if name.endswith(".bias") and name not in params_dict:
302
+ continue
303
param = params_dict[name]
304
weight_loader = getattr(param, "weight_loader",
305
default_weight_loader)
vllm/model_executor/models/opt.py
+8
-1
@@ -345,11 +345,18 @@ class OPTForCausalLM(nn.Module):
345
for (param_name, weight_name, shard_id) in stacked_params_mapping:
346
if weight_name not in name:
347
continue
348
- param = params_dict[name.replace(weight_name, param_name)]
348
+ name = name.replace(weight_name, param_name)
349
+ # Skip loading extra bias for GPTQ models.
350
+ if name.endswith(".bias") and name not in params_dict:
351
+ continue
352
+ param = params_dict[name]
353
weight_loader = param.weight_loader
354
weight_loader(param, loaded_weight, shard_id)
355
break
356
else:
357
+ # Skip loading extra bias for GPTQ models.
358
+ if name.endswith(".bias") and name not in params_dict:
359
+ continue
360
param = params_dict[name]
361
weight_loader = getattr(param, "weight_loader",
362
default_weight_loader)
vllm/model_executor/models/phi_1_5.py
+3
@@ -305,6 +305,9 @@ class PhiForCausalLM(nn.Module):
305
if "rotary_emb.inv_freq" in name:
306
continue
307
308
+ # Skip loading extra bias for GPTQ models.
309
+ if name.endswith(".bias") and name not in params_dict:
310
+ continue
311
# pylint: disable=E1136
312
param = params_dict[name]
313
weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/qwen.py
+8
-2
@@ -82,7 +82,6 @@ class QWenAttention(nn.Module):
82
self.num_heads = (self.total_num_heads //
83
tensor_model_parallel_world_size)
84
self.head_dim = hidden_size // self.total_num_heads
85
-
85
self.c_attn = QKVParallelLinear(
86
hidden_size,
87
self.head_dim,
@@ -279,11 +278,18 @@ class QWenLMHeadModel(nn.Module):
278
for (param_name, weight_name, shard_id) in stacked_params_mapping:
279
if weight_name not in name:
280
continue
282
- param = params_dict[name.replace(weight_name, param_name)]
281
+ name = name.replace(weight_name, param_name)
282
+ # Skip loading extra bias for GPTQ models.
283
+ if name.endswith(".bias") and name not in params_dict:
284
+ continue
285
+ param = params_dict[name]
286
weight_loader = param.weight_loader
287
weight_loader(param, loaded_weight, shard_id)
288
break
289
else:
290
+ # Skip loading extra bias for GPTQ models.
291
+ if name.endswith(".bias") and name not in params_dict:
292
+ continue
293
param = params_dict[name]
294
weight_loader = getattr(param, "weight_loader",
295
default_weight_loader)
vllm/model_executor/models/yi.py
+8
-1
@@ -320,11 +320,18 @@ class YiForCausalLM(nn.Module):
320
for (param_name, weight_name, shard_id) in stacked_params_mapping:
321
if weight_name not in name:
322
continue
323
- param = params_dict[name.replace(weight_name, param_name)]
323
+ name = name.replace(weight_name, param_name)
324
+ # Skip loading extra bias for GPTQ models.
325
+ if name.endswith(".bias") and name not in params_dict:
326
+ continue
327
+ param = params_dict[name]
328
weight_loader = param.weight_loader
329
weight_loader(param, loaded_weight, shard_id)
330
break
331
else:
332
+ # Skip loading extra bias for GPTQ models.
333
+ if name.endswith(".bias") and name not in params_dict:
334
+ continue
335
param = params_dict[name]
336
weight_loader = getattr(param, "weight_loader",
337
default_weight_loader)
vllm/model_executor/weight_utils.py
+2
-1
@@ -287,4 +287,5 @@ def initialize_dummy_weights(
287
values between -1e-3 and 1e-3 works well for most models.
288
"""
289
for param in model.state_dict().values():
290
- param.data.uniform_(low, high)
290
+ if torch.is_floating_point(param):
291
+ param.data.uniform_(low, high)