A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add Support for 2/3/8-bit GPTQ Quantization Models (#2330)
CHU Tianxiang committed
Feb 29, 2024 at 13:52 UTC
01a5d18a537b65a156cfa1a77706693a24c869c1
8 files changed
+1736
-229
csrc/ops.h
+4
-2
@@ -98,11 +98,13 @@ torch::Tensor gptq_gemm(
98
torch::Tensor b_gptq_qzeros,
99
torch::Tensor b_gptq_scales,
100
torch::Tensor b_g_idx,
101
- bool use_exllama);
101
+ bool use_exllama,
102
+ int bit);
103
104
void gptq_shuffle(
105
torch::Tensor q_weight,
105
- torch::Tensor q_perm);
106
+ torch::Tensor q_perm,
107
+ int bit);
108
109
void moe_align_block_size(
110
torch::Tensor topk_ids,
csrc/quantization/gptq/matrix_view.cuh
+123
@@ -146,6 +146,129 @@ public:
146
__device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; }
147
};
148
149
+class MatrixView_q2_row
150
+{
151
+public:
152
+ const uint32_t* data;
153
+ const int height;
154
+ const int width;
155
+
156
+ __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data, const int height, const int width)
157
+ : data(data), height(height), width(width)
158
+ { }
159
+
160
+ __device__ __forceinline__ int item(int row, int column) const
161
+ {
162
+ int shift = (column & 0x0f) * 2;
163
+ return (data[row * width / 16 + column / 16] >> shift) & 0x03;
164
+ }
165
+
166
+ __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
167
+ {
168
+ int shift = (column & 0x0f) * 2;
169
+ uint32_t d = data[row * width / 16 + column / 16] >> shift;
170
+ items[0] = d & 0x03;
171
+ items[1] = (d >> 2) & 0x03;
172
+ }
173
+
174
+ __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
175
+ {
176
+ int shift = (column & 0x0f) * 2;
177
+ uint32_t d = data[row * width / 16 + column / 16] >> shift;
178
+ items[0] = d & 0x03;
179
+ items[1] = (d >> 2) & 0x03;
180
+ items[2] = (d >> 4) & 0x03;
181
+ items[3] = (d >> 6) & 0x03;
182
+ }
183
+};
184
+
185
+class MatrixView_q3_row
186
+{
187
+public:
188
+ const uint32_t* data;
189
+ const int height;
190
+ const int width;
191
+
192
+ __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data, const int height, const int width)
193
+ : data(data), height(height), width(width)
194
+ { }
195
+
196
+ __device__ __forceinline__ int item(int row, int column) const
197
+ {
198
+ int z_w = column * 3 / 32;
199
+ int z_mod = column & 0x1f;
200
+
201
+ if (z_mod == 10) {
202
+ return (data[row * width * 3 / 32 + z_w] >> 30) | ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4);
203
+ } else if (z_mod == 21) {
204
+ return (data[row * width * 3 / 32 + z_w] >> 31) | ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6);
205
+ } else if (z_mod < 10) {
206
+ return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07;
207
+ } else if (z_mod < 21) {
208
+ return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07;
209
+ } else {
210
+ return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07;
211
+ }
212
+ }
213
+
214
+ __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
215
+ {
216
+ int shift = (column & 0x1f);
217
+ uint32_t d;
218
+ if (shift <= 4) {
219
+ d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3);
220
+ } else if (shift == 8) {
221
+ d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8);
222
+ } else if (shift <= 16) {
223
+ d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32);
224
+ } else if (shift == 20) {
225
+ d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4);
226
+ } else {
227
+ d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64);
228
+ }
229
+ items[0] = d & 0x07;
230
+ items[1] = (d >> 3) & 0x07;
231
+ items[2] = (d >> 6) & 0x07;
232
+ items[3] = (d >> 9) & 0x07;
233
+ }
234
+};
235
+
236
+class MatrixView_q8_row
237
+{
238
+public:
239
+ const uint32_t* data;
240
+ const int height;
241
+ const int width;
242
+
243
+ __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data, const int height, const int width)
244
+ : data(data), height(height), width(width)
245
+ { }
246
+
247
+ __device__ __forceinline__ int item(int row, int column) const
248
+ {
249
+ int shift = (column & 0x03) * 8;
250
+ return (data[row * width / 4 + column / 4] >> shift) & 0xff;
251
+ }
252
+
253
+ __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
254
+ {
255
+ int shift = (column & 0x03) * 8;
256
+ uint32_t d = data[row * width / 4 + column / 4] >> shift;
257
+ items[0] = d & 0xff;
258
+ items[1] = (d >> 8) & 0xff;
259
+ }
260
+
261
+ __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
262
+ {
263
+ int shift = (column & 0x03) * 2;
264
+ uint32_t d = data[row * width / 4 + column / 4] >> shift;
265
+ items[0] = d & 0xff;
266
+ items[1] = (d >> 8) & 0xff;
267
+ items[2] = (d >> 16) & 0xff;
268
+ items[3] = (d >> 24) & 0xff;
269
+ }
270
+};
271
+
272
} // namespace gptq
273
} // namespace vllm
274
#endif
csrc/quantization/gptq/q_gemm.cu
+1326
-126
@@ -13,7 +13,10 @@ Adapted from https://github.com/turboderp/exllamav2 and https://github.com/qwopq
13
14
#include "compat.cuh"
15
#include "matrix_view.cuh"
16
+#include "qdq_2.cuh"
17
+#include "qdq_3.cuh"
18
#include "qdq_4.cuh"
19
+#include "qdq_8.cuh"
20
21
namespace vllm {
22
namespace gptq {
@@ -22,6 +25,7 @@ namespace gptq {
25
#define BLOCK_M_SIZE_MAX 8
26
#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)
27
#define MAX_Q_GEMM_ROWS 50
28
+#define MAX_Q_GEMM_ROWS_8BIT 24
29
#define MAX_ALT_GEMM_ROWS 8
30
#define THREADS_X 32
31
#define THREADS_Y 32
@@ -75,6 +79,106 @@ __forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
79
return __half2float(__low2half(result)) + __half2float(__high2half(result));
80
}
81
82
+__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h)
83
+{
84
+ half2 result = {};
85
+ const half2* a2_ptr = (const half2*)a_ptr;
86
+ #pragma unroll
87
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
88
+ return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
89
+}
90
+
91
+__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h)
92
+{
93
+ half2 result = {};
94
+ const half2* a2_ptr = (const half2*)a_ptr;
95
+ #pragma unroll
96
+ for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
97
+ return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
98
+}
99
+
100
+__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h)
101
+{
102
+ half2 result = {};
103
+ const half2* a2_ptr = (const half2*)a_ptr;
104
+ #pragma unroll
105
+ for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
106
+ return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
107
+}
108
+
109
+__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f)
110
+{
111
+ half2 result = {};
112
+ const half2* a2_ptr = (const half2*)a_ptr;
113
+ #pragma unroll
114
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
115
+ float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
116
+ return fma(result_f, qs_f, g_result);
117
+}
118
+
119
+__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f)
120
+{
121
+ half2 result = {};
122
+ const half2* a2_ptr = (const half2*)a_ptr;
123
+ #pragma unroll
124
+ for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
125
+ float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
126
+ return fma(result_f, qs_f, g_result);
127
+}
128
+
129
+__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f)
130
+{
131
+ half2 result = {};
132
+ const half2* a2_ptr = (const half2*)a_ptr;
133
+ #pragma unroll
134
+ for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
135
+ float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
136
+ return fma(result_f, qs_f, g_result);
137
+}
138
+
139
+__forceinline__ __device__ half dot22_8_h(half2(&dq)[4], const half* a_ptr, const half g_result, const half qs_h)
140
+{
141
+ // Use FP32 accumulator to avoid potential overflow since unscaled weights are in the range -128..127
142
+
143
+ float result = {};
144
+ #pragma unroll
145
+ for (int i = 0; i < 4; i++)
146
+ {
147
+ half2 w01 = dq[i];
148
+ float w0 = __low2float(w01);
149
+ float w1 = __high2float(w01);
150
+ float x0 = __half2float(*a_ptr++);
151
+ float x1 = __half2float(*a_ptr++);
152
+ result = fma(w0, x0, result);
153
+ result = fma(w1, x1, result);
154
+ }
155
+ float qs = __half2float(qs_h);
156
+ result *= qs;
157
+ half result_h = __float2half_rn(result);
158
+ return __hadd(result_h, g_result);
159
+}
160
+
161
+__forceinline__ __device__ half dot22_16_h(half2(&dq)[8], const half* a_ptr, const half g_result, const half qs_h)
162
+{
163
+ half2 result = {};
164
+ const half2* a2_ptr = (const half2*)a_ptr;
165
+ #pragma unroll
166
+ for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
167
+ half result_h = __hadd(__low2half(result), __high2half(result));
168
+ return __hfma(result_h, qs_h, g_result);
169
+}
170
+
171
+__forceinline__ __device__ half dot22_32_h(half2(&dq)[16], const half* a_ptr, const half g_result, const half qs_h)
172
+{
173
+ half2 result = {};
174
+ const half2* a2_ptr = (const half2*)a_ptr;
175
+ #pragma unroll
176
+ for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
177
+ half result_h = __hadd(__low2half(result), __high2half(result));
178
+ return __hfma(result_h, qs_h, g_result);
179
+}
180
+
181
+
182
typedef void (*fp_gemm_half_q_half_gptq_kernel)
183
(
184
const half*,
@@ -89,8 +193,9 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel)
193
const int*
194
);
195
196
+
197
template <bool first_block, int m_count>
93
-__global__ void gemm_half_q_half_gptq_kernel
198
+__global__ void gemm_half_q_half_gptq_4bit_kernel
199
(
200
const half* __restrict__ a,
201
const uint32_t* __restrict__ b_q_weight,
@@ -231,80 +336,794 @@ __global__ void gemm_half_q_half_gptq_kernel
336
}
337
}
338
234
-
235
-fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count)
339
+template <bool first_block, int m_count>
340
+__global__ void gemm_half_q_half_gptq_2bit_kernel
341
+(
342
+ const half* __restrict__ a,
343
+ const uint32_t* __restrict__ b_q_weight,
344
+ const uint32_t* __restrict__ b_gptq_qzeros,
345
+ const half* __restrict__ b_gptq_scales,
346
+ half* __restrict__ c,
347
+ const int size_m,
348
+ const int size_n,
349
+ const int size_k,
350
+ const int groups,
351
+ const int* __restrict__ b_q_perm
352
+)
353
{
237
- #if BLOCK_M_SIZE_MAX >= 1
238
- if (m_count == 1) return gemm_half_q_half_gptq_kernel<true, 1>;
239
- #endif
240
- #if BLOCK_M_SIZE_MAX >= 2
241
- if (m_count == 2) return gemm_half_q_half_gptq_kernel<true, 2>;
242
- #endif
243
- #if BLOCK_M_SIZE_MAX >= 3
244
- if (m_count == 3) return gemm_half_q_half_gptq_kernel<true, 3>;
245
- #endif
246
- #if BLOCK_M_SIZE_MAX >= 4
247
- if (m_count == 4) return gemm_half_q_half_gptq_kernel<true, 4>;
248
- #endif
249
- #if BLOCK_M_SIZE_MAX >= 5
250
- if (m_count == 5) return gemm_half_q_half_gptq_kernel<true, 5>;
251
- #endif
252
- #if BLOCK_M_SIZE_MAX >= 6
253
- if (m_count == 6) return gemm_half_q_half_gptq_kernel<true, 6>;
254
- #endif
255
- #if BLOCK_M_SIZE_MAX >= 7
256
- if (m_count == 7) return gemm_half_q_half_gptq_kernel<true, 7>;
257
- #endif
258
- #if BLOCK_M_SIZE_MAX >= 8
259
- if (m_count == 8) return gemm_half_q_half_gptq_kernel<true, 8>;
260
- #endif
261
- return NULL;
262
-}
354
+ MatrixView_half a_(a, size_m, size_k);
355
+ MatrixView_half_rw c_(c, size_m, size_n);
356
+ MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
357
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
358
359
+ int t = threadIdx.x;
360
265
-void gemm_half_q_half_cuda_part
361
+ // Block
362
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
363
+ int offset_m = blockIdx.y * m_count;
364
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
365
+
366
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
367
+ int end_m = min(offset_m + m_count, size_m);
368
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
369
+
370
+ int n = offset_n + t * 4;
371
+
372
+ // Preload block_a
373
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
374
+
375
+ if (offset_k + t < end_k)
376
+ {
377
+ for (int m = 0; m < m_count; ++m)
378
+ {
379
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
380
+ half* block_a_ptr = block_a[m];
381
+
382
+ half a0;
383
+ if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
384
+ else a0 = a_ptr[offset_k + t];
385
+ block_a_ptr[t] = a0;
386
+ }
387
+ }
388
+
389
+ // Zero output
390
+ if (n >= size_n) return;
391
+
392
+ if (blockIdx.z == 0)
393
+ {
394
+ for (int m = 0; m < m_count; m++)
395
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
396
+ }
397
+
398
+ __syncthreads();
399
+
400
+ // Find initial group
401
+ int groupsize = size_k / groups;
402
+ int group = offset_k / groupsize;
403
+ int nextgroup = offset_k + groupsize;
404
+
405
+ // a, b offset
406
+ int qk = offset_k / (32 / 2);
407
+
408
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
409
+ const half* a_ptr = &block_a[0][0];
410
+ int a_stride = BLOCK_KN_SIZE;
411
+
412
+ // Initial group
413
+ int zeros[4];
414
+ half scales[4];
415
+ b_gptq_qzeros_.item4(zeros, group, n);
416
+ b_gptq_scales_.item4(scales, group, n);
417
+ // Column result
418
+ half block_c[m_count][4] = {};
419
+
420
+ // Dequantize and multiply
421
+ int k = offset_k;
422
+ while (k < end_k)
423
+ {
424
+ if (k == nextgroup)
425
+ {
426
+ group++;
427
+ nextgroup += groupsize;
428
+ b_gptq_qzeros_.item4(zeros, group, n);
429
+ b_gptq_scales_.item4(scales, group, n);
430
+ }
431
+
432
+ #pragma unroll
433
+ for (int j = 0; j < 1; j++)
434
+ {
435
+ const int4* b_ptr4 = (int4*) b_ptr;
436
+ int4 load_int4 = *b_ptr4;
437
+
438
+ half2 dq[4][8];
439
+ dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
440
+ dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
441
+ dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
442
+ dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
443
+
444
+ #pragma unroll
445
+ for (int m = 0; m < m_count; m++)
446
+ {
447
+ block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
448
+ block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
449
+ block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
450
+ block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
451
+ }
452
+
453
+ b_ptr += size_n;
454
+ a_ptr += 16;
455
+ }
456
+
457
+ k += 16;
458
+ }
459
+
460
+ for (int m = 0; m < m_count; m++)
461
+ {
462
+ half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
463
+ half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
464
+ half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
465
+ atomicAdd(out , result01);
466
+ atomicAdd(out + 1, result23);
467
+ }
468
+}
469
+
470
+template <bool first_block, int m_count>
471
+__global__ void gemm_half_q_half_gptq_3bit_kernel
472
(
267
- const half* a,
268
- const uint32_t* b_q_weight,
269
- const uint32_t* b_gptq_qzeros,
270
- const half* b_gptq_scales,
271
- const int* b_q_perm,
272
- half* c,
273
- int size_m,
274
- int size_n,
275
- int size_k,
276
- int m_count,
277
- int groups
473
+ const half* __restrict__ a,
474
+ const uint32_t* __restrict__ b_q_weight,
475
+ const uint32_t* __restrict__ b_gptq_qzeros,
476
+ const half* __restrict__ b_gptq_scales,
477
+ half* __restrict__ c,
478
+ const int size_m,
479
+ const int size_n,
480
+ const int size_k,
481
+ const int groups,
482
+ const int* __restrict__ b_q_perm
483
)
484
{
280
- dim3 blockDim, gridDim;
281
- blockDim.x = BLOCK_KN_SIZE;
282
- blockDim.y = 1;
283
- blockDim.z = 1;
284
- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
285
- gridDim.y = DIVIDE(size_m, m_count);
286
- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
485
+ MatrixView_half a_(a, size_m, size_k);
486
+ MatrixView_half_rw c_(c, size_m, size_n);
487
+ MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
488
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
489
288
- fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
490
+ int t = threadIdx.x;
491
290
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
291
- kernel<<<gridDim, blockDim, 0, stream>>>
292
- (
293
- a,
294
- b_q_weight,
295
- b_gptq_qzeros,
296
- b_gptq_scales,
297
- c,
298
- size_m,
299
- size_n,
300
- size_k,
301
- groups,
302
- b_q_perm
303
- );
304
-}
492
+ // Block
493
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
494
+ int offset_m = blockIdx.y * m_count;
495
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
496
+
497
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
498
+ int end_m = min(offset_m + m_count, size_m);
499
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
500
+
501
+ int n = offset_n + t * 4;
502
+
503
+ // Preload block_a
504
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
505
+
506
+ if (offset_k + t < end_k)
507
+ {
508
+ for (int m = 0; m < m_count; ++m)
509
+ {
510
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
511
+ half* block_a_ptr = block_a[m];
512
513
+ half a0;
514
+ if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
515
+ else a0 = a_ptr[offset_k + t];
516
+ block_a_ptr[t] = a0;
517
+ }
518
+ }
519
+
520
+ // Zero output
521
+ if (n >= size_n) return;
522
+
523
+ if (blockIdx.z == 0)
524
+ {
525
+ for (int m = 0; m < m_count; m++)
526
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
527
+ }
528
+
529
+ __syncthreads();
530
+
531
+ // Find initial group
532
+ int groupsize = size_k / groups;
533
+ int group = offset_k / groupsize;
534
+ int nextgroup = offset_k + groupsize;
535
+
536
+ // a, b offset
537
+ int qk = offset_k / 32 * 3;
538
+
539
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
540
+ const half* a_ptr = &block_a[0][0];
541
+ int a_stride = BLOCK_KN_SIZE;
542
+
543
+ // Initial group
544
+ int zeros[4];
545
+ half scales[4];
546
+ b_gptq_qzeros_.item4(zeros, group, n);
547
+ b_gptq_scales_.item4(scales, group, n);
548
+ // Column result
549
+ half block_c[m_count][4] = {};
550
+
551
+ // Dequantize and multiply
552
+ int k = offset_k;
553
+ while (k < end_k)
554
+ {
555
+ if (k == nextgroup)
556
+ {
557
+ group++;
558
+ nextgroup += groupsize;
559
+ b_gptq_qzeros_.item4(zeros, group, n);
560
+ b_gptq_scales_.item4(scales, group, n);
561
+ }
562
+
563
+ #pragma unroll
564
+ for (int j = 0; j < 1; j++)
565
+ {
566
+ int4 load_int4[3];
567
+ load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
568
+ load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
569
+ load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
570
+
571
+ half2 dq[4][16];
572
+ dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
573
+ dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
574
+ dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
575
+ dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
576
+
577
+ #pragma unroll
578
+ for (int m = 0; m < m_count; m++)
579
+ {
580
+ block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
581
+ block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
582
+ block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
583
+ block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
584
+ }
585
+ a_ptr += 32;
586
+ }
587
+
588
+ k += 32;
589
+ }
590
+
591
+ for (int m = 0; m < m_count; m++)
592
+ {
593
+ half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
594
+ half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
595
+ half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
596
+ atomicAdd(out , result01);
597
+ atomicAdd(out + 1, result23);
598
+ }
599
+}
600
+
601
+template <bool first_block, int m_count>
602
+__global__ void gemm_half_q_half_gptq_8bit_kernel
603
+(
604
+ const half* __restrict__ a,
605
+ const uint32_t* __restrict__ b_q_weight,
606
+ const uint32_t* __restrict__ b_gptq_qzeros,
607
+ const half* __restrict__ b_gptq_scales,
608
+ half* __restrict__ c,
609
+ const int size_m,
610
+ const int size_n,
611
+ const int size_k,
612
+ const int groups,
613
+ const int* __restrict__ b_q_perm
614
+)
615
+{
616
+ MatrixView_half a_(a, size_m, size_k);
617
+ MatrixView_half_rw c_(c, size_m, size_n);
618
+ MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
619
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
620
+
621
+ int t = threadIdx.x;
622
+
623
+ // Block
624
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
625
+ int offset_m = blockIdx.y * m_count;
626
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
627
+
628
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
629
+ int end_m = min(offset_m + m_count, size_m);
630
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
631
+
632
+ int n = offset_n + t * 4;
633
+
634
+ // Preload block_a
635
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
636
+
637
+ if (offset_k + t < end_k)
638
+ {
639
+ for (int m = 0; m < m_count; ++m)
640
+ {
641
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
642
+ half* block_a_ptr = block_a[m];
643
+
644
+ half a0;
645
+ if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
646
+ else a0 = a_ptr[offset_k + t];
647
+ block_a_ptr[t] = a0;
648
+ }
649
+ }
650
+
651
+ // Zero output
652
+ if (n >= size_n) return;
653
+
654
+ if (blockIdx.z == 0)
655
+ {
656
+ for (int m = 0; m < m_count; m++)
657
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
658
+ }
659
+
660
+ __syncthreads();
661
+
662
+ // Find initial group
663
+ int groupsize = size_k / groups;
664
+ int group = offset_k / groupsize;
665
+ int nextgroup = offset_k + groupsize;
666
+
667
+ // a, b offset
668
+ int qk = offset_k / (32 / 8);
669
+
670
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
671
+ const half* a_ptr = &block_a[0][0];
672
+ int a_stride = BLOCK_KN_SIZE;
673
+
674
+ // Initial group
675
+ int zeros[4];
676
+ half scales[4];
677
+ b_gptq_qzeros_.item4(zeros, group, n);
678
+ b_gptq_scales_.item4(scales, group, n);
679
+ // Column result
680
+ half block_c[m_count][4] = {};
681
+
682
+ // Dequantize and multiply
683
+ int k = offset_k;
684
+ while (k < end_k)
685
+ {
686
+ if (k == nextgroup)
687
+ {
688
+ group++;
689
+ nextgroup += groupsize;
690
+ b_gptq_qzeros_.item4(zeros, group, n);
691
+ b_gptq_scales_.item4(scales, group, n);
692
+ }
693
+
694
+ #pragma unroll
695
+ for (int j = 0; j < 4; j++)
696
+ {
697
+ int4 load_int4[2];
698
+ load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
699
+ load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
700
+
701
+ half2 dq[4][4];
702
+ dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
703
+ dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
704
+ dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
705
+ dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
706
+
707
+ for (int m = 0; m < m_count; m++)
708
+ {
709
+ block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
710
+ block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
711
+ block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
712
+ block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
713
+ }
714
+ a_ptr += 8;
715
+ }
716
+ k += 32;
717
+ }
718
+
719
+ for (int m = 0; m < m_count; m++)
720
+ {
721
+ half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
722
+ half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
723
+ half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
724
+ atomicAdd(out , result01);
725
+ atomicAdd(out + 1, result23);
726
+ }
727
+}
728
+
729
+fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
730
+ bool first_block, const int m_count, const int bit)
731
+{
732
+ #define SELECT_KERNEL(M_COUNT) \
733
+ if (m_count == M_COUNT) { \
734
+ if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel<true, M_COUNT>; \
735
+ if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel<true, M_COUNT>; \
736
+ if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel<true, M_COUNT>; \
737
+ if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel<true, M_COUNT>; \
738
+ }
739
+ #if BLOCK_M_SIZE_MAX >= 1
740
+ SELECT_KERNEL(1);
741
+ #endif
742
+ #if BLOCK_M_SIZE_MAX >= 2
743
+ SELECT_KERNEL(2);
744
+ #endif
745
+ #if BLOCK_M_SIZE_MAX >= 3
746
+ SELECT_KERNEL(3);
747
+ #endif
748
+ #if BLOCK_M_SIZE_MAX >= 4
749
+ SELECT_KERNEL(4);
750
+ #endif
751
+ #if BLOCK_M_SIZE_MAX >= 5
752
+ SELECT_KERNEL(5);
753
+ #endif
754
+ #if BLOCK_M_SIZE_MAX >= 6
755
+ SELECT_KERNEL(6);
756
+ #endif
757
+ #if BLOCK_M_SIZE_MAX >= 7
758
+ SELECT_KERNEL(7);
759
+ #endif
760
+ #if BLOCK_M_SIZE_MAX >= 8
761
+ SELECT_KERNEL(8);
762
+ #endif
763
+ return NULL;
764
+}
765
+
766
+
767
+void gemm_half_q_half_cuda_part
768
+(
769
+ const half* a,
770
+ const uint32_t* b_q_weight,
771
+ const uint32_t* b_gptq_qzeros,
772
+ const half* b_gptq_scales,
773
+ const int* b_q_perm,
774
+ half* c,
775
+ int size_m,
776
+ int size_n,
777
+ int size_k,
778
+ int m_count,
779
+ int groups,
780
+ int bit
781
+)
782
+{
783
+ dim3 blockDim, gridDim;
784
+ blockDim.x = BLOCK_KN_SIZE;
785
+ blockDim.y = 1;
786
+ blockDim.z = 1;
787
+ gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
788
+ gridDim.y = DIVIDE(size_m, m_count);
789
+ gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
790
+
791
+ fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
792
+
793
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
794
+ kernel<<<gridDim, blockDim, 0, stream>>>
795
+ (
796
+ a,
797
+ b_q_weight,
798
+ b_gptq_qzeros,
799
+ b_gptq_scales,
800
+ c,
801
+ size_m,
802
+ size_n,
803
+ size_k,
804
+ groups,
805
+ b_q_perm
806
+ );
807
+}
808
+
809
+
810
+__global__ void reconstruct_exllama_8bit_kernel
811
+(
812
+ const uint32_t* __restrict__ b_q_weight,
813
+ const int* __restrict__ b_q_perm,
814
+ const uint32_t* __restrict__ b_gptq_qzeros,
815
+ const half* __restrict__ b_gptq_scales,
816
+ const int size_k,
817
+ const int size_n,
818
+ const int groups,
819
+ half* __restrict__ b
820
+)
821
+{
822
+ MatrixView_half_rw b_(b, size_k, size_n);
823
+ MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
824
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
825
+
826
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
827
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
828
+
829
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
830
+
831
+ // Preload remapping table
832
+ __shared__ int perm[BLOCK_KN_SIZE];
833
+ int t = threadIdx.x;
834
+
835
+ if (b_q_perm)
836
+ {
837
+ if (offset_k + t < size_k)
838
+ perm[t] = b_q_perm[offset_k + t];
839
+ }
840
+
841
+ // Column
842
+ int n = offset_n + t * 4;
843
+ if (n >= size_n) return;
844
+
845
+ // Find initial group
846
+ int groupsize = size_k / groups;
847
+ int group = offset_k / groupsize;
848
+ int nextgroup = offset_k + groupsize;
849
+
850
+ // b offset
851
+ int qk = offset_k / (32 / 8);
852
+
853
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
854
+
855
+ // Initial zeros/scale
856
+ int zeros[4];
857
+ half2 scales[4];
858
+ b_gptq_qzeros_.item4(zeros, group, n);
859
+ b_gptq_scales_.item4_h2(scales, group, n);
860
+
861
+ __syncthreads();
862
+
863
+ int k = offset_k;
864
+ int lk = 0;
865
+
866
+ while (k < end_k)
867
+ {
868
+ if (k == nextgroup)
869
+ {
870
+ group++;
871
+ nextgroup += groupsize;
872
+ b_gptq_qzeros_.item4(zeros, group, n);
873
+ b_gptq_scales_.item4_h2(scales, group, n);
874
+ }
875
+
876
+ for (int p = 0; p < 4; p++)
877
+ {
878
+ int4 load_int4[2];
879
+ load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
880
+ load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
881
+
882
+ half2 dq[4][4];
883
+ dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
884
+ dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
885
+ dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
886
+ dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
887
+
888
+ //half* dqh = (half*)dq;
889
+ if (b_q_perm)
890
+ {
891
+ for (int j = 0; j < 4; j++)
892
+ {
893
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
894
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
895
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
896
+ }
897
+ }
898
+ else
899
+ {
900
+ for (int j = 0; j < 4; j++)
901
+ {
902
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
903
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
904
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
905
+ }
906
+ }
907
+ }
908
+ k += 32;
909
+ }
910
+}
911
+
912
+__global__ void reconstruct_exllama_4bit_kernel
913
+(
914
+ const uint32_t* __restrict__ b_q_weight,
915
+ const int* __restrict__ b_q_perm,
916
+ const uint32_t* __restrict__ b_gptq_qzeros,
917
+ const half* __restrict__ b_gptq_scales,
918
+ const int size_k,
919
+ const int size_n,
920
+ const int groups,
921
+ half* __restrict__ b
922
+)
923
+{
924
+ MatrixView_half_rw b_(b, size_k, size_n);
925
+ MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
926
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
927
+
928
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
929
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
930
+
931
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
932
+
933
+ // Preload remapping table
934
+ __shared__ int perm[BLOCK_KN_SIZE];
935
+ int t = threadIdx.x;
936
+
937
+ if (b_q_perm)
938
+ {
939
+ if (offset_k + t < size_k)
940
+ perm[t] = b_q_perm[offset_k + t];
941
+ }
942
+
943
+ // Column
944
+ int n = offset_n + t * 4;
945
+ if (n >= size_n) return;
946
+
947
+ // Find initial group
948
+ int groupsize = size_k / groups;
949
+ int group = offset_k / groupsize;
950
+ int nextgroup = offset_k + groupsize;
951
+
952
+ // b offset
953
+ int qk = offset_k / (32 / 4);
954
+
955
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
956
+
957
+ // Initial zeros/scale
958
+ int zeros[4];
959
+ half2 scales[4];
960
+ half2 z1z16[4][2];
961
+ half2 y1y16[4][2];
962
+ b_gptq_qzeros_.item4(zeros, group, n);
963
+ b_gptq_scales_.item4_h2(scales, group, n);
964
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
965
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
966
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
967
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
968
+
969
+ __syncthreads();
970
+
971
+ int k = offset_k;
972
+ int lk = 0;
973
+
974
+ while (k < end_k)
975
+ {
976
+ if (k == nextgroup)
977
+ {
978
+ group++;
979
+ nextgroup += groupsize;
980
+ b_gptq_qzeros_.item4(zeros, group, n);
981
+ b_gptq_scales_.item4_h2(scales, group, n);
982
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
983
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
984
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
985
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
986
+ }
987
+
988
+ for (int p = 0; p < 4; p++)
989
+ {
990
+ half2 dq[4][4];
991
+ const int4* b_ptr4 = (int4*) b_ptr;
992
+ int4 load_int4 = *b_ptr4;
993
+
994
+ dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
995
+ dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
996
+ dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
997
+ dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
998
+
999
+ b_ptr += size_n;
1000
+ //half* dqh = (half*)dq;
1001
+ if (b_q_perm)
1002
+ {
1003
+ for (int j = 0; j < 4; j++)
1004
+ {
1005
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1006
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1007
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1008
+ }
1009
+ }
1010
+ else
1011
+ {
1012
+ for (int j = 0; j < 4; j++)
1013
+ {
1014
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1015
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1016
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1017
+ }
1018
+ }
1019
+ }
1020
+ k += 32;
1021
+ }
1022
+}
1023
+
1024
+__global__ void reconstruct_exllama_3bit_kernel
1025
+(
1026
+ const uint32_t* __restrict__ b_q_weight,
1027
+ const int* __restrict__ b_q_perm,
1028
+ const uint32_t* __restrict__ b_gptq_qzeros,
1029
+ const half* __restrict__ b_gptq_scales,
1030
+ const int size_k,
1031
+ const int size_n,
1032
+ const int groups,
1033
+ half* __restrict__ b
1034
+)
1035
+{
1036
+ MatrixView_half_rw b_(b, size_k, size_n);
1037
+ MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1038
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1039
+
1040
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1041
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1042
+
1043
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1044
+
1045
+ // Preload remapping table
1046
+ __shared__ int perm[BLOCK_KN_SIZE];
1047
+ int t = threadIdx.x;
1048
+
1049
+ if (b_q_perm)
1050
+ {
1051
+ if (offset_k + t < size_k)
1052
+ perm[t] = b_q_perm[offset_k + t];
1053
+ }
1054
+
1055
+ // Column
1056
+ int n = offset_n + t * 4;
1057
+ if (n >= size_n) return;
1058
+
1059
+ // Find initial group
1060
+ int groupsize = size_k / groups;
1061
+ int group = offset_k / groupsize;
1062
+ int nextgroup = offset_k + groupsize;
1063
+
1064
+ // b offset
1065
+ int qk = offset_k / 32* 3;
1066
+
1067
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1068
+
1069
+ // Initial zeros/scale
1070
+ int zeros[4];
1071
+ half2 scales[4];
1072
+ b_gptq_qzeros_.item4(zeros, group, n);
1073
+ b_gptq_scales_.item4_h2(scales, group, n);
1074
+
1075
+ __syncthreads();
1076
+
1077
+ int k = offset_k;
1078
+ int lk = 0;
1079
+
1080
+ while (k < end_k)
1081
+ {
1082
+ if (k == nextgroup)
1083
+ {
1084
+ group++;
1085
+ nextgroup += groupsize;
1086
+ b_gptq_qzeros_.item4(zeros, group, n);
1087
+ b_gptq_scales_.item4_h2(scales, group, n);
1088
+ }
1089
+
1090
+ for (int p = 0; p < 1; p++)
1091
+ {
1092
+ int4 load_int4[3];
1093
+ load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
1094
+ load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
1095
+ load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
1096
+
1097
+ half2 dq[4][16];
1098
+ dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
1099
+ dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
1100
+ dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
1101
+ dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
1102
+
1103
+ if (b_q_perm)
1104
+ {
1105
+ for (int j = 0; j < 16; j++)
1106
+ {
1107
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1108
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1109
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1110
+ }
1111
+ }
1112
+ else
1113
+ {
1114
+ for (int j = 0; j < 16; j++)
1115
+ {
1116
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1117
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1118
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1119
+ }
1120
+ }
1121
+ }
1122
+ k += 32;
1123
+ }
1124
+}
1125
307
-__global__ void reconstruct_exllama_kernel
1126
+__global__ void reconstruct_exllama_2bit_kernel
1127
(
1128
const uint32_t* __restrict__ b_q_weight,
1129
const int* __restrict__ b_q_perm,
@@ -317,7 +1136,7 @@ __global__ void reconstruct_exllama_kernel
1136
)
1137
{
1138
MatrixView_half_rw b_(b, size_k, size_n);
320
- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1139
+ MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1140
MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1141
1142
int offset_k = BLOCK_KN_SIZE * blockIdx.y;
@@ -345,21 +1164,15 @@ __global__ void reconstruct_exllama_kernel
1164
int nextgroup = offset_k + groupsize;
1165
1166
// b offset
348
- int qk = offset_k / (32 / 4);
1167
+ int qk = offset_k / (32 / 2);
1168
1169
const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1170
1171
// Initial zeros/scale
1172
int zeros[4];
1173
half2 scales[4];
355
- half2 z1z16[4][2];
356
- half2 y1y16[4][2];
1174
b_gptq_qzeros_.item4(zeros, group, n);
1175
b_gptq_scales_.item4_h2(scales, group, n);
359
- dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
360
- dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
361
- dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
362
- dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
1176
1177
__syncthreads();
1178
@@ -374,28 +1187,24 @@ __global__ void reconstruct_exllama_kernel
1187
nextgroup += groupsize;
1188
b_gptq_qzeros_.item4(zeros, group, n);
1189
b_gptq_scales_.item4_h2(scales, group, n);
377
- dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
378
- dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
379
- dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
380
- dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
1190
}
1191
383
- for (int p = 0; p < 4; p++)
1192
+ for (int p = 0; p < 2; p++)
1193
{
385
- half2 dq[4][4];
1194
const int4* b_ptr4 = (int4*) b_ptr;
1195
int4 load_int4 = *b_ptr4;
1196
389
- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
390
- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
391
- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
392
- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
1197
+ half2 dq[4][8];
1198
+ dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
1199
+ dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
1200
+ dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
1201
+ dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
1202
1203
b_ptr += size_n;
1204
//half* dqh = (half*)dq;
1205
if (b_q_perm)
1206
{
398
- for (int j = 0; j < 4; j++)
1207
+ for (int j = 0; j < 8; j++)
1208
{
1209
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1210
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
@@ -404,7 +1213,7 @@ __global__ void reconstruct_exllama_kernel
1213
}
1214
else
1215
{
407
- for (int j = 0; j < 4; j++)
1216
+ for (int j = 0; j < 8; j++)
1217
{
1218
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1219
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
@@ -416,7 +1225,6 @@ __global__ void reconstruct_exllama_kernel
1225
}
1226
}
1227
419
-
1228
void reconstruct_exllama
1229
(
1230
const uint32_t* b_q_weight,
@@ -426,7 +1234,8 @@ void reconstruct_exllama
1234
half* out,
1235
int height,
1236
int width,
429
- int groups
1237
+ int groups,
1238
+ int bit
1239
)
1240
{
1241
dim3 blockDim, gridDim;
@@ -435,6 +1244,15 @@ void reconstruct_exllama
1244
gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
1245
gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1246
1247
+ auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel;
1248
+ if (bit == 2) {
1249
+ reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel;
1250
+ } else if (bit == 3) {
1251
+ reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel;
1252
+ } else if (bit == 8) {
1253
+ reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel;
1254
+ }
1255
+
1256
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1257
reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>
1258
(
@@ -450,7 +1268,7 @@ void reconstruct_exllama
1268
}
1269
1270
453
-__global__ void gemm_half_q_half_alt_kernel(
1271
+__global__ void gemm_half_q_half_alt_4bit_kernel(
1272
const half2* __restrict__ vec,
1273
const uint32_t* __restrict__ mat,
1274
half* __restrict__ mul,
@@ -548,6 +1366,95 @@ __global__ void gemm_half_q_half_alt_kernel(
1366
}
1367
1368
1369
+__global__ void gemm_half_q_half_alt_8bit_kernel(
1370
+ const half2* __restrict__ vec,
1371
+ const uint32_t* __restrict__ mat,
1372
+ half* __restrict__ mul,
1373
+ const half* __restrict__ scales,
1374
+ const uint32_t* __restrict__ zeros,
1375
+ const int* __restrict__ g_idx,
1376
+ int batch,
1377
+ int height,
1378
+ int width
1379
+)
1380
+{
1381
+ int zero_width = width / 4;
1382
+ int vec_height = height * 2;
1383
+ const int blockwidth2 = BLOCK_KN_SIZE / 2;
1384
+ int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1385
+ int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1386
+ int h = BLOCK_KN_SIZE * blockIdx.z / 4;
1387
+ int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;
1388
+ int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1389
+
1390
+ __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1391
+ if (threadIdx.x < h_end) {
1392
+ for (int m = 0; m < b_end; ++m) {
1393
+ blockvec[m][threadIdx.x] =
1394
+ vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1395
+ threadIdx.x];
1396
+ }
1397
+ }
1398
+
1399
+
1400
+ if (blockIdx.z == 0)
1401
+ {
1402
+ for (int m = 0; m < b_end; m++)
1403
+ mul[(b + m) * width + w] = __int2half_rn(0);
1404
+ }
1405
+ __syncthreads();
1406
+
1407
+ int i = width * h + w;
1408
+ int g_h = h * 4;
1409
+ int k = 0;
1410
+ int z_w = w / 4;
1411
+ int z_mod = (w % 4) * 8;
1412
+ half2 res2;
1413
+ half res[BLOCK_M_SIZE_MAX] = {};
1414
+
1415
+ unsigned int tmp;
1416
+ while (k < h_end) {
1417
+ tmp = mat[i];
1418
+ half2 scales_tmp[2];
1419
+ half2 zeros_tmp[2];
1420
+ for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
1421
+ int g = g_idx[g_h + (k + tmp_k) * 2];
1422
+ int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1423
+ half scale_f = scales[g * width + w];
1424
+ half scale_f2 = scales[g2 * width + w];
1425
+ half2 scale = __halves2half2(scale_f, scale_f2);
1426
+ half2 zero = __halves2half2(
1427
+ __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),
1428
+ __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1))
1429
+ );
1430
+ scales_tmp[tmp_k] = scale;
1431
+ zeros_tmp[tmp_k] = zero;
1432
+ }
1433
+ for (int m = 0; m < b_end; m++) {
1434
+#ifndef USE_ROCM
1435
+ res2 = {};
1436
+#else
1437
+ res2.x = __half_as_ushort(__float2half(0));
1438
+ res2.y = __half_as_ushort(__float2half(0));
1439
+#endif
1440
+ half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF), __int2half_rn((tmp >> 8) & 0xFF));
1441
+ res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
1442
+ half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF), __int2half_rn((tmp >> 24) & 0xFF));
1443
+ res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
1444
+#ifndef USE_ROCM
1445
+ res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1446
+#else
1447
+ res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1448
+#endif
1449
+ }
1450
+ i += width;
1451
+ k += 2;
1452
+ }
1453
+ for (int m = 0; m < b_end; m++) {
1454
+ atomicAdd(&mul[(b + m) * width + w], res[m]);
1455
+ }
1456
+}
1457
+
1458
void gemm_half_q_half_alt
1459
(
1460
const half* a,
@@ -558,7 +1465,8 @@ void gemm_half_q_half_alt
1465
half* c,
1466
int size_m,
1467
int size_n,
561
- int size_k
1468
+ int size_k,
1469
+ int bit
1470
)
1471
{
1472
dim3 blockDim, gridDim;
@@ -569,8 +1477,13 @@ void gemm_half_q_half_alt
1477
gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
1478
gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
1479
1480
+ auto kernel = gemm_half_q_half_alt_4bit_kernel;
1481
+ if (bit == 8) {
1482
+ kernel = gemm_half_q_half_alt_8bit_kernel;
1483
+ }
1484
+
1485
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
573
- gemm_half_q_half_alt_kernel<<<gridDim, blockDim, 0, stream>>>
1486
+ kernel<<<gridDim, blockDim, 0, stream>>>
1487
(
1488
(const half2*) a,
1489
b_q_weight,
@@ -579,12 +1492,12 @@ void gemm_half_q_half_alt
1492
b_gptq_qzeros,
1493
b_g_idx,
1494
size_m,
582
- size_k / 8,
1495
+ size_k / 32 * bit,
1496
size_n
1497
);
1498
}
1499
587
-
1500
+template<class T, int bit>
1501
__global__ void reconstruct_gptq_kernel
1502
(
1503
const uint32_t* __restrict__ w,
@@ -600,30 +1513,79 @@ __global__ void reconstruct_gptq_kernel
1513
// Start of block
1514
1515
int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
603
- int row = blockIdx.y * 8;
1516
+ int row = blockIdx.y * 32 / bit;
1517
if (column >= width) return;
1518
1519
// Views
1520
608
- MatrixView_q4_column w_(w, height, width);
1521
MatrixView_half_rw out_(out, height, width);
1522
MatrixView_half w_scales_(w_scales, group, width);
611
- MatrixView_q4_row w_zeros_(w_zeros, group, width);
1523
+ T w_zeros_(w_zeros, group, width);
1524
613
- uint32_t w_read = w_.item_uint32_t(row, column);
1525
+ uint32_t w_read = w[blockIdx.y * width + column];
1526
half* out_ptr = out_.item_ptr(row, column);
1527
1528
#pragma unroll
617
- for (int s = 0; s < 32; s += 4)
1529
+ for (int s = 0; s < 32; s += bit)
1530
{
619
- int group = g_idx[row + s / 4];
1531
+ int group = g_idx[row + s / bit];
1532
half w_scale = w_scales_.item(group, column);
1533
uint32_t w_zero = w_zeros_.item(group, column) + 1;
622
- half w_item = __hmul(__int2half_rn((int)((w_read >> s) & 0x0f) - w_zero), w_scale);
1534
+ half w_item = __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero), w_scale);
1535
*out_ptr = w_item; out_ptr += out_.width;
1536
}
1537
}
1538
1539
+__global__ void reconstruct_gptq_3bit_kernel
1540
+(
1541
+ const uint32_t* __restrict__ w,
1542
+ const half* __restrict__ w_scales,
1543
+ const uint32_t* __restrict__ w_zeros,
1544
+ const int* __restrict__ g_idx,
1545
+ const int height,
1546
+ const int width,
1547
+ const int group,
1548
+ half* __restrict__ out
1549
+)
1550
+{
1551
+ // Start of block
1552
+ int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1553
+ int row = blockIdx.y * 32;
1554
+ if (column >= width) return;
1555
+
1556
+ // Views
1557
+
1558
+ MatrixView_half_rw out_(out, height, width);
1559
+ MatrixView_half w_scales_(w_scales, group, width);
1560
+ MatrixView_q3_row w_zeros_(w_zeros, group, width);
1561
+
1562
+ uint32_t w1 = w[(blockIdx.y * 3) * width + column];
1563
+ uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];
1564
+ uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];
1565
+ half* out_ptr = out_.item_ptr(row, column);
1566
+
1567
+ #pragma unroll
1568
+ for (int i = 0; i < 32; i += 1)
1569
+ {
1570
+ int group = g_idx[row + i];
1571
+ half w_scale = w_scales_.item(group, column);
1572
+ uint32_t w_zero = w_zeros_.item(group, column) + 1;
1573
+ int w_item;
1574
+ if (i == 10) {
1575
+ w_item = (w1 >> 30) | ((w2 << 2) & 0x4);
1576
+ } else if (i == 21) {
1577
+ w_item = (w2 >> 31) | ((w3 << 1) & 0x6);
1578
+ } else if (i < 10) {
1579
+ w_item = ((w1 >> (i * 3)) & 0x7);
1580
+ } else if (i < 21) {
1581
+ w_item = ((w2 >> (i * 3 - 32)) & 0x7);
1582
+ } else {
1583
+ w_item = ((w3 >> (i * 3 - 64)) & 0x7);
1584
+ }
1585
+ *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale);
1586
+ out_ptr += out_.width;
1587
+ }
1588
+}
1589
1590
void reconstruct_gptq
1591
(
@@ -634,16 +1596,28 @@ void reconstruct_gptq
1596
half* out,
1597
int height,
1598
int width,
637
- int groups
1599
+ int groups,
1600
+ int bit
1601
)
1602
{
1603
dim3 blockDim, gridDim;
1604
blockDim.x = BLOCK_KN_SIZE;
1605
blockDim.y = 1;
643
- gridDim.y = DIVIDE(height, 8);
1606
+ gridDim.y = DIVIDE(height, 32 / bit);
1607
gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1608
+
1609
+ auto kernel = reconstruct_gptq_kernel<MatrixView_q4_row, 4>;
1610
+ if (bit == 2) {
1611
+ kernel = reconstruct_gptq_kernel<MatrixView_q2_row, 2>;
1612
+ } else if (bit == 8) {
1613
+ kernel = reconstruct_gptq_kernel<MatrixView_q8_row, 8>;
1614
+ } else if (bit == 3) {
1615
+ kernel = reconstruct_gptq_3bit_kernel;
1616
+ gridDim.y = DIVIDE(height, 32);
1617
+ }
1618
+
1619
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
646
- reconstruct_gptq_kernel<<<gridDim, blockDim, 0, stream>>>
1620
+ kernel<<<gridDim, blockDim, 0, stream>>>
1621
(
1622
b_q_weight,
1623
b_gptq_scales,
@@ -671,19 +1645,27 @@ void gemm_half_q_half_cuda
1645
int size_n,
1646
int size_k,
1647
int groups,
674
- bool use_exllama
1648
+ bool use_exllama,
1649
+ int bit
1650
)
1651
{
677
- if ((use_exllama && size_m > MAX_Q_GEMM_ROWS) || (!use_exllama && size_m > MAX_ALT_GEMM_ROWS)) {
1652
+ bool use_reconstruct;
1653
+ if (use_exllama) {
1654
+ use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) || (bit != 8 && size_m > MAX_Q_GEMM_ROWS));
1655
+ } else {
1656
+ // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so we disabled them for now.
1657
+ use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS);
1658
+ }
1659
+ if (use_reconstruct) {
1660
// Reconstruct FP16 matrix, then cuBLAS
1661
if (use_exllama) {
1662
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq,
681
- size_k, size_n, groups);
1663
+ size_k, size_n, groups, bit);
1664
}
1665
else
1666
{
1667
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
686
- temp_dq, size_k, size_n, groups);
1668
+ temp_dq, size_k, size_n, groups, bit);
1669
}
1670
1671
const half alpha = __float2half(1.0f);
@@ -707,7 +1689,7 @@ void gemm_half_q_half_cuda
1689
{
1690
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1691
c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
710
- groups);
1692
+ groups, bit);
1693
}
1694
1695
if (last_chunk_size)
@@ -715,18 +1697,17 @@ void gemm_half_q_half_cuda
1697
gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight, b_gptq_qzeros,
1698
b_gptq_scales, b_g_idx, c + last_chunk * size_n,
1699
last_chunk_size, size_n, size_k, last_chunk_size,
718
- groups);
1700
+ groups, bit);
1701
}
1702
}
1703
else
1704
{
1705
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
724
- c, size_m, size_n, size_k);
1706
+ c, size_m, size_n, size_k, bit);
1707
}
1708
}
1709
728
-
729
-__global__ void shuffle_kernel
1710
+__global__ void shuffle_4bit_kernel
1711
(
1712
uint32_t* __restrict__ b_q_weight,
1713
const int size_k,
@@ -740,13 +1721,53 @@ __global__ void shuffle_kernel
1721
while (k < size_k) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
1722
}
1723
1724
+__global__ void shuffle_8bit_kernel
1725
+(
1726
+ uint32_t* __restrict__ b_q_weight,
1727
+ const int size_k,
1728
+ const int size_n
1729
+)
1730
+{
1731
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1732
+ if (n >= size_n) return;
1733
+ int k = 0;
1734
+ uint32_t* b_ptr = b_q_weight + n;
1735
+ while (k < size_k) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; }
1736
+}
1737
+
1738
+__global__ void shuffle_2bit_kernel
1739
+(
1740
+ uint32_t* __restrict__ b_q_weight,
1741
+ const int size_k,
1742
+ const int size_n
1743
+)
1744
+{
1745
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1746
+ if (n >= size_n) return;
1747
+ int k = 0;
1748
+ uint32_t* b_ptr = b_q_weight + n;
1749
+ while (k < size_k) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; }
1750
+}
1751
+
1752
+__global__ void shuffle_3bit_kernel
1753
+(
1754
+ uint32_t* __restrict__ b_q_weight,
1755
+ const int size_k,
1756
+ const int size_n
1757
+)
1758
+{
1759
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1760
+ if (n >= size_n) return;
1761
+ int k = 0;
1762
+ uint32_t* b_ptr = b_q_weight + n;
1763
+ while (k < size_k) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; }
1764
+}
1765
744
-__global__ void make_sequential_kernel
1766
+__global__ void make_sequential_4bit_kernel
1767
(
1768
const uint32_t* __restrict__ w,
1769
uint32_t* __restrict__ w_new,
1770
const int* __restrict__ q_perm,
749
- const int w_height,
1771
const int w_width
1772
)
1773
{
@@ -778,37 +1799,204 @@ __global__ void make_sequential_kernel
1799
w_new2[w_new2_row * w2_stride + w2_column] = dst;
1800
}
1801
1802
+__global__ void make_sequential_2bit_kernel
1803
+(
1804
+ const uint32_t* __restrict__ w,
1805
+ uint32_t* __restrict__ w_new,
1806
+ const int* __restrict__ q_perm,
1807
+ const int w_width
1808
+)
1809
+{
1810
+ const uint64_t* w2 = (uint64_t*) w;
1811
+ uint64_t* w_new2 = (uint64_t*) w_new;
1812
+ int w2_stride = w_width >> 1;
1813
+ int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1814
+ if (w2_column >= w2_stride) return;
1815
+ int w_new2_row = blockIdx.y;
1816
+ int q_perm_idx = w_new2_row << 4;
1817
+ uint64_t dst = 0;
1818
+
1819
+ #pragma unroll
1820
+ for (int i = 0; i < 16; i++)
1821
+ {
1822
+ int source_row = q_perm[q_perm_idx++];
1823
+
1824
+ int w2_row = source_row >> 4;
1825
+ int w2_subrow = source_row & 0x0f;
1826
+ int w2_row_shift = w2_subrow << 1;
1827
+ int wnew2_row_shift = i << 1;
1828
+
1829
+ uint64_t src = w2[w2_row * w2_stride + w2_column];
1830
+ src >>= w2_row_shift;
1831
+ src &= 0x0000000300000003;
1832
+ src <<= wnew2_row_shift;
1833
+ dst |= src;
1834
+ }
1835
+ w_new2[w_new2_row * w2_stride + w2_column] = dst;
1836
+}
1837
+
1838
+__global__ void make_sequential_3bit_kernel
1839
+(
1840
+ const uint32_t* __restrict__ w,
1841
+ uint32_t* __restrict__ w_new,
1842
+ const int* __restrict__ q_perm,
1843
+ const int w_width
1844
+)
1845
+{
1846
+ int w_column = THREADS_X * blockIdx.x + threadIdx.x;
1847
+ if (w_column >= w_width) return;
1848
+ int w_new_row = blockIdx.y * 3;
1849
+ int q_perm_idx = blockIdx.y << 5;
1850
+ uint32_t dst[3] = {0, 0, 0};
1851
+
1852
+ #pragma unroll
1853
+ for (int i = 0; i < 32; i++)
1854
+ {
1855
+ int source_row = q_perm[q_perm_idx++];
1856
+ int z_w = (source_row / 32) * 3;
1857
+ int z_mod = source_row % 32;
1858
+ int z_bit;
1859
+
1860
+ if (z_mod != 10){
1861
+ if (z_mod != 21){
1862
+ z_bit = z_mod;
1863
+ if (z_bit > 21){
1864
+ z_bit *= 3;
1865
+ z_bit -= 64;
1866
+ z_w += 2;
1867
+ } else if (z_bit > 10){
1868
+ z_bit *= 3;
1869
+ z_bit -= 32;
1870
+ z_w += 1;
1871
+ } else {
1872
+ z_bit *= 3;
1873
+ }
1874
+ } else {
1875
+ z_w += 1;
1876
+ }
1877
+ }
1878
+
1879
+ uint64_t src;
1880
+ if (z_mod == 10) {
1881
+ src = (w[z_w * w_width + w_column] >> 30) | ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
1882
+ } else if (z_mod == 21){
1883
+ src = (w[z_w * w_width + w_column] >> 31) | ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
1884
+ } else {
1885
+ src = w[z_w * w_width + w_column];
1886
+ src >>= z_bit;
1887
+ src &= 0x07;
1888
+ }
1889
+
1890
+ z_w = 0;
1891
+ if (i != 10){
1892
+ if (i != 21){
1893
+ z_bit = i;
1894
+ if (z_bit > 21){
1895
+ z_bit *= 3;
1896
+ z_bit -= 64;
1897
+ z_w += 2;
1898
+ } else if (z_bit > 10){
1899
+ z_bit *= 3;
1900
+ z_bit -= 32;
1901
+ z_w += 1;
1902
+ } else {
1903
+ z_bit *= 3;
1904
+ }
1905
+ } else {
1906
+ z_w += 1;
1907
+ }
1908
+ }
1909
+ if (i == 10) {
1910
+ dst[z_w] |= (src & 0x03) << 30;
1911
+ dst[z_w + 1] |= ((src & 0x4) >> 2);
1912
+ } else if (i == 21) {
1913
+ dst[z_w] |= (src & 0x01) << 31;
1914
+ dst[z_w + 1] |= ((src & 0x6) >> 1);
1915
+ } else {
1916
+ dst[z_w] |= (src << z_bit);
1917
+ }
1918
+ }
1919
+ w_new[w_new_row * w_width + w_column] = dst[0];
1920
+ w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
1921
+ w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
1922
+}
1923
+
1924
+__global__ void make_sequential_8bit_kernel
1925
+(
1926
+ const uint32_t* __restrict__ w,
1927
+ uint32_t* __restrict__ w_new,
1928
+ const int* __restrict__ q_perm,
1929
+ const int w_width
1930
+)
1931
+{
1932
+ const uint64_t* w2 = (uint64_t*) w;
1933
+ uint64_t* w_new2 = (uint64_t*) w_new;
1934
+ int w2_stride = w_width >> 1;
1935
+ int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1936
+ if (w2_column >= w2_stride) return;
1937
+ int w_new2_row = blockIdx.y;
1938
+ int q_perm_idx = w_new2_row << 2;
1939
+ uint64_t dst = 0;
1940
+
1941
+ #pragma unroll
1942
+ for (int i = 0; i < 4; i++)
1943
+ {
1944
+ int source_row = q_perm[q_perm_idx++];
1945
+
1946
+ int w2_row = source_row >> 2;
1947
+ int w2_subrow = source_row & 0x03;
1948
+ int w2_row_shift = w2_subrow << 3;
1949
+ int wnew2_row_shift = i << 3;
1950
+
1951
+ uint64_t src = w2[w2_row * w2_stride + w2_column];
1952
+ src >>= w2_row_shift;
1953
+ src &= 0x000000ff000000ff;
1954
+ src <<= wnew2_row_shift;
1955
+ dst |= src;
1956
+ }
1957
+ w_new2[w_new2_row * w2_stride + w2_column] = dst;
1958
+}
1959
+
1960
1961
void shuffle_exllama_weight
1962
(
1963
uint32_t* q_weight,
1964
int* q_perm,
1965
int height,
787
- int width
1966
+ int width,
1967
+ int bit
1968
)
1969
{
1970
if (q_perm)
1971
{
1972
uint32_t* new_qweight = NULL;
793
- cudaMalloc(&new_qweight, height / 8 * width * sizeof(uint32_t));
1973
+ cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
1974
1975
dim3 blockDim, gridDim;
1976
blockDim.x = THREADS_X;
1977
blockDim.y = 1;
1978
gridDim.x = DIVIDE(width, THREADS_X);
799
- gridDim.y = height / 8;
800
-
1979
+ gridDim.y = height / 32 * bit;
1980
+
1981
+ auto kernel = make_sequential_4bit_kernel;
1982
+ if (bit == 2) {
1983
+ kernel = make_sequential_2bit_kernel;
1984
+ } else if (bit == 3) {
1985
+ kernel = make_sequential_3bit_kernel;
1986
+ gridDim.y = height / 32;
1987
+ } else if (bit == 8) {
1988
+ kernel = make_sequential_8bit_kernel;
1989
+ }
1990
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
802
- make_sequential_kernel<<<gridDim, blockDim, 0, stream>>>
1991
+ kernel<<<gridDim, blockDim, 0, stream>>>
1992
(
1993
q_weight,
1994
new_qweight,
1995
q_perm,
807
- height / 8,
1996
width
1997
);
1998
// Replace qweights
811
- cudaMemcpyAsync(q_weight, new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
1999
+ cudaMemcpyAsync(q_weight, new_qweight, height / 32 * bit * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
2000
// Cleanup
2001
cudaDeviceSynchronize();
2002
cudaFree(new_qweight);
@@ -818,6 +2006,14 @@ void shuffle_exllama_weight
2006
blockDim.y = 1;
2007
gridDim.x = DIVIDE(width, THREADS_X);
2008
gridDim.y = 1;
2009
+ auto shuffle_kernel = shuffle_4bit_kernel;
2010
+ if (bit == 2) {
2011
+ shuffle_kernel = shuffle_2bit_kernel;
2012
+ } else if (bit == 3) {
2013
+ shuffle_kernel = shuffle_3bit_kernel;
2014
+ } else if (bit == 8) {
2015
+ shuffle_kernel = shuffle_8bit_kernel;
2016
+ }
2017
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
2018
shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
2019
}
@@ -832,13 +2028,14 @@ torch::Tensor gptq_gemm
2028
torch::Tensor b_gptq_qzeros,
2029
torch::Tensor b_gptq_scales,
2030
torch::Tensor b_g_idx,
835
- bool use_exllama
2031
+ bool use_exllama,
2032
+ int bit
2033
)
2034
{
2035
const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
2036
auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
2037
at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
841
- at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 8, b_q_weight.size(1)}, options);
2038
+ at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options);
2039
2040
vllm::gptq::gemm_half_q_half_cuda
2041
(
@@ -854,7 +2051,8 @@ torch::Tensor gptq_gemm
2051
c.size(1), // n
2052
a.size(1), // k
2053
b_gptq_qzeros.size(0), // group number
857
- use_exllama
2054
+ use_exllama,
2055
+ bit
2056
);
2057
return c;
2058
}
@@ -862,14 +2060,16 @@ torch::Tensor gptq_gemm
2060
void gptq_shuffle
2061
(
2062
torch::Tensor q_weight,
865
- torch::Tensor q_perm
2063
+ torch::Tensor q_perm,
2064
+ int bit
2065
)
2066
{
2067
const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
2068
vllm::gptq::shuffle_exllama_weight(
2069
(uint32_t*) q_weight.data_ptr(),
2070
q_perm.device().is_meta() ? NULL : (int*) q_perm.data_ptr(),
872
- q_weight.size(0) * 8,
873
- q_weight.size(1)
2071
+ q_weight.size(0) * 32 / bit,
2072
+ q_weight.size(1),
2073
+ bit
2074
);
2075
}
csrc/quantization/gptq/qdq_2.cuh
new
+87
@@ -0,0 +1,87 @@
1
+/*
2
+Copied from https://github.com/turboderp/exllamav2
3
+*/
4
+
5
+#ifndef _qdq_2_cuh
6
+#define _qdq_2_cuh
7
+
8
+#include "qdq_util.cuh"
9
+
10
+namespace vllm {
11
+namespace gptq {
12
+
13
+// Permutation:
14
+//
15
+// ffddbb99 77553311 eeccaa88 66442200
16
+
17
+__forceinline__ __device__ void shuffle_2bit_16
18
+(
19
+ uint32_t* q,
20
+ int stride
21
+)
22
+{
23
+ uint32_t qa = q[0];
24
+ uint32_t qb = 0;
25
+
26
+ #pragma unroll
27
+ for (int i = 0; i < 8; i++)
28
+ {
29
+ uint32_t qa0 = qa & 0x03;
30
+ uint32_t qa1 = (qa & 0x0c) >> 2;
31
+ qa >>= 4;
32
+ qb |= (qa1 << (i * 2 + 16));
33
+ qb |= (qa0 << (i * 2));
34
+ }
35
+ q[0] = qb;
36
+}
37
+
38
+__forceinline__ __device__ void dequant_2bit_16
39
+(
40
+ const uint32_t q_0,
41
+ half2 (&dq)[8],
42
+ int stride,
43
+ const uint32_t zero
44
+)
45
+{
46
+ const uint32_t c0 = 0x64006400;
47
+ const half y4_ = __float2half_rn(1.0f / 4.0f);
48
+ const half y16_ = __float2half_rn(1.0f / 16.0f);
49
+ const half y64_ = __float2half_rn(1.0f / 64.0f);
50
+ const half2 y4 = __halves2half2(y4_, y4_);
51
+ const half2 y16 = __halves2half2(y16_, y16_);
52
+ const half2 y64 = __halves2half2(y64_, y64_);
53
+
54
+ const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
55
+ const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero));
56
+ const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
57
+ const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
58
+ const half2 z1 = __half2half2(z1_.as_half);
59
+ const half2 z4 = __half2half2(z4_);
60
+ const half2 z16 = __half2half2(z16_);
61
+ const half2 z64 = __half2half2(z64_);
62
+
63
+ uint32_t qa = q_0;
64
+ half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
65
+ half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
66
+ half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
67
+ half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
68
+ qa >>= 8;
69
+ half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
70
+ half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
71
+ half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
72
+ half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
73
+
74
+ dq[0] = __hadd2(q0.as_half2, z1);
75
+ dq[1] = __hfma2(q1.as_half2, y4, z4);
76
+ dq[2] = __hfma2(q2.as_half2, y16, z16);
77
+ dq[3] = __hfma2(q3.as_half2, y64, z64);
78
+ dq[4] = __hadd2(q4.as_half2, z1);
79
+ dq[5] = __hfma2(q5.as_half2, y4, z4);
80
+ dq[6] = __hfma2(q6.as_half2, y16, z16);
81
+ dq[7] = __hfma2(q7.as_half2, y64, z64);
82
+}
83
+
84
+} // namespace gptq
85
+} // namespace vllm
86
+
87
+#endif
csrc/quantization/gptq/qdq_3.cuh
new
+141
@@ -0,0 +1,141 @@
1
+#ifndef _qdq_3_cuh
2
+#define _qdq_3_cuh
3
+
4
+#include "qdq_util.cuh"
5
+
6
+namespace vllm {
7
+namespace gptq {
8
+// Permutation:
9
+//
10
+// v9997775 55333111 u8886664 44222000 (u, v lsb)
11
+// vjjjhhhf ffdddbbb uiiiggge eecccaaa
12
+// vtttrrrp ppnnnlll usssqqqo oommmkkk
13
+
14
+__forceinline__ __device__ void shuffle_3bit_32
15
+(
16
+ uint32_t* q,
17
+ int stride
18
+)
19
+{
20
+ uint32_t qa = q[0 * stride];
21
+ uint32_t qb = q[1 * stride];
22
+ uint32_t qc = q[2 * stride];
23
+
24
+ // qa: aa999888 77766655 54443332 22111000
25
+ // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
26
+ // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
27
+
28
+ uint32_t qd = qc >> 26;
29
+ qc <<= 4;
30
+ qc |= qb >> 28;
31
+ qb <<= 2;
32
+ qb |= qa >> 30;
33
+
34
+ // qa: ..999888 77766655 54443332 22111000
35
+ // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
36
+ // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
37
+ // qd: vvvuuu
38
+
39
+ uint32_t za = 0;
40
+ uint32_t zb = 0;
41
+ uint32_t zc = 0;
42
+
43
+ for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); }
44
+ for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); }
45
+ for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); }
46
+
47
+ // za: 9997775 55333111 8886664 44222000
48
+ // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
49
+ // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
50
+ // qd: vvvuuu
51
+
52
+ za |= ((qd & 0x01) >> 0) << 15;
53
+ zb |= ((qd & 0x02) >> 1) << 15;
54
+ zc |= ((qd & 0x04) >> 2) << 15;
55
+ za |= ((qd & 0x08) >> 3) << 31;
56
+ zb |= ((qd & 0x10) >> 4) << 31;
57
+ zc |= ((qd & 0x20) >> 5) << 31;
58
+
59
+ // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
60
+ // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
61
+ // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
62
+
63
+ q[0 * stride] = za;
64
+ q[1 * stride] = zb;
65
+ q[2 * stride] = zc;
66
+}
67
+
68
+__forceinline__ __device__ void dequant_3bit_32
69
+(
70
+ const uint32_t q_0,
71
+ const uint32_t q_1,
72
+ const uint32_t q_2,
73
+ half2 (&dq)[16],
74
+ int stride,
75
+ const uint32_t zero
76
+)
77
+{
78
+ const uint32_t c0 = 0x64006400;
79
+ const half y8_ = __float2half_rn(1.0f / 8.0f);
80
+ const half y64_ = __float2half_rn(1.0f / 64.0f);
81
+ const half2 y8 = __halves2half2(y8_, y8_);
82
+ const half2 y64 = __halves2half2(y64_, y64_);
83
+ const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
84
+ const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero));
85
+ const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
86
+ const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half);
87
+ const half2 z8 = __halves2half2(z8_, z8_);
88
+ const half2 z64 = __halves2half2(z64_, z64_);
89
+
90
+ uint32_t qa = q_0;
91
+ uint32_t qb = q_1;
92
+ uint32_t qc = q_2;
93
+
94
+ half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
95
+ half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
96
+ qa >>= 6;
97
+ half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
98
+ half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
99
+ half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
100
+ qa >>= 9;
101
+ qa &= 0x00010001;
102
+ half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
103
+ half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
104
+ qb >>= 6;
105
+ half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
106
+ half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
107
+ half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
108
+ qb >>= 8;
109
+ qb &= 0x00020002;
110
+ half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
111
+ half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
112
+ qc >>= 6;
113
+ half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
114
+ half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
115
+ half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
116
+ qc >>= 7;
117
+ qc &= 0x00040004;
118
+ half2_uint32 q15((qa | qb | qc) | c0);
119
+
120
+ dq[ 0] = __hadd2( q0.as_half2, z1);
121
+ dq[ 1] = __hfma2( q1.as_half2, y8, z8);
122
+ dq[ 2] = __hadd2( q2.as_half2, z1);
123
+ dq[ 3] = __hfma2( q3.as_half2, y8, z8);
124
+ dq[ 4] = __hfma2( q4.as_half2, y64, z64);
125
+ dq[ 5] = __hadd2( q5.as_half2, z1);
126
+ dq[ 6] = __hfma2( q6.as_half2, y8, z8);
127
+ dq[ 7] = __hadd2( q7.as_half2, z1);
128
+ dq[ 8] = __hfma2( q8.as_half2, y8, z8);
129
+ dq[ 9] = __hfma2( q9.as_half2, y64, z64);
130
+ dq[10] = __hadd2(q10.as_half2, z1);
131
+ dq[11] = __hfma2(q11.as_half2, y8, z8);
132
+ dq[12] = __hadd2(q12.as_half2, z1);
133
+ dq[13] = __hfma2(q13.as_half2, y8, z8);
134
+ dq[14] = __hfma2(q14.as_half2, y64, z64);
135
+ dq[15] = __hadd2(q15.as_half2, z1);
136
+}
137
+
138
+} // namespace gptq
139
+} // namespace vllm
140
+
141
+#endif
csrc/quantization/gptq/qdq_4.cuh
+6
-94
@@ -38,16 +38,17 @@ __forceinline__ __device__ void dequant_4bit_8
38
(
39
const uint32_t q_0,
40
half2 (&dq)[4],
41
- int stride
41
+ int stride,
42
+ const uint32_t zero
43
)
44
{
45
const uint32_t c0 = 0x64006400;
46
const half y16_ = __float2half_rn(1.0f / 16.0f);
47
const half2 y16 = __halves2half2(y16_, y16_);
47
- const half z1_ = __float2half_rn(-1024.0f - 8.0f);
48
- const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);
49
- const half2 z1 = __halves2half2(z1_, z1_);
50
- const half2 z16 = __halves2half2(z16_, z16_);
48
+ const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
49
+ const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
50
+ const half2 z1 = __half2half2(z1_.as_half);
51
+ const half2 z16 = __half2half2(z16_);
52
53
uint32_t qa = q_0;
54
half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
@@ -143,93 +144,4 @@ __forceinline__ __device__ void dequant_4bit_8_gptq
144
} // namespace gptq
145
} // namespace vllm
146
146
-#else
147
-
148
-namespace vllm {
149
-namespace gptq {
150
-__forceinline__ __device__ void shuffle_4bit_8
151
-(
152
- uint32_t* q,
153
- int stride
154
-)
155
-{
156
-}
157
-
158
-__forceinline__ __device__ void dequant_4bit_8
159
-(
160
- const uint32_t q_0,
161
- half2 (&dq)[4],
162
- int stride
163
-)
164
-{
165
- half dqh[8];
166
- for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);
167
-
168
- for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
169
-}
170
-
171
-__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
172
-(
173
- const uint32_t zero,
174
- const half scale,
175
- half2 (&z1)[2],
176
- half2 (&y1)[2]
177
-)
178
-{
179
- half z = __int2half_rn(-((int)zero));
180
- z = __hmul(z, scale);
181
- z1[0] = __half2half2(z);
182
- y1[0] = __half2half2(scale);
183
-}
184
-
185
-__forceinline__ __device__ void dequant_4bit_8_prep_zero
186
-(
187
- const uint32_t zero,
188
- half2(&z1)[2],
189
- half2(&y1)[2]
190
-)
191
-{
192
- half z = __int2half_rn(-((int)zero));
193
- z1[0] = __half2half2(z);
194
-}
195
-
196
-__forceinline__ __device__ void dequant_4bit_8_gptq
197
-(
198
- const uint32_t q_0,
199
- half2 (&dq)[4],
200
- half2 (&z1)[2],
201
- half2 (&y1)[2],
202
- int stride,
203
- bool scaled
204
-)
205
-{
206
- half2 dqh2[8];
207
-
208
- uint32_t qa = q_0;
209
- for (int i = 0; i < 4; i++)
210
- {
211
- half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;
212
- half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;
213
- dqh2[i] = __halves2half2(d0, d1);
214
- }
215
-
216
- if (scaled)
217
- {
218
- dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);
219
- dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);
220
- dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);
221
- dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);
222
- }
223
- else
224
- {
225
- dq[0] = __hadd2(dqh2[0], z1[0]);
226
- dq[1] = __hadd2(dqh2[1], z1[0]);
227
- dq[2] = __hadd2(dqh2[2], z1[0]);
228
- dq[3] = __hadd2(dqh2[3], z1[0]);
229
- }
230
-}
231
-
232
-} // namespace gptq
233
-} // namespace vllm
234
-
147
#endif
csrc/quantization/gptq/qdq_8.cuh
new
+40
@@ -0,0 +1,40 @@
1
+/*
2
+Copied from https://github.com/turboderp/exllamav2
3
+*/
4
+
5
+#ifndef _qdq_8_cuh
6
+#define _qdq_8_cuh
7
+
8
+#include "qdq_util.cuh"
9
+
10
+namespace vllm {
11
+namespace gptq {
12
+
13
+__forceinline__ __device__ void shuffle_8bit_4
14
+(
15
+ uint32_t* q,
16
+ int stride
17
+)
18
+{
19
+}
20
+
21
+__forceinline__ __device__ void dequant_8bit_8
22
+(
23
+ const uint32_t q_0,
24
+ const uint32_t q_1,
25
+ half2 (&dq)[4],
26
+ int stride,
27
+ const uint32_t zero
28
+)
29
+{
30
+ half dqh[8];
31
+ for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), zero);
32
+ for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero);
33
+
34
+ for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
35
+}
36
+
37
+} // namespace gptq
38
+} // namespace vllm
39
+
40
+#endif
vllm/model_executor/layers/quantization/gptq.py
+9
-7
@@ -1,6 +1,7 @@
1
import enum
2
from enum import Enum
3
from typing import Any, Dict, List, Optional
4
+from fractions import Fraction
5
6
import torch
7
from torch.nn.parameter import Parameter
@@ -27,11 +28,10 @@ class GPTQConfig(QuantizationConfig):
28
self.weight_bits = weight_bits
29
self.group_size = group_size
30
self.desc_act = desc_act
30
- self.pack_factor = 32 // self.weight_bits
31
- # exllama kernel v1 only supports 4 bit
32
- if self.weight_bits != 4:
31
+ self.pack_factor = Fraction(32, self.weight_bits)
32
+ if self.weight_bits not in [2, 3, 4, 8]:
33
raise ValueError(
34
- "Currently, only 4-bit weight quantization is supported for "
34
+ "Currently, only 2/3/4/8-bit weight quantization is supported for "
35
f"GPTQ, but got {self.weight_bits} bits.")
36
37
def __repr__(self) -> str:
@@ -101,7 +101,7 @@ class GPTQLinearMethod(LinearMethodBase):
101
"The input size is not aligned with the quantized "
102
"weight shape. This can be caused by too large "
103
"tensor parallel size.")
104
- if output_size_per_partition % self.quant_config.pack_factor != 0:
104
+ if output_size_per_partition % self.quant_config.pack_factor.numerator != 0:
105
raise ValueError(
106
"The output size is not aligned with the quantized "
107
"weight shape. This can be caused by too large "
@@ -201,11 +201,13 @@ class GPTQLinearMethod(LinearMethodBase):
201
else:
202
weights["g_idx"] = torch.empty((1, 1), device="meta")
203
weights["exllama_state"] = ExllamaState.READY
204
- ops.gptq_shuffle(weights["qweight"], weights["g_idx"])
204
+ ops.gptq_shuffle(weights["qweight"], weights["g_idx"],
205
+ self.quant_config.weight_bits)
206
output = ops.gptq_gemm(reshaped_x, weights["qweight"],
207
weights["qzeros"], weights["scales"],
208
weights["g_idx"],
208
- weights["exllama_state"] == ExllamaState.READY)
209
+ weights["exllama_state"] == ExllamaState.READY,
210
+ self.quant_config.weight_bits)
211
if bias is not None:
212
output = output + bias
213
return output.reshape(out_shape)