@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01a5d18a53
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add Support for 2/3/8-bit GPTQ Quantization Models (#2330)

CHU Tianxiang committed Feb 29, 2024 at 13:52 UTC 01a5d18a537b65a156cfa1a77706693a24c869c1
8 files changed +1736 -229
csrc/ops.h
+4 -2
@@ -98,11 +98,13 @@ torch::Tensor gptq_gemm(
98 torch::Tensor b_gptq_qzeros,
99 torch::Tensor b_gptq_scales,
100 torch::Tensor b_g_idx,
101 - bool use_exllama);
101 + bool use_exllama,
102 + int bit);
103
104 void gptq_shuffle(
105 torch::Tensor q_weight,
105 - torch::Tensor q_perm);
106 + torch::Tensor q_perm,
107 + int bit);
108
109 void moe_align_block_size(
110 torch::Tensor topk_ids,
csrc/quantization/gptq/matrix_view.cuh
+123
@@ -146,6 +146,129 @@ public:
146 __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; }
147 };
148
149 +class MatrixView_q2_row
150 +{
151 +public:
152 + const uint32_t* data;
153 + const int height;
154 + const int width;
155 +
156 + __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data, const int height, const int width)
157 + : data(data), height(height), width(width)
158 + { }
159 +
160 + __device__ __forceinline__ int item(int row, int column) const
161 + {
162 + int shift = (column & 0x0f) * 2;
163 + return (data[row * width / 16 + column / 16] >> shift) & 0x03;
164 + }
165 +
166 + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
167 + {
168 + int shift = (column & 0x0f) * 2;
169 + uint32_t d = data[row * width / 16 + column / 16] >> shift;
170 + items[0] = d & 0x03;
171 + items[1] = (d >> 2) & 0x03;
172 + }
173 +
174 + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
175 + {
176 + int shift = (column & 0x0f) * 2;
177 + uint32_t d = data[row * width / 16 + column / 16] >> shift;
178 + items[0] = d & 0x03;
179 + items[1] = (d >> 2) & 0x03;
180 + items[2] = (d >> 4) & 0x03;
181 + items[3] = (d >> 6) & 0x03;
182 + }
183 +};
184 +
185 +class MatrixView_q3_row
186 +{
187 +public:
188 + const uint32_t* data;
189 + const int height;
190 + const int width;
191 +
192 + __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data, const int height, const int width)
193 + : data(data), height(height), width(width)
194 + { }
195 +
196 + __device__ __forceinline__ int item(int row, int column) const
197 + {
198 + int z_w = column * 3 / 32;
199 + int z_mod = column & 0x1f;
200 +
201 + if (z_mod == 10) {
202 + return (data[row * width * 3 / 32 + z_w] >> 30) | ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4);
203 + } else if (z_mod == 21) {
204 + return (data[row * width * 3 / 32 + z_w] >> 31) | ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6);
205 + } else if (z_mod < 10) {
206 + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07;
207 + } else if (z_mod < 21) {
208 + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07;
209 + } else {
210 + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07;
211 + }
212 + }
213 +
214 + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
215 + {
216 + int shift = (column & 0x1f);
217 + uint32_t d;
218 + if (shift <= 4) {
219 + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3);
220 + } else if (shift == 8) {
221 + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8);
222 + } else if (shift <= 16) {
223 + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32);
224 + } else if (shift == 20) {
225 + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4);
226 + } else {
227 + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64);
228 + }
229 + items[0] = d & 0x07;
230 + items[1] = (d >> 3) & 0x07;
231 + items[2] = (d >> 6) & 0x07;
232 + items[3] = (d >> 9) & 0x07;
233 + }
234 +};
235 +
236 +class MatrixView_q8_row
237 +{
238 +public:
239 + const uint32_t* data;
240 + const int height;
241 + const int width;
242 +
243 + __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data, const int height, const int width)
244 + : data(data), height(height), width(width)
245 + { }
246 +
247 + __device__ __forceinline__ int item(int row, int column) const
248 + {
249 + int shift = (column & 0x03) * 8;
250 + return (data[row * width / 4 + column / 4] >> shift) & 0xff;
251 + }
252 +
253 + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
254 + {
255 + int shift = (column & 0x03) * 8;
256 + uint32_t d = data[row * width / 4 + column / 4] >> shift;
257 + items[0] = d & 0xff;
258 + items[1] = (d >> 8) & 0xff;
259 + }
260 +
261 + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
262 + {
263 + int shift = (column & 0x03) * 2;
264 + uint32_t d = data[row * width / 4 + column / 4] >> shift;
265 + items[0] = d & 0xff;
266 + items[1] = (d >> 8) & 0xff;
267 + items[2] = (d >> 16) & 0xff;
268 + items[3] = (d >> 24) & 0xff;
269 + }
270 +};
271 +
272 } // namespace gptq
273 } // namespace vllm
274 #endif
csrc/quantization/gptq/q_gemm.cu
+1326 -126
@@ -13,7 +13,10 @@ Adapted from https://github.com/turboderp/exllamav2 and https://github.com/qwopq
13
14 #include "compat.cuh"
15 #include "matrix_view.cuh"
16 +#include "qdq_2.cuh"
17 +#include "qdq_3.cuh"
18 #include "qdq_4.cuh"
19 +#include "qdq_8.cuh"
20
21 namespace vllm {
22 namespace gptq {
@@ -22,6 +25,7 @@ namespace gptq {
25 #define BLOCK_M_SIZE_MAX 8
26 #define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)
27 #define MAX_Q_GEMM_ROWS 50
28 +#define MAX_Q_GEMM_ROWS_8BIT 24
29 #define MAX_ALT_GEMM_ROWS 8
30 #define THREADS_X 32
31 #define THREADS_Y 32
@@ -75,6 +79,106 @@ __forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
79 return __half2float(__low2half(result)) + __half2float(__high2half(result));
80 }
81
82 +__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h)
83 +{
84 + half2 result = {};
85 + const half2* a2_ptr = (const half2*)a_ptr;
86 + #pragma unroll
87 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
88 + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
89 +}
90 +
91 +__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h)
92 +{
93 + half2 result = {};
94 + const half2* a2_ptr = (const half2*)a_ptr;
95 + #pragma unroll
96 + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
97 + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
98 +}
99 +
100 +__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h)
101 +{
102 + half2 result = {};
103 + const half2* a2_ptr = (const half2*)a_ptr;
104 + #pragma unroll
105 + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
106 + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
107 +}
108 +
109 +__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f)
110 +{
111 + half2 result = {};
112 + const half2* a2_ptr = (const half2*)a_ptr;
113 + #pragma unroll
114 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
115 + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
116 + return fma(result_f, qs_f, g_result);
117 +}
118 +
119 +__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f)
120 +{
121 + half2 result = {};
122 + const half2* a2_ptr = (const half2*)a_ptr;
123 + #pragma unroll
124 + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
125 + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
126 + return fma(result_f, qs_f, g_result);
127 +}
128 +
129 +__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f)
130 +{
131 + half2 result = {};
132 + const half2* a2_ptr = (const half2*)a_ptr;
133 + #pragma unroll
134 + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
135 + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
136 + return fma(result_f, qs_f, g_result);
137 +}
138 +
139 +__forceinline__ __device__ half dot22_8_h(half2(&dq)[4], const half* a_ptr, const half g_result, const half qs_h)
140 +{
141 + // Use FP32 accumulator to avoid potential overflow since unscaled weights are in the range -128..127
142 +
143 + float result = {};
144 + #pragma unroll
145 + for (int i = 0; i < 4; i++)
146 + {
147 + half2 w01 = dq[i];
148 + float w0 = __low2float(w01);
149 + float w1 = __high2float(w01);
150 + float x0 = __half2float(*a_ptr++);
151 + float x1 = __half2float(*a_ptr++);
152 + result = fma(w0, x0, result);
153 + result = fma(w1, x1, result);
154 + }
155 + float qs = __half2float(qs_h);
156 + result *= qs;
157 + half result_h = __float2half_rn(result);
158 + return __hadd(result_h, g_result);
159 +}
160 +
161 +__forceinline__ __device__ half dot22_16_h(half2(&dq)[8], const half* a_ptr, const half g_result, const half qs_h)
162 +{
163 + half2 result = {};
164 + const half2* a2_ptr = (const half2*)a_ptr;
165 + #pragma unroll
166 + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
167 + half result_h = __hadd(__low2half(result), __high2half(result));
168 + return __hfma(result_h, qs_h, g_result);
169 +}
170 +
171 +__forceinline__ __device__ half dot22_32_h(half2(&dq)[16], const half* a_ptr, const half g_result, const half qs_h)
172 +{
173 + half2 result = {};
174 + const half2* a2_ptr = (const half2*)a_ptr;
175 + #pragma unroll
176 + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
177 + half result_h = __hadd(__low2half(result), __high2half(result));
178 + return __hfma(result_h, qs_h, g_result);
179 +}
180 +
181 +
182 typedef void (*fp_gemm_half_q_half_gptq_kernel)
183 (
184 const half*,
@@ -89,8 +193,9 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel)
193 const int*
194 );
195
196 +
197 template <bool first_block, int m_count>
93 -__global__ void gemm_half_q_half_gptq_kernel
198 +__global__ void gemm_half_q_half_gptq_4bit_kernel
199 (
200 const half* __restrict__ a,
201 const uint32_t* __restrict__ b_q_weight,
@@ -231,80 +336,794 @@ __global__ void gemm_half_q_half_gptq_kernel
336 }
337 }
338
234 -
235 -fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count)
339 +template <bool first_block, int m_count>
340 +__global__ void gemm_half_q_half_gptq_2bit_kernel
341 +(
342 + const half* __restrict__ a,
343 + const uint32_t* __restrict__ b_q_weight,
344 + const uint32_t* __restrict__ b_gptq_qzeros,
345 + const half* __restrict__ b_gptq_scales,
346 + half* __restrict__ c,
347 + const int size_m,
348 + const int size_n,
349 + const int size_k,
350 + const int groups,
351 + const int* __restrict__ b_q_perm
352 +)
353 {
237 - #if BLOCK_M_SIZE_MAX >= 1
238 - if (m_count == 1) return gemm_half_q_half_gptq_kernel<true, 1>;
239 - #endif
240 - #if BLOCK_M_SIZE_MAX >= 2
241 - if (m_count == 2) return gemm_half_q_half_gptq_kernel<true, 2>;
242 - #endif
243 - #if BLOCK_M_SIZE_MAX >= 3
244 - if (m_count == 3) return gemm_half_q_half_gptq_kernel<true, 3>;
245 - #endif
246 - #if BLOCK_M_SIZE_MAX >= 4
247 - if (m_count == 4) return gemm_half_q_half_gptq_kernel<true, 4>;
248 - #endif
249 - #if BLOCK_M_SIZE_MAX >= 5
250 - if (m_count == 5) return gemm_half_q_half_gptq_kernel<true, 5>;
251 - #endif
252 - #if BLOCK_M_SIZE_MAX >= 6
253 - if (m_count == 6) return gemm_half_q_half_gptq_kernel<true, 6>;
254 - #endif
255 - #if BLOCK_M_SIZE_MAX >= 7
256 - if (m_count == 7) return gemm_half_q_half_gptq_kernel<true, 7>;
257 - #endif
258 - #if BLOCK_M_SIZE_MAX >= 8
259 - if (m_count == 8) return gemm_half_q_half_gptq_kernel<true, 8>;
260 - #endif
261 - return NULL;
262 -}
354 + MatrixView_half a_(a, size_m, size_k);
355 + MatrixView_half_rw c_(c, size_m, size_n);
356 + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
357 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
358
359 + int t = threadIdx.x;
360
265 -void gemm_half_q_half_cuda_part
361 + // Block
362 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
363 + int offset_m = blockIdx.y * m_count;
364 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
365 +
366 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
367 + int end_m = min(offset_m + m_count, size_m);
368 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
369 +
370 + int n = offset_n + t * 4;
371 +
372 + // Preload block_a
373 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
374 +
375 + if (offset_k + t < end_k)
376 + {
377 + for (int m = 0; m < m_count; ++m)
378 + {
379 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
380 + half* block_a_ptr = block_a[m];
381 +
382 + half a0;
383 + if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
384 + else a0 = a_ptr[offset_k + t];
385 + block_a_ptr[t] = a0;
386 + }
387 + }
388 +
389 + // Zero output
390 + if (n >= size_n) return;
391 +
392 + if (blockIdx.z == 0)
393 + {
394 + for (int m = 0; m < m_count; m++)
395 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
396 + }
397 +
398 + __syncthreads();
399 +
400 + // Find initial group
401 + int groupsize = size_k / groups;
402 + int group = offset_k / groupsize;
403 + int nextgroup = offset_k + groupsize;
404 +
405 + // a, b offset
406 + int qk = offset_k / (32 / 2);
407 +
408 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
409 + const half* a_ptr = &block_a[0][0];
410 + int a_stride = BLOCK_KN_SIZE;
411 +
412 + // Initial group
413 + int zeros[4];
414 + half scales[4];
415 + b_gptq_qzeros_.item4(zeros, group, n);
416 + b_gptq_scales_.item4(scales, group, n);
417 + // Column result
418 + half block_c[m_count][4] = {};
419 +
420 + // Dequantize and multiply
421 + int k = offset_k;
422 + while (k < end_k)
423 + {
424 + if (k == nextgroup)
425 + {
426 + group++;
427 + nextgroup += groupsize;
428 + b_gptq_qzeros_.item4(zeros, group, n);
429 + b_gptq_scales_.item4(scales, group, n);
430 + }
431 +
432 + #pragma unroll
433 + for (int j = 0; j < 1; j++)
434 + {
435 + const int4* b_ptr4 = (int4*) b_ptr;
436 + int4 load_int4 = *b_ptr4;
437 +
438 + half2 dq[4][8];
439 + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
440 + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
441 + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
442 + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
443 +
444 + #pragma unroll
445 + for (int m = 0; m < m_count; m++)
446 + {
447 + block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
448 + block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
449 + block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
450 + block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
451 + }
452 +
453 + b_ptr += size_n;
454 + a_ptr += 16;
455 + }
456 +
457 + k += 16;
458 + }
459 +
460 + for (int m = 0; m < m_count; m++)
461 + {
462 + half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
463 + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
464 + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
465 + atomicAdd(out , result01);
466 + atomicAdd(out + 1, result23);
467 + }
468 +}
469 +
470 +template <bool first_block, int m_count>
471 +__global__ void gemm_half_q_half_gptq_3bit_kernel
472 (
267 - const half* a,
268 - const uint32_t* b_q_weight,
269 - const uint32_t* b_gptq_qzeros,
270 - const half* b_gptq_scales,
271 - const int* b_q_perm,
272 - half* c,
273 - int size_m,
274 - int size_n,
275 - int size_k,
276 - int m_count,
277 - int groups
473 + const half* __restrict__ a,
474 + const uint32_t* __restrict__ b_q_weight,
475 + const uint32_t* __restrict__ b_gptq_qzeros,
476 + const half* __restrict__ b_gptq_scales,
477 + half* __restrict__ c,
478 + const int size_m,
479 + const int size_n,
480 + const int size_k,
481 + const int groups,
482 + const int* __restrict__ b_q_perm
483 )
484 {
280 - dim3 blockDim, gridDim;
281 - blockDim.x = BLOCK_KN_SIZE;
282 - blockDim.y = 1;
283 - blockDim.z = 1;
284 - gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
285 - gridDim.y = DIVIDE(size_m, m_count);
286 - gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
485 + MatrixView_half a_(a, size_m, size_k);
486 + MatrixView_half_rw c_(c, size_m, size_n);
487 + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
488 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
489
288 - fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
490 + int t = threadIdx.x;
491
290 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
291 - kernel<<<gridDim, blockDim, 0, stream>>>
292 - (
293 - a,
294 - b_q_weight,
295 - b_gptq_qzeros,
296 - b_gptq_scales,
297 - c,
298 - size_m,
299 - size_n,
300 - size_k,
301 - groups,
302 - b_q_perm
303 - );
304 -}
492 + // Block
493 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
494 + int offset_m = blockIdx.y * m_count;
495 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
496 +
497 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
498 + int end_m = min(offset_m + m_count, size_m);
499 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
500 +
501 + int n = offset_n + t * 4;
502 +
503 + // Preload block_a
504 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
505 +
506 + if (offset_k + t < end_k)
507 + {
508 + for (int m = 0; m < m_count; ++m)
509 + {
510 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
511 + half* block_a_ptr = block_a[m];
512
513 + half a0;
514 + if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
515 + else a0 = a_ptr[offset_k + t];
516 + block_a_ptr[t] = a0;
517 + }
518 + }
519 +
520 + // Zero output
521 + if (n >= size_n) return;
522 +
523 + if (blockIdx.z == 0)
524 + {
525 + for (int m = 0; m < m_count; m++)
526 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
527 + }
528 +
529 + __syncthreads();
530 +
531 + // Find initial group
532 + int groupsize = size_k / groups;
533 + int group = offset_k / groupsize;
534 + int nextgroup = offset_k + groupsize;
535 +
536 + // a, b offset
537 + int qk = offset_k / 32 * 3;
538 +
539 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
540 + const half* a_ptr = &block_a[0][0];
541 + int a_stride = BLOCK_KN_SIZE;
542 +
543 + // Initial group
544 + int zeros[4];
545 + half scales[4];
546 + b_gptq_qzeros_.item4(zeros, group, n);
547 + b_gptq_scales_.item4(scales, group, n);
548 + // Column result
549 + half block_c[m_count][4] = {};
550 +
551 + // Dequantize and multiply
552 + int k = offset_k;
553 + while (k < end_k)
554 + {
555 + if (k == nextgroup)
556 + {
557 + group++;
558 + nextgroup += groupsize;
559 + b_gptq_qzeros_.item4(zeros, group, n);
560 + b_gptq_scales_.item4(scales, group, n);
561 + }
562 +
563 + #pragma unroll
564 + for (int j = 0; j < 1; j++)
565 + {
566 + int4 load_int4[3];
567 + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
568 + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
569 + load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
570 +
571 + half2 dq[4][16];
572 + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
573 + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
574 + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
575 + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
576 +
577 + #pragma unroll
578 + for (int m = 0; m < m_count; m++)
579 + {
580 + block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
581 + block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
582 + block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
583 + block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
584 + }
585 + a_ptr += 32;
586 + }
587 +
588 + k += 32;
589 + }
590 +
591 + for (int m = 0; m < m_count; m++)
592 + {
593 + half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
594 + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
595 + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
596 + atomicAdd(out , result01);
597 + atomicAdd(out + 1, result23);
598 + }
599 +}
600 +
601 +template <bool first_block, int m_count>
602 +__global__ void gemm_half_q_half_gptq_8bit_kernel
603 +(
604 + const half* __restrict__ a,
605 + const uint32_t* __restrict__ b_q_weight,
606 + const uint32_t* __restrict__ b_gptq_qzeros,
607 + const half* __restrict__ b_gptq_scales,
608 + half* __restrict__ c,
609 + const int size_m,
610 + const int size_n,
611 + const int size_k,
612 + const int groups,
613 + const int* __restrict__ b_q_perm
614 +)
615 +{
616 + MatrixView_half a_(a, size_m, size_k);
617 + MatrixView_half_rw c_(c, size_m, size_n);
618 + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
619 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
620 +
621 + int t = threadIdx.x;
622 +
623 + // Block
624 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
625 + int offset_m = blockIdx.y * m_count;
626 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
627 +
628 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
629 + int end_m = min(offset_m + m_count, size_m);
630 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
631 +
632 + int n = offset_n + t * 4;
633 +
634 + // Preload block_a
635 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
636 +
637 + if (offset_k + t < end_k)
638 + {
639 + for (int m = 0; m < m_count; ++m)
640 + {
641 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
642 + half* block_a_ptr = block_a[m];
643 +
644 + half a0;
645 + if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
646 + else a0 = a_ptr[offset_k + t];
647 + block_a_ptr[t] = a0;
648 + }
649 + }
650 +
651 + // Zero output
652 + if (n >= size_n) return;
653 +
654 + if (blockIdx.z == 0)
655 + {
656 + for (int m = 0; m < m_count; m++)
657 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
658 + }
659 +
660 + __syncthreads();
661 +
662 + // Find initial group
663 + int groupsize = size_k / groups;
664 + int group = offset_k / groupsize;
665 + int nextgroup = offset_k + groupsize;
666 +
667 + // a, b offset
668 + int qk = offset_k / (32 / 8);
669 +
670 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
671 + const half* a_ptr = &block_a[0][0];
672 + int a_stride = BLOCK_KN_SIZE;
673 +
674 + // Initial group
675 + int zeros[4];
676 + half scales[4];
677 + b_gptq_qzeros_.item4(zeros, group, n);
678 + b_gptq_scales_.item4(scales, group, n);
679 + // Column result
680 + half block_c[m_count][4] = {};
681 +
682 + // Dequantize and multiply
683 + int k = offset_k;
684 + while (k < end_k)
685 + {
686 + if (k == nextgroup)
687 + {
688 + group++;
689 + nextgroup += groupsize;
690 + b_gptq_qzeros_.item4(zeros, group, n);
691 + b_gptq_scales_.item4(scales, group, n);
692 + }
693 +
694 + #pragma unroll
695 + for (int j = 0; j < 4; j++)
696 + {
697 + int4 load_int4[2];
698 + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
699 + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
700 +
701 + half2 dq[4][4];
702 + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
703 + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
704 + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
705 + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
706 +
707 + for (int m = 0; m < m_count; m++)
708 + {
709 + block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
710 + block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
711 + block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
712 + block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
713 + }
714 + a_ptr += 8;
715 + }
716 + k += 32;
717 + }
718 +
719 + for (int m = 0; m < m_count; m++)
720 + {
721 + half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
722 + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
723 + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
724 + atomicAdd(out , result01);
725 + atomicAdd(out + 1, result23);
726 + }
727 +}
728 +
729 +fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
730 + bool first_block, const int m_count, const int bit)
731 +{
732 + #define SELECT_KERNEL(M_COUNT) \
733 + if (m_count == M_COUNT) { \
734 + if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel<true, M_COUNT>; \
735 + if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel<true, M_COUNT>; \
736 + if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel<true, M_COUNT>; \
737 + if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel<true, M_COUNT>; \
738 + }
739 + #if BLOCK_M_SIZE_MAX >= 1
740 + SELECT_KERNEL(1);
741 + #endif
742 + #if BLOCK_M_SIZE_MAX >= 2
743 + SELECT_KERNEL(2);
744 + #endif
745 + #if BLOCK_M_SIZE_MAX >= 3
746 + SELECT_KERNEL(3);
747 + #endif
748 + #if BLOCK_M_SIZE_MAX >= 4
749 + SELECT_KERNEL(4);
750 + #endif
751 + #if BLOCK_M_SIZE_MAX >= 5
752 + SELECT_KERNEL(5);
753 + #endif
754 + #if BLOCK_M_SIZE_MAX >= 6
755 + SELECT_KERNEL(6);
756 + #endif
757 + #if BLOCK_M_SIZE_MAX >= 7
758 + SELECT_KERNEL(7);
759 + #endif
760 + #if BLOCK_M_SIZE_MAX >= 8
761 + SELECT_KERNEL(8);
762 + #endif
763 + return NULL;
764 +}
765 +
766 +
767 +void gemm_half_q_half_cuda_part
768 +(
769 + const half* a,
770 + const uint32_t* b_q_weight,
771 + const uint32_t* b_gptq_qzeros,
772 + const half* b_gptq_scales,
773 + const int* b_q_perm,
774 + half* c,
775 + int size_m,
776 + int size_n,
777 + int size_k,
778 + int m_count,
779 + int groups,
780 + int bit
781 +)
782 +{
783 + dim3 blockDim, gridDim;
784 + blockDim.x = BLOCK_KN_SIZE;
785 + blockDim.y = 1;
786 + blockDim.z = 1;
787 + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
788 + gridDim.y = DIVIDE(size_m, m_count);
789 + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
790 +
791 + fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
792 +
793 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
794 + kernel<<<gridDim, blockDim, 0, stream>>>
795 + (
796 + a,
797 + b_q_weight,
798 + b_gptq_qzeros,
799 + b_gptq_scales,
800 + c,
801 + size_m,
802 + size_n,
803 + size_k,
804 + groups,
805 + b_q_perm
806 + );
807 +}
808 +
809 +
810 +__global__ void reconstruct_exllama_8bit_kernel
811 +(
812 + const uint32_t* __restrict__ b_q_weight,
813 + const int* __restrict__ b_q_perm,
814 + const uint32_t* __restrict__ b_gptq_qzeros,
815 + const half* __restrict__ b_gptq_scales,
816 + const int size_k,
817 + const int size_n,
818 + const int groups,
819 + half* __restrict__ b
820 +)
821 +{
822 + MatrixView_half_rw b_(b, size_k, size_n);
823 + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
824 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
825 +
826 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
827 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
828 +
829 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
830 +
831 + // Preload remapping table
832 + __shared__ int perm[BLOCK_KN_SIZE];
833 + int t = threadIdx.x;
834 +
835 + if (b_q_perm)
836 + {
837 + if (offset_k + t < size_k)
838 + perm[t] = b_q_perm[offset_k + t];
839 + }
840 +
841 + // Column
842 + int n = offset_n + t * 4;
843 + if (n >= size_n) return;
844 +
845 + // Find initial group
846 + int groupsize = size_k / groups;
847 + int group = offset_k / groupsize;
848 + int nextgroup = offset_k + groupsize;
849 +
850 + // b offset
851 + int qk = offset_k / (32 / 8);
852 +
853 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
854 +
855 + // Initial zeros/scale
856 + int zeros[4];
857 + half2 scales[4];
858 + b_gptq_qzeros_.item4(zeros, group, n);
859 + b_gptq_scales_.item4_h2(scales, group, n);
860 +
861 + __syncthreads();
862 +
863 + int k = offset_k;
864 + int lk = 0;
865 +
866 + while (k < end_k)
867 + {
868 + if (k == nextgroup)
869 + {
870 + group++;
871 + nextgroup += groupsize;
872 + b_gptq_qzeros_.item4(zeros, group, n);
873 + b_gptq_scales_.item4_h2(scales, group, n);
874 + }
875 +
876 + for (int p = 0; p < 4; p++)
877 + {
878 + int4 load_int4[2];
879 + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
880 + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
881 +
882 + half2 dq[4][4];
883 + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
884 + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
885 + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
886 + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
887 +
888 + //half* dqh = (half*)dq;
889 + if (b_q_perm)
890 + {
891 + for (int j = 0; j < 4; j++)
892 + {
893 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
894 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
895 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
896 + }
897 + }
898 + else
899 + {
900 + for (int j = 0; j < 4; j++)
901 + {
902 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
903 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
904 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
905 + }
906 + }
907 + }
908 + k += 32;
909 + }
910 +}
911 +
912 +__global__ void reconstruct_exllama_4bit_kernel
913 +(
914 + const uint32_t* __restrict__ b_q_weight,
915 + const int* __restrict__ b_q_perm,
916 + const uint32_t* __restrict__ b_gptq_qzeros,
917 + const half* __restrict__ b_gptq_scales,
918 + const int size_k,
919 + const int size_n,
920 + const int groups,
921 + half* __restrict__ b
922 +)
923 +{
924 + MatrixView_half_rw b_(b, size_k, size_n);
925 + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
926 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
927 +
928 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
929 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
930 +
931 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
932 +
933 + // Preload remapping table
934 + __shared__ int perm[BLOCK_KN_SIZE];
935 + int t = threadIdx.x;
936 +
937 + if (b_q_perm)
938 + {
939 + if (offset_k + t < size_k)
940 + perm[t] = b_q_perm[offset_k + t];
941 + }
942 +
943 + // Column
944 + int n = offset_n + t * 4;
945 + if (n >= size_n) return;
946 +
947 + // Find initial group
948 + int groupsize = size_k / groups;
949 + int group = offset_k / groupsize;
950 + int nextgroup = offset_k + groupsize;
951 +
952 + // b offset
953 + int qk = offset_k / (32 / 4);
954 +
955 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
956 +
957 + // Initial zeros/scale
958 + int zeros[4];
959 + half2 scales[4];
960 + half2 z1z16[4][2];
961 + half2 y1y16[4][2];
962 + b_gptq_qzeros_.item4(zeros, group, n);
963 + b_gptq_scales_.item4_h2(scales, group, n);
964 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
965 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
966 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
967 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
968 +
969 + __syncthreads();
970 +
971 + int k = offset_k;
972 + int lk = 0;
973 +
974 + while (k < end_k)
975 + {
976 + if (k == nextgroup)
977 + {
978 + group++;
979 + nextgroup += groupsize;
980 + b_gptq_qzeros_.item4(zeros, group, n);
981 + b_gptq_scales_.item4_h2(scales, group, n);
982 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
983 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
984 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
985 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
986 + }
987 +
988 + for (int p = 0; p < 4; p++)
989 + {
990 + half2 dq[4][4];
991 + const int4* b_ptr4 = (int4*) b_ptr;
992 + int4 load_int4 = *b_ptr4;
993 +
994 + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
995 + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
996 + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
997 + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
998 +
999 + b_ptr += size_n;
1000 + //half* dqh = (half*)dq;
1001 + if (b_q_perm)
1002 + {
1003 + for (int j = 0; j < 4; j++)
1004 + {
1005 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1006 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1007 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1008 + }
1009 + }
1010 + else
1011 + {
1012 + for (int j = 0; j < 4; j++)
1013 + {
1014 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1015 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1016 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1017 + }
1018 + }
1019 + }
1020 + k += 32;
1021 + }
1022 +}
1023 +
1024 +__global__ void reconstruct_exllama_3bit_kernel
1025 +(
1026 + const uint32_t* __restrict__ b_q_weight,
1027 + const int* __restrict__ b_q_perm,
1028 + const uint32_t* __restrict__ b_gptq_qzeros,
1029 + const half* __restrict__ b_gptq_scales,
1030 + const int size_k,
1031 + const int size_n,
1032 + const int groups,
1033 + half* __restrict__ b
1034 +)
1035 +{
1036 + MatrixView_half_rw b_(b, size_k, size_n);
1037 + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1038 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1039 +
1040 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1041 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1042 +
1043 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1044 +
1045 + // Preload remapping table
1046 + __shared__ int perm[BLOCK_KN_SIZE];
1047 + int t = threadIdx.x;
1048 +
1049 + if (b_q_perm)
1050 + {
1051 + if (offset_k + t < size_k)
1052 + perm[t] = b_q_perm[offset_k + t];
1053 + }
1054 +
1055 + // Column
1056 + int n = offset_n + t * 4;
1057 + if (n >= size_n) return;
1058 +
1059 + // Find initial group
1060 + int groupsize = size_k / groups;
1061 + int group = offset_k / groupsize;
1062 + int nextgroup = offset_k + groupsize;
1063 +
1064 + // b offset
1065 + int qk = offset_k / 32* 3;
1066 +
1067 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1068 +
1069 + // Initial zeros/scale
1070 + int zeros[4];
1071 + half2 scales[4];
1072 + b_gptq_qzeros_.item4(zeros, group, n);
1073 + b_gptq_scales_.item4_h2(scales, group, n);
1074 +
1075 + __syncthreads();
1076 +
1077 + int k = offset_k;
1078 + int lk = 0;
1079 +
1080 + while (k < end_k)
1081 + {
1082 + if (k == nextgroup)
1083 + {
1084 + group++;
1085 + nextgroup += groupsize;
1086 + b_gptq_qzeros_.item4(zeros, group, n);
1087 + b_gptq_scales_.item4_h2(scales, group, n);
1088 + }
1089 +
1090 + for (int p = 0; p < 1; p++)
1091 + {
1092 + int4 load_int4[3];
1093 + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
1094 + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
1095 + load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
1096 +
1097 + half2 dq[4][16];
1098 + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
1099 + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
1100 + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
1101 + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
1102 +
1103 + if (b_q_perm)
1104 + {
1105 + for (int j = 0; j < 16; j++)
1106 + {
1107 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1108 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1109 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1110 + }
1111 + }
1112 + else
1113 + {
1114 + for (int j = 0; j < 16; j++)
1115 + {
1116 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1117 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1118 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1119 + }
1120 + }
1121 + }
1122 + k += 32;
1123 + }
1124 +}
1125
307 -__global__ void reconstruct_exllama_kernel
1126 +__global__ void reconstruct_exllama_2bit_kernel
1127 (
1128 const uint32_t* __restrict__ b_q_weight,
1129 const int* __restrict__ b_q_perm,
@@ -317,7 +1136,7 @@ __global__ void reconstruct_exllama_kernel
1136 )
1137 {
1138 MatrixView_half_rw b_(b, size_k, size_n);
320 - MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1139 + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1140 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1141
1142 int offset_k = BLOCK_KN_SIZE * blockIdx.y;
@@ -345,21 +1164,15 @@ __global__ void reconstruct_exllama_kernel
1164 int nextgroup = offset_k + groupsize;
1165
1166 // b offset
348 - int qk = offset_k / (32 / 4);
1167 + int qk = offset_k / (32 / 2);
1168
1169 const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1170
1171 // Initial zeros/scale
1172 int zeros[4];
1173 half2 scales[4];
355 - half2 z1z16[4][2];
356 - half2 y1y16[4][2];
1174 b_gptq_qzeros_.item4(zeros, group, n);
1175 b_gptq_scales_.item4_h2(scales, group, n);
359 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
360 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
361 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
362 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
1176
1177 __syncthreads();
1178
@@ -374,28 +1187,24 @@ __global__ void reconstruct_exllama_kernel
1187 nextgroup += groupsize;
1188 b_gptq_qzeros_.item4(zeros, group, n);
1189 b_gptq_scales_.item4_h2(scales, group, n);
377 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
378 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
379 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
380 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
1190 }
1191
383 - for (int p = 0; p < 4; p++)
1192 + for (int p = 0; p < 2; p++)
1193 {
385 - half2 dq[4][4];
1194 const int4* b_ptr4 = (int4*) b_ptr;
1195 int4 load_int4 = *b_ptr4;
1196
389 - dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
390 - dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
391 - dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
392 - dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
1197 + half2 dq[4][8];
1198 + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
1199 + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
1200 + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
1201 + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
1202
1203 b_ptr += size_n;
1204 //half* dqh = (half*)dq;
1205 if (b_q_perm)
1206 {
398 - for (int j = 0; j < 4; j++)
1207 + for (int j = 0; j < 8; j++)
1208 {
1209 for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1210 b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
@@ -404,7 +1213,7 @@ __global__ void reconstruct_exllama_kernel
1213 }
1214 else
1215 {
407 - for (int j = 0; j < 4; j++)
1216 + for (int j = 0; j < 8; j++)
1217 {
1218 for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1219 b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
@@ -416,7 +1225,6 @@ __global__ void reconstruct_exllama_kernel
1225 }
1226 }
1227
419 -
1228 void reconstruct_exllama
1229 (
1230 const uint32_t* b_q_weight,
@@ -426,7 +1234,8 @@ void reconstruct_exllama
1234 half* out,
1235 int height,
1236 int width,
429 - int groups
1237 + int groups,
1238 + int bit
1239 )
1240 {
1241 dim3 blockDim, gridDim;
@@ -435,6 +1244,15 @@ void reconstruct_exllama
1244 gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
1245 gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1246
1247 + auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel;
1248 + if (bit == 2) {
1249 + reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel;
1250 + } else if (bit == 3) {
1251 + reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel;
1252 + } else if (bit == 8) {
1253 + reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel;
1254 + }
1255 +
1256 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1257 reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>
1258 (
@@ -450,7 +1268,7 @@ void reconstruct_exllama
1268 }
1269
1270
453 -__global__ void gemm_half_q_half_alt_kernel(
1271 +__global__ void gemm_half_q_half_alt_4bit_kernel(
1272 const half2* __restrict__ vec,
1273 const uint32_t* __restrict__ mat,
1274 half* __restrict__ mul,
@@ -548,6 +1366,95 @@ __global__ void gemm_half_q_half_alt_kernel(
1366 }
1367
1368
1369 +__global__ void gemm_half_q_half_alt_8bit_kernel(
1370 + const half2* __restrict__ vec,
1371 + const uint32_t* __restrict__ mat,
1372 + half* __restrict__ mul,
1373 + const half* __restrict__ scales,
1374 + const uint32_t* __restrict__ zeros,
1375 + const int* __restrict__ g_idx,
1376 + int batch,
1377 + int height,
1378 + int width
1379 +)
1380 +{
1381 + int zero_width = width / 4;
1382 + int vec_height = height * 2;
1383 + const int blockwidth2 = BLOCK_KN_SIZE / 2;
1384 + int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1385 + int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1386 + int h = BLOCK_KN_SIZE * blockIdx.z / 4;
1387 + int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;
1388 + int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1389 +
1390 + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1391 + if (threadIdx.x < h_end) {
1392 + for (int m = 0; m < b_end; ++m) {
1393 + blockvec[m][threadIdx.x] =
1394 + vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1395 + threadIdx.x];
1396 + }
1397 + }
1398 +
1399 +
1400 + if (blockIdx.z == 0)
1401 + {
1402 + for (int m = 0; m < b_end; m++)
1403 + mul[(b + m) * width + w] = __int2half_rn(0);
1404 + }
1405 + __syncthreads();
1406 +
1407 + int i = width * h + w;
1408 + int g_h = h * 4;
1409 + int k = 0;
1410 + int z_w = w / 4;
1411 + int z_mod = (w % 4) * 8;
1412 + half2 res2;
1413 + half res[BLOCK_M_SIZE_MAX] = {};
1414 +
1415 + unsigned int tmp;
1416 + while (k < h_end) {
1417 + tmp = mat[i];
1418 + half2 scales_tmp[2];
1419 + half2 zeros_tmp[2];
1420 + for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
1421 + int g = g_idx[g_h + (k + tmp_k) * 2];
1422 + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1423 + half scale_f = scales[g * width + w];
1424 + half scale_f2 = scales[g2 * width + w];
1425 + half2 scale = __halves2half2(scale_f, scale_f2);
1426 + half2 zero = __halves2half2(
1427 + __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),
1428 + __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1))
1429 + );
1430 + scales_tmp[tmp_k] = scale;
1431 + zeros_tmp[tmp_k] = zero;
1432 + }
1433 + for (int m = 0; m < b_end; m++) {
1434 +#ifndef USE_ROCM
1435 + res2 = {};
1436 +#else
1437 + res2.x = __half_as_ushort(__float2half(0));
1438 + res2.y = __half_as_ushort(__float2half(0));
1439 +#endif
1440 + half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF), __int2half_rn((tmp >> 8) & 0xFF));
1441 + res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
1442 + half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF), __int2half_rn((tmp >> 24) & 0xFF));
1443 + res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
1444 +#ifndef USE_ROCM
1445 + res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1446 +#else
1447 + res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1448 +#endif
1449 + }
1450 + i += width;
1451 + k += 2;
1452 + }
1453 + for (int m = 0; m < b_end; m++) {
1454 + atomicAdd(&mul[(b + m) * width + w], res[m]);
1455 + }
1456 +}
1457 +
1458 void gemm_half_q_half_alt
1459 (
1460 const half* a,
@@ -558,7 +1465,8 @@ void gemm_half_q_half_alt
1465 half* c,
1466 int size_m,
1467 int size_n,
561 - int size_k
1468 + int size_k,
1469 + int bit
1470 )
1471 {
1472 dim3 blockDim, gridDim;
@@ -569,8 +1477,13 @@ void gemm_half_q_half_alt
1477 gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
1478 gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
1479
1480 + auto kernel = gemm_half_q_half_alt_4bit_kernel;
1481 + if (bit == 8) {
1482 + kernel = gemm_half_q_half_alt_8bit_kernel;
1483 + }
1484 +
1485 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
573 - gemm_half_q_half_alt_kernel<<<gridDim, blockDim, 0, stream>>>
1486 + kernel<<<gridDim, blockDim, 0, stream>>>
1487 (
1488 (const half2*) a,
1489 b_q_weight,
@@ -579,12 +1492,12 @@ void gemm_half_q_half_alt
1492 b_gptq_qzeros,
1493 b_g_idx,
1494 size_m,
582 - size_k / 8,
1495 + size_k / 32 * bit,
1496 size_n
1497 );
1498 }
1499
587 -
1500 +template<class T, int bit>
1501 __global__ void reconstruct_gptq_kernel
1502 (
1503 const uint32_t* __restrict__ w,
@@ -600,30 +1513,79 @@ __global__ void reconstruct_gptq_kernel
1513 // Start of block
1514
1515 int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
603 - int row = blockIdx.y * 8;
1516 + int row = blockIdx.y * 32 / bit;
1517 if (column >= width) return;
1518
1519 // Views
1520
608 - MatrixView_q4_column w_(w, height, width);
1521 MatrixView_half_rw out_(out, height, width);
1522 MatrixView_half w_scales_(w_scales, group, width);
611 - MatrixView_q4_row w_zeros_(w_zeros, group, width);
1523 + T w_zeros_(w_zeros, group, width);
1524
613 - uint32_t w_read = w_.item_uint32_t(row, column);
1525 + uint32_t w_read = w[blockIdx.y * width + column];
1526 half* out_ptr = out_.item_ptr(row, column);
1527
1528 #pragma unroll
617 - for (int s = 0; s < 32; s += 4)
1529 + for (int s = 0; s < 32; s += bit)
1530 {
619 - int group = g_idx[row + s / 4];
1531 + int group = g_idx[row + s / bit];
1532 half w_scale = w_scales_.item(group, column);
1533 uint32_t w_zero = w_zeros_.item(group, column) + 1;
622 - half w_item = __hmul(__int2half_rn((int)((w_read >> s) & 0x0f) - w_zero), w_scale);
1534 + half w_item = __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero), w_scale);
1535 *out_ptr = w_item; out_ptr += out_.width;
1536 }
1537 }
1538
1539 +__global__ void reconstruct_gptq_3bit_kernel
1540 +(
1541 + const uint32_t* __restrict__ w,
1542 + const half* __restrict__ w_scales,
1543 + const uint32_t* __restrict__ w_zeros,
1544 + const int* __restrict__ g_idx,
1545 + const int height,
1546 + const int width,
1547 + const int group,
1548 + half* __restrict__ out
1549 +)
1550 +{
1551 + // Start of block
1552 + int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1553 + int row = blockIdx.y * 32;
1554 + if (column >= width) return;
1555 +
1556 + // Views
1557 +
1558 + MatrixView_half_rw out_(out, height, width);
1559 + MatrixView_half w_scales_(w_scales, group, width);
1560 + MatrixView_q3_row w_zeros_(w_zeros, group, width);
1561 +
1562 + uint32_t w1 = w[(blockIdx.y * 3) * width + column];
1563 + uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];
1564 + uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];
1565 + half* out_ptr = out_.item_ptr(row, column);
1566 +
1567 + #pragma unroll
1568 + for (int i = 0; i < 32; i += 1)
1569 + {
1570 + int group = g_idx[row + i];
1571 + half w_scale = w_scales_.item(group, column);
1572 + uint32_t w_zero = w_zeros_.item(group, column) + 1;
1573 + int w_item;
1574 + if (i == 10) {
1575 + w_item = (w1 >> 30) | ((w2 << 2) & 0x4);
1576 + } else if (i == 21) {
1577 + w_item = (w2 >> 31) | ((w3 << 1) & 0x6);
1578 + } else if (i < 10) {
1579 + w_item = ((w1 >> (i * 3)) & 0x7);
1580 + } else if (i < 21) {
1581 + w_item = ((w2 >> (i * 3 - 32)) & 0x7);
1582 + } else {
1583 + w_item = ((w3 >> (i * 3 - 64)) & 0x7);
1584 + }
1585 + *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale);
1586 + out_ptr += out_.width;
1587 + }
1588 +}
1589
1590 void reconstruct_gptq
1591 (
@@ -634,16 +1596,28 @@ void reconstruct_gptq
1596 half* out,
1597 int height,
1598 int width,
637 - int groups
1599 + int groups,
1600 + int bit
1601 )
1602 {
1603 dim3 blockDim, gridDim;
1604 blockDim.x = BLOCK_KN_SIZE;
1605 blockDim.y = 1;
643 - gridDim.y = DIVIDE(height, 8);
1606 + gridDim.y = DIVIDE(height, 32 / bit);
1607 gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1608 +
1609 + auto kernel = reconstruct_gptq_kernel<MatrixView_q4_row, 4>;
1610 + if (bit == 2) {
1611 + kernel = reconstruct_gptq_kernel<MatrixView_q2_row, 2>;
1612 + } else if (bit == 8) {
1613 + kernel = reconstruct_gptq_kernel<MatrixView_q8_row, 8>;
1614 + } else if (bit == 3) {
1615 + kernel = reconstruct_gptq_3bit_kernel;
1616 + gridDim.y = DIVIDE(height, 32);
1617 + }
1618 +
1619 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
646 - reconstruct_gptq_kernel<<<gridDim, blockDim, 0, stream>>>
1620 + kernel<<<gridDim, blockDim, 0, stream>>>
1621 (
1622 b_q_weight,
1623 b_gptq_scales,
@@ -671,19 +1645,27 @@ void gemm_half_q_half_cuda
1645 int size_n,
1646 int size_k,
1647 int groups,
674 - bool use_exllama
1648 + bool use_exllama,
1649 + int bit
1650 )
1651 {
677 - if ((use_exllama && size_m > MAX_Q_GEMM_ROWS) || (!use_exllama && size_m > MAX_ALT_GEMM_ROWS)) {
1652 + bool use_reconstruct;
1653 + if (use_exllama) {
1654 + use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) || (bit != 8 && size_m > MAX_Q_GEMM_ROWS));
1655 + } else {
1656 + // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so we disabled them for now.
1657 + use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS);
1658 + }
1659 + if (use_reconstruct) {
1660 // Reconstruct FP16 matrix, then cuBLAS
1661 if (use_exllama) {
1662 reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq,
681 - size_k, size_n, groups);
1663 + size_k, size_n, groups, bit);
1664 }
1665 else
1666 {
1667 reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
686 - temp_dq, size_k, size_n, groups);
1668 + temp_dq, size_k, size_n, groups, bit);
1669 }
1670
1671 const half alpha = __float2half(1.0f);
@@ -707,7 +1689,7 @@ void gemm_half_q_half_cuda
1689 {
1690 gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1691 c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
710 - groups);
1692 + groups, bit);
1693 }
1694
1695 if (last_chunk_size)
@@ -715,18 +1697,17 @@ void gemm_half_q_half_cuda
1697 gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight, b_gptq_qzeros,
1698 b_gptq_scales, b_g_idx, c + last_chunk * size_n,
1699 last_chunk_size, size_n, size_k, last_chunk_size,
718 - groups);
1700 + groups, bit);
1701 }
1702 }
1703 else
1704 {
1705 gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
724 - c, size_m, size_n, size_k);
1706 + c, size_m, size_n, size_k, bit);
1707 }
1708 }
1709
728 -
729 -__global__ void shuffle_kernel
1710 +__global__ void shuffle_4bit_kernel
1711 (
1712 uint32_t* __restrict__ b_q_weight,
1713 const int size_k,
@@ -740,13 +1721,53 @@ __global__ void shuffle_kernel
1721 while (k < size_k) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
1722 }
1723
1724 +__global__ void shuffle_8bit_kernel
1725 +(
1726 + uint32_t* __restrict__ b_q_weight,
1727 + const int size_k,
1728 + const int size_n
1729 +)
1730 +{
1731 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1732 + if (n >= size_n) return;
1733 + int k = 0;
1734 + uint32_t* b_ptr = b_q_weight + n;
1735 + while (k < size_k) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; }
1736 +}
1737 +
1738 +__global__ void shuffle_2bit_kernel
1739 +(
1740 + uint32_t* __restrict__ b_q_weight,
1741 + const int size_k,
1742 + const int size_n
1743 +)
1744 +{
1745 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1746 + if (n >= size_n) return;
1747 + int k = 0;
1748 + uint32_t* b_ptr = b_q_weight + n;
1749 + while (k < size_k) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; }
1750 +}
1751 +
1752 +__global__ void shuffle_3bit_kernel
1753 +(
1754 + uint32_t* __restrict__ b_q_weight,
1755 + const int size_k,
1756 + const int size_n
1757 +)
1758 +{
1759 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1760 + if (n >= size_n) return;
1761 + int k = 0;
1762 + uint32_t* b_ptr = b_q_weight + n;
1763 + while (k < size_k) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; }
1764 +}
1765
744 -__global__ void make_sequential_kernel
1766 +__global__ void make_sequential_4bit_kernel
1767 (
1768 const uint32_t* __restrict__ w,
1769 uint32_t* __restrict__ w_new,
1770 const int* __restrict__ q_perm,
749 - const int w_height,
1771 const int w_width
1772 )
1773 {
@@ -778,37 +1799,204 @@ __global__ void make_sequential_kernel
1799 w_new2[w_new2_row * w2_stride + w2_column] = dst;
1800 }
1801
1802 +__global__ void make_sequential_2bit_kernel
1803 +(
1804 + const uint32_t* __restrict__ w,
1805 + uint32_t* __restrict__ w_new,
1806 + const int* __restrict__ q_perm,
1807 + const int w_width
1808 +)
1809 +{
1810 + const uint64_t* w2 = (uint64_t*) w;
1811 + uint64_t* w_new2 = (uint64_t*) w_new;
1812 + int w2_stride = w_width >> 1;
1813 + int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1814 + if (w2_column >= w2_stride) return;
1815 + int w_new2_row = blockIdx.y;
1816 + int q_perm_idx = w_new2_row << 4;
1817 + uint64_t dst = 0;
1818 +
1819 + #pragma unroll
1820 + for (int i = 0; i < 16; i++)
1821 + {
1822 + int source_row = q_perm[q_perm_idx++];
1823 +
1824 + int w2_row = source_row >> 4;
1825 + int w2_subrow = source_row & 0x0f;
1826 + int w2_row_shift = w2_subrow << 1;
1827 + int wnew2_row_shift = i << 1;
1828 +
1829 + uint64_t src = w2[w2_row * w2_stride + w2_column];
1830 + src >>= w2_row_shift;
1831 + src &= 0x0000000300000003;
1832 + src <<= wnew2_row_shift;
1833 + dst |= src;
1834 + }
1835 + w_new2[w_new2_row * w2_stride + w2_column] = dst;
1836 +}
1837 +
1838 +__global__ void make_sequential_3bit_kernel
1839 +(
1840 + const uint32_t* __restrict__ w,
1841 + uint32_t* __restrict__ w_new,
1842 + const int* __restrict__ q_perm,
1843 + const int w_width
1844 +)
1845 +{
1846 + int w_column = THREADS_X * blockIdx.x + threadIdx.x;
1847 + if (w_column >= w_width) return;
1848 + int w_new_row = blockIdx.y * 3;
1849 + int q_perm_idx = blockIdx.y << 5;
1850 + uint32_t dst[3] = {0, 0, 0};
1851 +
1852 + #pragma unroll
1853 + for (int i = 0; i < 32; i++)
1854 + {
1855 + int source_row = q_perm[q_perm_idx++];
1856 + int z_w = (source_row / 32) * 3;
1857 + int z_mod = source_row % 32;
1858 + int z_bit;
1859 +
1860 + if (z_mod != 10){
1861 + if (z_mod != 21){
1862 + z_bit = z_mod;
1863 + if (z_bit > 21){
1864 + z_bit *= 3;
1865 + z_bit -= 64;
1866 + z_w += 2;
1867 + } else if (z_bit > 10){
1868 + z_bit *= 3;
1869 + z_bit -= 32;
1870 + z_w += 1;
1871 + } else {
1872 + z_bit *= 3;
1873 + }
1874 + } else {
1875 + z_w += 1;
1876 + }
1877 + }
1878 +
1879 + uint64_t src;
1880 + if (z_mod == 10) {
1881 + src = (w[z_w * w_width + w_column] >> 30) | ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
1882 + } else if (z_mod == 21){
1883 + src = (w[z_w * w_width + w_column] >> 31) | ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
1884 + } else {
1885 + src = w[z_w * w_width + w_column];
1886 + src >>= z_bit;
1887 + src &= 0x07;
1888 + }
1889 +
1890 + z_w = 0;
1891 + if (i != 10){
1892 + if (i != 21){
1893 + z_bit = i;
1894 + if (z_bit > 21){
1895 + z_bit *= 3;
1896 + z_bit -= 64;
1897 + z_w += 2;
1898 + } else if (z_bit > 10){
1899 + z_bit *= 3;
1900 + z_bit -= 32;
1901 + z_w += 1;
1902 + } else {
1903 + z_bit *= 3;
1904 + }
1905 + } else {
1906 + z_w += 1;
1907 + }
1908 + }
1909 + if (i == 10) {
1910 + dst[z_w] |= (src & 0x03) << 30;
1911 + dst[z_w + 1] |= ((src & 0x4) >> 2);
1912 + } else if (i == 21) {
1913 + dst[z_w] |= (src & 0x01) << 31;
1914 + dst[z_w + 1] |= ((src & 0x6) >> 1);
1915 + } else {
1916 + dst[z_w] |= (src << z_bit);
1917 + }
1918 + }
1919 + w_new[w_new_row * w_width + w_column] = dst[0];
1920 + w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
1921 + w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
1922 +}
1923 +
1924 +__global__ void make_sequential_8bit_kernel
1925 +(
1926 + const uint32_t* __restrict__ w,
1927 + uint32_t* __restrict__ w_new,
1928 + const int* __restrict__ q_perm,
1929 + const int w_width
1930 +)
1931 +{
1932 + const uint64_t* w2 = (uint64_t*) w;
1933 + uint64_t* w_new2 = (uint64_t*) w_new;
1934 + int w2_stride = w_width >> 1;
1935 + int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1936 + if (w2_column >= w2_stride) return;
1937 + int w_new2_row = blockIdx.y;
1938 + int q_perm_idx = w_new2_row << 2;
1939 + uint64_t dst = 0;
1940 +
1941 + #pragma unroll
1942 + for (int i = 0; i < 4; i++)
1943 + {
1944 + int source_row = q_perm[q_perm_idx++];
1945 +
1946 + int w2_row = source_row >> 2;
1947 + int w2_subrow = source_row & 0x03;
1948 + int w2_row_shift = w2_subrow << 3;
1949 + int wnew2_row_shift = i << 3;
1950 +
1951 + uint64_t src = w2[w2_row * w2_stride + w2_column];
1952 + src >>= w2_row_shift;
1953 + src &= 0x000000ff000000ff;
1954 + src <<= wnew2_row_shift;
1955 + dst |= src;
1956 + }
1957 + w_new2[w_new2_row * w2_stride + w2_column] = dst;
1958 +}
1959 +
1960
1961 void shuffle_exllama_weight
1962 (
1963 uint32_t* q_weight,
1964 int* q_perm,
1965 int height,
787 - int width
1966 + int width,
1967 + int bit
1968 )
1969 {
1970 if (q_perm)
1971 {
1972 uint32_t* new_qweight = NULL;
793 - cudaMalloc(&new_qweight, height / 8 * width * sizeof(uint32_t));
1973 + cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
1974
1975 dim3 blockDim, gridDim;
1976 blockDim.x = THREADS_X;
1977 blockDim.y = 1;
1978 gridDim.x = DIVIDE(width, THREADS_X);
799 - gridDim.y = height / 8;
800 -
1979 + gridDim.y = height / 32 * bit;
1980 +
1981 + auto kernel = make_sequential_4bit_kernel;
1982 + if (bit == 2) {
1983 + kernel = make_sequential_2bit_kernel;
1984 + } else if (bit == 3) {
1985 + kernel = make_sequential_3bit_kernel;
1986 + gridDim.y = height / 32;
1987 + } else if (bit == 8) {
1988 + kernel = make_sequential_8bit_kernel;
1989 + }
1990 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
802 - make_sequential_kernel<<<gridDim, blockDim, 0, stream>>>
1991 + kernel<<<gridDim, blockDim, 0, stream>>>
1992 (
1993 q_weight,
1994 new_qweight,
1995 q_perm,
807 - height / 8,
1996 width
1997 );
1998 // Replace qweights
811 - cudaMemcpyAsync(q_weight, new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
1999 + cudaMemcpyAsync(q_weight, new_qweight, height / 32 * bit * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
2000 // Cleanup
2001 cudaDeviceSynchronize();
2002 cudaFree(new_qweight);
@@ -818,6 +2006,14 @@ void shuffle_exllama_weight
2006 blockDim.y = 1;
2007 gridDim.x = DIVIDE(width, THREADS_X);
2008 gridDim.y = 1;
2009 + auto shuffle_kernel = shuffle_4bit_kernel;
2010 + if (bit == 2) {
2011 + shuffle_kernel = shuffle_2bit_kernel;
2012 + } else if (bit == 3) {
2013 + shuffle_kernel = shuffle_3bit_kernel;
2014 + } else if (bit == 8) {
2015 + shuffle_kernel = shuffle_8bit_kernel;
2016 + }
2017 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
2018 shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
2019 }
@@ -832,13 +2028,14 @@ torch::Tensor gptq_gemm
2028 torch::Tensor b_gptq_qzeros,
2029 torch::Tensor b_gptq_scales,
2030 torch::Tensor b_g_idx,
835 - bool use_exllama
2031 + bool use_exllama,
2032 + int bit
2033 )
2034 {
2035 const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
2036 auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
2037 at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
841 - at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 8, b_q_weight.size(1)}, options);
2038 + at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options);
2039
2040 vllm::gptq::gemm_half_q_half_cuda
2041 (
@@ -854,7 +2051,8 @@ torch::Tensor gptq_gemm
2051 c.size(1), // n
2052 a.size(1), // k
2053 b_gptq_qzeros.size(0), // group number
857 - use_exllama
2054 + use_exllama,
2055 + bit
2056 );
2057 return c;
2058 }
@@ -862,14 +2060,16 @@ torch::Tensor gptq_gemm
2060 void gptq_shuffle
2061 (
2062 torch::Tensor q_weight,
865 - torch::Tensor q_perm
2063 + torch::Tensor q_perm,
2064 + int bit
2065 )
2066 {
2067 const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
2068 vllm::gptq::shuffle_exllama_weight(
2069 (uint32_t*) q_weight.data_ptr(),
2070 q_perm.device().is_meta() ? NULL : (int*) q_perm.data_ptr(),
872 - q_weight.size(0) * 8,
873 - q_weight.size(1)
2071 + q_weight.size(0) * 32 / bit,
2072 + q_weight.size(1),
2073 + bit
2074 );
2075 }
csrc/quantization/gptq/qdq_2.cuh new
+87
@@ -0,0 +1,87 @@
1 +/*
2 +Copied from https://github.com/turboderp/exllamav2
3 +*/
4 +
5 +#ifndef _qdq_2_cuh
6 +#define _qdq_2_cuh
7 +
8 +#include "qdq_util.cuh"
9 +
10 +namespace vllm {
11 +namespace gptq {
12 +
13 +// Permutation:
14 +//
15 +// ffddbb99 77553311 eeccaa88 66442200
16 +
17 +__forceinline__ __device__ void shuffle_2bit_16
18 +(
19 + uint32_t* q,
20 + int stride
21 +)
22 +{
23 + uint32_t qa = q[0];
24 + uint32_t qb = 0;
25 +
26 + #pragma unroll
27 + for (int i = 0; i < 8; i++)
28 + {
29 + uint32_t qa0 = qa & 0x03;
30 + uint32_t qa1 = (qa & 0x0c) >> 2;
31 + qa >>= 4;
32 + qb |= (qa1 << (i * 2 + 16));
33 + qb |= (qa0 << (i * 2));
34 + }
35 + q[0] = qb;
36 +}
37 +
38 +__forceinline__ __device__ void dequant_2bit_16
39 +(
40 + const uint32_t q_0,
41 + half2 (&dq)[8],
42 + int stride,
43 + const uint32_t zero
44 +)
45 +{
46 + const uint32_t c0 = 0x64006400;
47 + const half y4_ = __float2half_rn(1.0f / 4.0f);
48 + const half y16_ = __float2half_rn(1.0f / 16.0f);
49 + const half y64_ = __float2half_rn(1.0f / 64.0f);
50 + const half2 y4 = __halves2half2(y4_, y4_);
51 + const half2 y16 = __halves2half2(y16_, y16_);
52 + const half2 y64 = __halves2half2(y64_, y64_);
53 +
54 + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
55 + const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero));
56 + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
57 + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
58 + const half2 z1 = __half2half2(z1_.as_half);
59 + const half2 z4 = __half2half2(z4_);
60 + const half2 z16 = __half2half2(z16_);
61 + const half2 z64 = __half2half2(z64_);
62 +
63 + uint32_t qa = q_0;
64 + half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
65 + half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
66 + half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
67 + half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
68 + qa >>= 8;
69 + half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
70 + half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
71 + half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
72 + half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
73 +
74 + dq[0] = __hadd2(q0.as_half2, z1);
75 + dq[1] = __hfma2(q1.as_half2, y4, z4);
76 + dq[2] = __hfma2(q2.as_half2, y16, z16);
77 + dq[3] = __hfma2(q3.as_half2, y64, z64);
78 + dq[4] = __hadd2(q4.as_half2, z1);
79 + dq[5] = __hfma2(q5.as_half2, y4, z4);
80 + dq[6] = __hfma2(q6.as_half2, y16, z16);
81 + dq[7] = __hfma2(q7.as_half2, y64, z64);
82 +}
83 +
84 +} // namespace gptq
85 +} // namespace vllm
86 +
87 +#endif
csrc/quantization/gptq/qdq_3.cuh new
+141
@@ -0,0 +1,141 @@
1 +#ifndef _qdq_3_cuh
2 +#define _qdq_3_cuh
3 +
4 +#include "qdq_util.cuh"
5 +
6 +namespace vllm {
7 +namespace gptq {
8 +// Permutation:
9 +//
10 +// v9997775 55333111 u8886664 44222000 (u, v lsb)
11 +// vjjjhhhf ffdddbbb uiiiggge eecccaaa
12 +// vtttrrrp ppnnnlll usssqqqo oommmkkk
13 +
14 +__forceinline__ __device__ void shuffle_3bit_32
15 +(
16 + uint32_t* q,
17 + int stride
18 +)
19 +{
20 + uint32_t qa = q[0 * stride];
21 + uint32_t qb = q[1 * stride];
22 + uint32_t qc = q[2 * stride];
23 +
24 + // qa: aa999888 77766655 54443332 22111000
25 + // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
26 + // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
27 +
28 + uint32_t qd = qc >> 26;
29 + qc <<= 4;
30 + qc |= qb >> 28;
31 + qb <<= 2;
32 + qb |= qa >> 30;
33 +
34 + // qa: ..999888 77766655 54443332 22111000
35 + // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
36 + // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
37 + // qd: vvvuuu
38 +
39 + uint32_t za = 0;
40 + uint32_t zb = 0;
41 + uint32_t zc = 0;
42 +
43 + for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); }
44 + for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); }
45 + for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); }
46 +
47 + // za: 9997775 55333111 8886664 44222000
48 + // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
49 + // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
50 + // qd: vvvuuu
51 +
52 + za |= ((qd & 0x01) >> 0) << 15;
53 + zb |= ((qd & 0x02) >> 1) << 15;
54 + zc |= ((qd & 0x04) >> 2) << 15;
55 + za |= ((qd & 0x08) >> 3) << 31;
56 + zb |= ((qd & 0x10) >> 4) << 31;
57 + zc |= ((qd & 0x20) >> 5) << 31;
58 +
59 + // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
60 + // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
61 + // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
62 +
63 + q[0 * stride] = za;
64 + q[1 * stride] = zb;
65 + q[2 * stride] = zc;
66 +}
67 +
68 +__forceinline__ __device__ void dequant_3bit_32
69 +(
70 + const uint32_t q_0,
71 + const uint32_t q_1,
72 + const uint32_t q_2,
73 + half2 (&dq)[16],
74 + int stride,
75 + const uint32_t zero
76 +)
77 +{
78 + const uint32_t c0 = 0x64006400;
79 + const half y8_ = __float2half_rn(1.0f / 8.0f);
80 + const half y64_ = __float2half_rn(1.0f / 64.0f);
81 + const half2 y8 = __halves2half2(y8_, y8_);
82 + const half2 y64 = __halves2half2(y64_, y64_);
83 + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
84 + const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero));
85 + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
86 + const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half);
87 + const half2 z8 = __halves2half2(z8_, z8_);
88 + const half2 z64 = __halves2half2(z64_, z64_);
89 +
90 + uint32_t qa = q_0;
91 + uint32_t qb = q_1;
92 + uint32_t qc = q_2;
93 +
94 + half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
95 + half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
96 + qa >>= 6;
97 + half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
98 + half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
99 + half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
100 + qa >>= 9;
101 + qa &= 0x00010001;
102 + half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
103 + half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
104 + qb >>= 6;
105 + half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
106 + half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
107 + half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
108 + qb >>= 8;
109 + qb &= 0x00020002;
110 + half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
111 + half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
112 + qc >>= 6;
113 + half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
114 + half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
115 + half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
116 + qc >>= 7;
117 + qc &= 0x00040004;
118 + half2_uint32 q15((qa | qb | qc) | c0);
119 +
120 + dq[ 0] = __hadd2( q0.as_half2, z1);
121 + dq[ 1] = __hfma2( q1.as_half2, y8, z8);
122 + dq[ 2] = __hadd2( q2.as_half2, z1);
123 + dq[ 3] = __hfma2( q3.as_half2, y8, z8);
124 + dq[ 4] = __hfma2( q4.as_half2, y64, z64);
125 + dq[ 5] = __hadd2( q5.as_half2, z1);
126 + dq[ 6] = __hfma2( q6.as_half2, y8, z8);
127 + dq[ 7] = __hadd2( q7.as_half2, z1);
128 + dq[ 8] = __hfma2( q8.as_half2, y8, z8);
129 + dq[ 9] = __hfma2( q9.as_half2, y64, z64);
130 + dq[10] = __hadd2(q10.as_half2, z1);
131 + dq[11] = __hfma2(q11.as_half2, y8, z8);
132 + dq[12] = __hadd2(q12.as_half2, z1);
133 + dq[13] = __hfma2(q13.as_half2, y8, z8);
134 + dq[14] = __hfma2(q14.as_half2, y64, z64);
135 + dq[15] = __hadd2(q15.as_half2, z1);
136 +}
137 +
138 +} // namespace gptq
139 +} // namespace vllm
140 +
141 +#endif
csrc/quantization/gptq/qdq_4.cuh
+6 -94
@@ -38,16 +38,17 @@ __forceinline__ __device__ void dequant_4bit_8
38 (
39 const uint32_t q_0,
40 half2 (&dq)[4],
41 - int stride
41 + int stride,
42 + const uint32_t zero
43 )
44 {
45 const uint32_t c0 = 0x64006400;
46 const half y16_ = __float2half_rn(1.0f / 16.0f);
47 const half2 y16 = __halves2half2(y16_, y16_);
47 - const half z1_ = __float2half_rn(-1024.0f - 8.0f);
48 - const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);
49 - const half2 z1 = __halves2half2(z1_, z1_);
50 - const half2 z16 = __halves2half2(z16_, z16_);
48 + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
49 + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
50 + const half2 z1 = __half2half2(z1_.as_half);
51 + const half2 z16 = __half2half2(z16_);
52
53 uint32_t qa = q_0;
54 half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
@@ -143,93 +144,4 @@ __forceinline__ __device__ void dequant_4bit_8_gptq
144 } // namespace gptq
145 } // namespace vllm
146
146 -#else
147 -
148 -namespace vllm {
149 -namespace gptq {
150 -__forceinline__ __device__ void shuffle_4bit_8
151 -(
152 - uint32_t* q,
153 - int stride
154 -)
155 -{
156 -}
157 -
158 -__forceinline__ __device__ void dequant_4bit_8
159 -(
160 - const uint32_t q_0,
161 - half2 (&dq)[4],
162 - int stride
163 -)
164 -{
165 - half dqh[8];
166 - for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);
167 -
168 - for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
169 -}
170 -
171 -__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
172 -(
173 - const uint32_t zero,
174 - const half scale,
175 - half2 (&z1)[2],
176 - half2 (&y1)[2]
177 -)
178 -{
179 - half z = __int2half_rn(-((int)zero));
180 - z = __hmul(z, scale);
181 - z1[0] = __half2half2(z);
182 - y1[0] = __half2half2(scale);
183 -}
184 -
185 -__forceinline__ __device__ void dequant_4bit_8_prep_zero
186 -(
187 - const uint32_t zero,
188 - half2(&z1)[2],
189 - half2(&y1)[2]
190 -)
191 -{
192 - half z = __int2half_rn(-((int)zero));
193 - z1[0] = __half2half2(z);
194 -}
195 -
196 -__forceinline__ __device__ void dequant_4bit_8_gptq
197 -(
198 - const uint32_t q_0,
199 - half2 (&dq)[4],
200 - half2 (&z1)[2],
201 - half2 (&y1)[2],
202 - int stride,
203 - bool scaled
204 -)
205 -{
206 - half2 dqh2[8];
207 -
208 - uint32_t qa = q_0;
209 - for (int i = 0; i < 4; i++)
210 - {
211 - half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;
212 - half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;
213 - dqh2[i] = __halves2half2(d0, d1);
214 - }
215 -
216 - if (scaled)
217 - {
218 - dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);
219 - dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);
220 - dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);
221 - dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);
222 - }
223 - else
224 - {
225 - dq[0] = __hadd2(dqh2[0], z1[0]);
226 - dq[1] = __hadd2(dqh2[1], z1[0]);
227 - dq[2] = __hadd2(dqh2[2], z1[0]);
228 - dq[3] = __hadd2(dqh2[3], z1[0]);
229 - }
230 -}
231 -
232 -} // namespace gptq
233 -} // namespace vllm
234 -
147 #endif
csrc/quantization/gptq/qdq_8.cuh new
+40
@@ -0,0 +1,40 @@
1 +/*
2 +Copied from https://github.com/turboderp/exllamav2
3 +*/
4 +
5 +#ifndef _qdq_8_cuh
6 +#define _qdq_8_cuh
7 +
8 +#include "qdq_util.cuh"
9 +
10 +namespace vllm {
11 +namespace gptq {
12 +
13 +__forceinline__ __device__ void shuffle_8bit_4
14 +(
15 + uint32_t* q,
16 + int stride
17 +)
18 +{
19 +}
20 +
21 +__forceinline__ __device__ void dequant_8bit_8
22 +(
23 + const uint32_t q_0,
24 + const uint32_t q_1,
25 + half2 (&dq)[4],
26 + int stride,
27 + const uint32_t zero
28 +)
29 +{
30 + half dqh[8];
31 + for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), zero);
32 + for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero);
33 +
34 + for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
35 +}
36 +
37 +} // namespace gptq
38 +} // namespace vllm
39 +
40 +#endif
vllm/model_executor/layers/quantization/gptq.py
+9 -7
@@ -1,6 +1,7 @@
1 import enum
2 from enum import Enum
3 from typing import Any, Dict, List, Optional
4 +from fractions import Fraction
5
6 import torch
7 from torch.nn.parameter import Parameter
@@ -27,11 +28,10 @@ class GPTQConfig(QuantizationConfig):
28 self.weight_bits = weight_bits
29 self.group_size = group_size
30 self.desc_act = desc_act
30 - self.pack_factor = 32 // self.weight_bits
31 - # exllama kernel v1 only supports 4 bit
32 - if self.weight_bits != 4:
31 + self.pack_factor = Fraction(32, self.weight_bits)
32 + if self.weight_bits not in [2, 3, 4, 8]:
33 raise ValueError(
34 - "Currently, only 4-bit weight quantization is supported for "
34 + "Currently, only 2/3/4/8-bit weight quantization is supported for "
35 f"GPTQ, but got {self.weight_bits} bits.")
36
37 def __repr__(self) -> str:
@@ -101,7 +101,7 @@ class GPTQLinearMethod(LinearMethodBase):
101 "The input size is not aligned with the quantized "
102 "weight shape. This can be caused by too large "
103 "tensor parallel size.")
104 - if output_size_per_partition % self.quant_config.pack_factor != 0:
104 + if output_size_per_partition % self.quant_config.pack_factor.numerator != 0:
105 raise ValueError(
106 "The output size is not aligned with the quantized "
107 "weight shape. This can be caused by too large "
@@ -201,11 +201,13 @@ class GPTQLinearMethod(LinearMethodBase):
201 else:
202 weights["g_idx"] = torch.empty((1, 1), device="meta")
203 weights["exllama_state"] = ExllamaState.READY
204 - ops.gptq_shuffle(weights["qweight"], weights["g_idx"])
204 + ops.gptq_shuffle(weights["qweight"], weights["g_idx"],
205 + self.quant_config.weight_bits)
206 output = ops.gptq_gemm(reshaped_x, weights["qweight"],
207 weights["qzeros"], weights["scales"],
208 weights["g_idx"],
208 - weights["exllama_state"] == ExllamaState.READY)
209 + weights["exllama_state"] == ExllamaState.READY,
210 + self.quant_config.weight_bits)
211 if bias is not None:
212 output = output + bias
213 return output.reshape(out_shape)