@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7038e8b803
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Support running GPTQ 8-bit models in Marlin (#4533)

alexm-nm committed May 2, 2024 at 12:56 UTC 7038e8b80303bf6128acbe508dec910183a1be56
7 files changed +553 -324
csrc/ops.h
+3 -1
@@ -132,6 +132,7 @@ torch::Tensor gptq_marlin_gemm(
132 torch::Tensor &g_idx,
133 torch::Tensor &perm,
134 torch::Tensor &workspace,
135 + int64_t num_bits,
136 int64_t size_m,
137 int64_t size_n,
138 int64_t size_k,
@@ -141,7 +142,8 @@ torch::Tensor gptq_marlin_repack(
142 torch::Tensor &b_q_weight,
143 torch::Tensor &perm,
144 int64_t size_k,
144 - int64_t size_n);
145 + int64_t size_n,
146 + int64_t num_bits);
147 #endif
148
149 void squeezellm_gemm(
csrc/quantization/gptq_marlin/gptq_marlin.cu
+377 -175
@@ -32,7 +32,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
32 int4 *__restrict__ out_int4_ptr, int size_m,
33 int size_k, int block_rows) {}
34
35 -template <const int threads, // number of threads in a threadblock
35 +template <const int num_bits, // number of bits used for weights
36 + const int threads, // number of threads in a threadblock
37 const int thread_m_blocks, // number of 16x16 blocks in the m
38 // dimension (batchsize) of the threadblock
39 const int thread_n_blocks, // same for n dimension (output)
@@ -62,8 +63,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
63 torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
64 torch::Tensor &b_scales, torch::Tensor &g_idx,
65 torch::Tensor &perm, torch::Tensor &workspace,
65 - int64_t size_m, int64_t size_n, int64_t size_k,
66 - bool is_k_full) {
66 + int64_t num_bits, int64_t size_m, int64_t size_n,
67 + int64_t size_k, bool is_k_full) {
68 TORCH_CHECK_NOT_IMPLEMENTED(false,
69 "marlin_gemm(..) requires CUDA_ARCH >= 8.0");
70 return torch::empty({1, 1});
@@ -114,11 +115,21 @@ template <int lut> __device__ inline int lop3(int a, int b, int c) {
115 return res;
116 }
117
118 +// Constructs destination register by taking bytes from 2 sources (based on mask)
119 +template <int start_byte, int mask>
120 +__device__ inline uint32_t prmt(uint32_t a) {
121 + uint32_t res;
122 + asm volatile("prmt.b32 %0, %1, %2, %3;\n"
123 + : "=r"(res)
124 + : "r"(a), "n"(start_byte), "n"(mask));
125 + return res;
126 +}
127 +
128 // Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
129 // values. We mostly follow the strategy in the link below, with some small
130 // changes:
131 // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
121 -__device__ inline FragB dequant(int q) {
132 +__device__ inline FragB dequant_4bit(int q) {
133 const int LO = 0x000f000f;
134 const int HI = 0x00f000f0;
135 const int EX = 0x64006400;
@@ -139,6 +150,24 @@ __device__ inline FragB dequant(int q) {
150 return frag_b;
151 }
152
153 +__device__ inline FragB dequant_8bit(int q) {
154 + static constexpr uint32_t mask_for_elt_01 = 0x5250;
155 + static constexpr uint32_t mask_for_elt_23 = 0x5351;
156 + static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
157 +
158 + uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
159 + uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
160 +
161 + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
162 +
163 + FragB frag_b;
164 + frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
165 + *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
166 + frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
167 + *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
168 + return frag_b;
169 +}
170 +
171 // Multiply dequantized values by the corresponding quantization scale; used
172 // only for grouped quantization.
173 __device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
@@ -162,6 +191,13 @@ __device__ inline void scale4(FragB &frag_b, FragS &frag_s_1, FragS &frag_s_2,
191 frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
192 }
193
194 +// Given 2 floats multiply by 2 scales (halves)
195 +__device__ inline void scale_float(float *c, FragS &s) {
196 + __half *s_ptr = reinterpret_cast<__half *>(&s);
197 + c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
198 + c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
199 +}
200 +
201 // Wait until barrier reaches `count`, then lock for current threadblock.
202 __device__ inline void barrier_acquire(int *lock, int count) {
203 if (threadIdx.x == 0) {
@@ -250,7 +286,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
286 }
287 }
288
253 -template <const int threads, // number of threads in a threadblock
289 +template <const int num_bits, // number of bits used for weights
290 + const int threads, // number of threads in a threadblock
291 const int thread_m_blocks, // number of 16x16 blocks in the m
292 // dimension (batchsize) of the threadblock
293 const int thread_n_blocks, // same for n dimension (output)
@@ -286,6 +323,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
323 // configurations, while requiring as few slow global cross-threadblock
324 // reductions as possible.
325
326 + constexpr int pack_factor = 32 / num_bits;
327 +
328 // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
329 // better partitioning with less reductions
330 int parallel = 1;
@@ -385,21 +424,25 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
424 constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta);
425
426 // B sizes/strides
388 - int b_gl_stride = 16 * prob_n / 32;
389 - constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
427 + int b_gl_stride = 16 * prob_n / (pack_factor * 4);
428 + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
429 + constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
430 + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
431 +
432 int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
391 - int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
392 - constexpr int b_sh_wr_delta = threads;
393 - constexpr int b_sh_rd_delta = threads;
433 + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
434 + constexpr int b_sh_wr_delta = threads * b_thread_vecs;
435 + constexpr int b_sh_rd_delta = threads * b_thread_vecs;
436 constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
437 constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
438
439 // Scale sizes/strides without act_order
440 int s_gl_stride = prob_n / 8;
441 constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
400 - constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
401 - ? thread_k_blocks / group_blocks
402 - : 1;
442 + constexpr int s_tb_groups =
443 + !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
444 + ? thread_k_blocks / group_blocks
445 + : 1;
446 constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
447 int s_gl_rd_delta = s_gl_stride;
448
@@ -425,12 +468,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
468 a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
469 a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
470
428 - int b_gl_rd =
429 - b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
471 + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
472 + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
473 b_gl_rd += b_sh_stride * slice_col;
474 b_gl_rd += b_gl_rd_delta_o * slice_row;
432 - int b_sh_wr = threadIdx.x;
433 - int b_sh_rd = threadIdx.x;
475 + int b_sh_wr = threadIdx.x * b_thread_vecs;
476 + int b_sh_rd = threadIdx.x * b_thread_vecs;
477
478 // For act_order
479 constexpr int k_iter_size = tb_k / b_sh_wr_iters;
@@ -442,8 +485,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
485 // No act_order
486 int s_gl_rd;
487 if constexpr (!has_act_order) {
445 - s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
446 - s_sh_stride * slice_col + threadIdx.x;
488 + if constexpr (group_blocks == -1) {
489 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
490 + } else {
491 + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
492 + s_sh_stride * slice_col + threadIdx.x;
493 + }
494 }
495 int s_sh_wr = threadIdx.x;
496 bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
@@ -511,7 +558,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
558
559 // Register storage for double buffer of shared memory reads.
560 FragA frag_a[2][thread_m_blocks];
514 - I4 frag_b_quant[2];
561 + I4 frag_b_quant[2][b_thread_vecs];
562 FragC frag_c[thread_m_blocks][4][2];
563 FragS frag_s[2][4]; // No act-order
564 FragS act_frag_s[2][4][4]; // For act-order
@@ -575,7 +622,11 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
622 int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
623 #pragma unroll
624 for (int i = 0; i < b_sh_wr_iters; i++) {
578 - cp_async4_stream(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
625 +#pragma unroll
626 + for (int j = 0; j < b_thread_vecs; j++) {
627 + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
628 + }
629 +
630 B_ptr[i] += b_gl_rd_delta_o;
631 }
632
@@ -602,15 +653,15 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
653 // Only fetch scales if this tile starts a new group
654 if (pipe % (group_blocks / thread_k_blocks) == 0) {
655 if (s_sh_wr_pred) {
605 - cp_async4_stream(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
656 + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
657 }
658 s_gl_rd += s_gl_rd_delta;
659 }
660 } else {
661 for (int i = 0; i < s_tb_groups; i++) {
662 if (s_sh_wr_pred) {
612 - cp_async4_stream(&sh_s_stage[i * s_sh_stride + s_sh_wr],
613 - &scales_ptr[s_gl_rd]);
663 + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
664 + &scales_ptr[s_gl_rd]);
665 }
666 s_gl_rd += s_gl_rd_delta;
667 }
@@ -641,14 +692,24 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
692 for (int i = 0; i < thread_m_blocks; i++)
693 ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
694 int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
644 - frag_b_quant[k % 2] = *reinterpret_cast<I4 *>(
645 - &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
695 +
696 +#pragma unroll
697 + for (int i = 0; i < b_thread_vecs; i++) {
698 + frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
699 + &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
700 + }
701 };
702
703 bool is_same_group[stages];
704 int same_group_id[stages];
705
706 auto init_same_group = [&](int pipe) {
707 + if constexpr (!has_act_order) {
708 + is_same_group[pipe] = false;
709 + same_group_id[pipe] = 0;
710 + return;
711 + }
712 +
713 int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
714 int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
715
@@ -767,10 +828,23 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
828 // dequantization and matmul operations.
829 #pragma unroll
830 for (int j = 0; j < 4; j++) {
770 - int b_quant = frag_b_quant[k % 2][j];
771 - int b_quant_shift = b_quant >> 8;
831 + FragB frag_b0;
832 + FragB frag_b1;
833 + if constexpr (num_bits == 4) {
834 + int b_quant = frag_b_quant[k % 2][0][j];
835 + int b_quant_shift = b_quant >> 8;
836 +
837 + frag_b0 = dequant_4bit(b_quant);
838 + frag_b1 = dequant_4bit(b_quant_shift);
839
773 - FragB frag_b0 = dequant(b_quant);
840 + } else {
841 + int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
842 + int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
843 + int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
844 +
845 + frag_b0 = dequant_8bit(b_quant_0);
846 + frag_b1 = dequant_8bit(b_quant_1);
847 + }
848
849 // Apply scale to frag_b0
850 if constexpr (has_act_order) {
@@ -782,8 +856,6 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
856 }
857 }
858
785 - FragB frag_b1 = dequant(b_quant_shift);
786 -
859 // Apply scale to frag_b1
860 if constexpr (has_act_order) {
861 scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
@@ -808,13 +880,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
880 // multiple warps that accumulate their partial sums of the same output
881 // location; which we have to reduce over in the end. We do in shared memory.
882 auto thread_block_reduce = [&]() {
811 - constexpr int red_off = threads / b_sh_stride / 2;
883 + constexpr int red_off = threads / b_sh_stride_threads / 2;
884 if (red_off >= 1) {
813 - int red_idx = threadIdx.x / b_sh_stride;
814 - constexpr int red_sh_stride = b_sh_stride * 4 * 2;
815 - constexpr int red_sh_delta = b_sh_stride;
816 - int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
817 - (threadIdx.x % b_sh_stride);
885 + int red_idx = threadIdx.x / b_sh_stride_threads;
886 + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
887 + constexpr int red_sh_delta = b_sh_stride_threads;
888 + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
889 + (threadIdx.x % b_sh_stride_threads);
890
891 // Parallel logarithmic shared memory reduction. We make sure to avoid any
892 // unnecessary read or write iterations, e.g., for two warps we write only
@@ -861,7 +933,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
933 };
934
935 // Since multiple threadblocks may process parts of the same column slice, we
864 - // finally have to globally reduce over the results. As the striped portioning
936 + // finally have to globally reduce over the results. As the striped partitioning
937 // minimizes the number of such reductions and our outputs are usually rather
938 // small, we perform this reduction serially in L2 cache.
939 auto global_reduce = [&](bool first = false, bool last = false) {
@@ -951,13 +1023,15 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1023 auto write = [&](int idx, float c0, float c1, FragS &s) {
1024 half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1025
954 - // For per-column quantization we finally apply the scale here
955 - if constexpr (!has_act_order && group_blocks == -1) {
1026 + // For per-column quantization we finally apply the scale here (only for
1027 + // 4-bit)
1028 + if constexpr (!has_act_order && group_blocks == -1 && num_bits == 4) {
1029 res = __hmul2(res, s[0]);
1030 }
1031
1032 ((half2 *)sh)[idx] = res;
1033 };
1034 +
1035 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1036 #pragma unroll
1037 for (int i = 0; i < thread_m_blocks; i++) {
@@ -1023,6 +1097,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1097 // ensure all shared memory accesses are static. Note that both pipelines
1098 // have even length meaning that the next iteration will always start at
1099 // index 0.
1100 +
1101 #pragma unroll
1102 for (int pipe = 0; pipe < stages;) {
1103 #pragma unroll
@@ -1070,23 +1145,63 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1145 // For per-column scales, we only fetch them here in the final step before
1146 // write-out
1147 if constexpr (!has_act_order && group_blocks == -1) {
1073 - if (last) {
1148 + if constexpr (num_bits == 8) {
1149 if (s_sh_wr_pred) {
1075 - cp_async4_stream(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1150 + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1151 }
1152 cp_async_fence();
1153 + } else {
1154 + if (last) {
1155 + if (s_sh_wr_pred) {
1156 + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1157 + }
1158 + cp_async_fence();
1159 + }
1160 }
1161 }
1162
1163 thread_block_reduce();
1164 if constexpr (!has_act_order && group_blocks == -1) {
1083 - if (last) {
1165 + if constexpr (num_bits == 8) {
1166 cp_async_wait<0>();
1167 __syncthreads();
1168 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1169 reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1170 reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1171 }
1172 +
1173 + } else {
1174 + if (last) {
1175 + cp_async_wait<0>();
1176 + __syncthreads();
1177 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1178 + reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1179 + reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1180 + }
1181 + }
1182 + }
1183 + }
1184 +
1185 + // For 8-bit channelwise, we apply the scale before the global reduction
1186 + // that converts the fp32 results to fp16 (so that we avoid possible
1187 + // overflow in fp16)
1188 + if constexpr (!has_act_order && group_blocks == -1 && num_bits == 8) {
1189 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1190 +#pragma unroll
1191 + for (int i = 0; i < thread_m_blocks; i++) {
1192 +#pragma unroll
1193 + for (int j = 0; j < 4; j++) {
1194 + scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1195 + frag_s[j / 2][2 * (j % 2) + 0]);
1196 + scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1197 + frag_s[j / 2][2 * (j % 2) + 0]);
1198 +
1199 + scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1200 + frag_s[j / 2][2 * (j % 2) + 1]);
1201 + scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1202 + frag_s[j / 2][2 * (j % 2) + 1]);
1203 + }
1204 + }
1205 }
1206 }
1207
@@ -1125,28 +1240,25 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1240 s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1241 }
1242
1128 - // if (blockIdx.x == 0 && threadIdx.x == 0) {
1129 - // printf("Move\n");
1130 - // }
1243 start_pipes();
1244 }
1245 }
1246 }
1247 }
1248
1137 -#define __CALL_IF(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1249 +#define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1250 HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1139 - else if (thread_m_blocks == THREAD_M_BLOCKS && \
1251 + else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1252 thread_n_blocks == THREAD_N_BLOCKS && \
1253 thread_k_blocks == THREAD_K_BLOCKS && \
1254 has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1255 num_threads == NUM_THREADS) { \
1256 cudaFuncSetAttribute( \
1145 - Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1146 - pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1257 + Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1258 + THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1259 cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1148 - Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1149 - pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1260 + Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1261 + THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1262 <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1263 A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1264 prob_k, locks); \
@@ -1158,28 +1270,92 @@ typedef struct {
1270 int num_threads;
1271 } thread_config_t;
1272
1161 -thread_config_t small_batch_thread_configs[] = {
1273 +typedef struct {
1274 + int max_m_blocks;
1275 + thread_config_t tb_cfg;
1276 +} exec_config_t;
1277 +
1278 +thread_config_t thread_configs[] = {
1279 // Ordered by priority
1280
1281 // thread_k, thread_n, num_threads
1165 - {128, 128, 256}, // Default
1166 - {128, 64, 128}, // Reduce N 2X, same K
1167 - {64, 256, 256}, // Reduce K 2X, increase N 2X
1168 - {64, 128, 128}, // Reduce K 2X, same N
1282 + {64, 256, 256}, // Default (max cache usage)
1283 + {64, 128, 128}, // Reduce N, reduce warps
1284 + {128, 64, 128}, // Reduce N more, but increase K
1285 +
1286 };
1287
1171 -thread_config_t large_batch_thread_configs[] = {
1172 - // Ordered by priority
1288 +int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1289 + int prob_n, int prob_k, int num_bits, int group_size,
1290 + bool has_act_order, bool is_k_full) {
1291 + bool cache_scales_chunk = has_act_order && !is_k_full;
1292
1174 - // thread_k, thread_n, num_threads
1175 - {64, 256, 256}, // Default
1176 - {128, 64, 128}, // Reduce N 2X, same K
1177 - {64, 128, 128}, // Reduce N 2X, same K
1178 - // {128, 64, 128}, // Reduce N 4X, increase K 2X
1179 -};
1293 + int tb_n = th_config.thread_n;
1294 + int tb_k = th_config.thread_k;
1295 +
1296 + // Get max scale groups per thread-block
1297 + int tb_groups;
1298 + if (group_size == -1) {
1299 + tb_groups = 1;
1300 + } else if (group_size == 0) {
1301 + tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
1302 + } else {
1303 + tb_groups = div_ceil(tb_k, group_size);
1304 + }
1305 +
1306 + if (cache_scales_chunk) {
1307 + int load_groups =
1308 + tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K
1309 + load_groups = max(load_groups, 32); // We load at least 32 scale groups
1310 + return load_groups * tb_n * 2;
1311 +
1312 + } else {
1313 + int tb_scales = tb_groups * tb_n * 2;
1314 +
1315 + return tb_scales * pipe_stages;
1316 + }
1317 +}
1318 +
1319 +bool is_valid_cache_size(thread_config_t const &th_config, int max_m_blocks,
1320 + int prob_m, int prob_n, int prob_k, int num_bits,
1321 + int scales_cache_size, int max_shared_mem) {
1322 + int pack_factor = 32 / num_bits;
1323 +
1324 + // Get B size
1325 + int tb_k = th_config.thread_k;
1326 + int tb_n = th_config.thread_n;
1327 +
1328 + int b_size = (tb_k * tb_n / pack_factor) * 4;
1329 +
1330 + // Get A size
1331 + int m_blocks = div_ceil(prob_m, 16);
1332 + int tb_max_m = 16;
1333
1181 -bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
1182 - int prob_k) {
1334 + while (true) {
1335 + if (m_blocks >= max_m_blocks) {
1336 + tb_max_m *= max_m_blocks;
1337 + break;
1338 + }
1339 +
1340 + max_m_blocks--;
1341 + if (max_m_blocks == 0) {
1342 + TORCH_CHECK(false, "Unexpected m_blocks = ", m_blocks);
1343 + }
1344 + }
1345 +
1346 + int a_size = (tb_max_m * tb_k) * 2;
1347 +
1348 + float pipe_size = (a_size + b_size) * pipe_stages;
1349 +
1350 + TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1351 +
1352 + return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1353 +}
1354 +
1355 +bool is_valid_config(thread_config_t const &th_config, int max_m_blocks,
1356 + int prob_m, int prob_n, int prob_k, int num_bits,
1357 + int group_size, bool has_act_order, bool is_k_full,
1358 + int max_shared_mem) {
1359 // Sanity
1360 if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1361 th_config.num_threads == -1) {
@@ -1201,62 +1377,79 @@ bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
1377 return false;
1378 }
1379
1380 + // Determine cache for scales
1381 + int scales_cache_size =
1382 + get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits,
1383 + group_size, has_act_order, is_k_full);
1384 +
1385 + // Check that pipeline fits into cache
1386 + if (!is_valid_cache_size(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1387 + num_bits, scales_cache_size, max_shared_mem)) {
1388 + return false;
1389 + }
1390 +
1391 return true;
1392 }
1393
1207 -thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1208 -
1209 - // TODO: Enable if needed after some more testing
1210 - if (prob_m <= 0) {
1211 - for (auto th_config : small_batch_thread_configs) {
1212 - if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1213 - return th_config;
1394 +exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1395 + int num_bits, int group_size,
1396 + bool has_act_order, bool is_k_full,
1397 + int max_shared_mem) {
1398 + int max_m_blocks = 4;
1399 + while (max_m_blocks > 0) {
1400 + for (auto th_config : thread_configs) {
1401 + if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1402 + num_bits, group_size, has_act_order, is_k_full,
1403 + max_shared_mem)) {
1404 + return exec_config_t{max_m_blocks, th_config};
1405 }
1406 }
1407
1217 - } else {
1218 - for (auto th_config : large_batch_thread_configs) {
1219 - if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1220 - return th_config;
1221 - }
1222 - }
1408 + printf("WARNING: Marlin kernel is reducing max_m_blocks due to small SM "
1409 + "GPU cache. This may "
1410 + "hurt performance. Consider upgrading your GPU.\n");
1411 +
1412 + max_m_blocks--; // Process less M blocks per invocation to reduce cache
1413 + // usage
1414 }
1415
1225 - return thread_config_t{-1, -1, -1};
1416 + return exec_config_t{0, {-1, -1, -1}};
1417 }
1418
1228 -#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1229 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1230 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1231 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1232 - __CALL_IF(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1419 +#define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1420 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1421 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1422 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1423 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1424 \
1234 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1235 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1236 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1237 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1425 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1426 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1427 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1428 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1429 \
1239 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1240 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1241 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1242 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1430 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1431 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1432 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1433 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1434 \
1244 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1245 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1246 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1247 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1435 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1436 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1437 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1438 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1439 \
1249 - __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1250 - __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1251 - __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1252 - __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1253 -
1254 -void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1255 - void *perm, void *a_tmp, int prob_m, int prob_n, int prob_k,
1256 - void *workspace, bool has_act_order, bool is_k_full,
1257 - int num_groups, int group_size, int dev = 0,
1258 - cudaStream_t stream = 0, int thread_k = -1, int thread_n = -1,
1259 - int sms = -1, int max_par = 16) {
1440 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1441 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1442 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1443 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1444 +
1445 +void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1446 + void *g_idx, void *perm, void *a_tmp, int prob_m,
1447 + int prob_n, int prob_k, void *workspace, int num_bits,
1448 + bool has_act_order, bool is_k_full, int num_groups,
1449 + int group_size, int dev, cudaStream_t stream, int thread_k,
1450 + int thread_n, int sms, int max_par) {
1451 + TORCH_CHECK(num_bits == 4 || num_bits == 8,
1452 + "num_bits must be 4 or 8. Got = ", num_bits);
1453 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1454 ", ", prob_n, ", ", prob_k, "]");
1455
@@ -1274,25 +1467,34 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1467 TORCH_CHECK(max_shared_mem > 0);
1468
1469 // Set thread config
1277 - thread_config_t th_config;
1470 + exec_config_t exec_cfg;
1471 if (thread_k != -1 && thread_n != -1) {
1472 // User-defined config
1280 - th_config = thread_config_t{thread_k, thread_n, default_threads};
1473 + exec_cfg =
1474 + exec_config_t{4, thread_config_t{thread_k, thread_n, default_threads}};
1475 } else {
1476 // Auto config
1283 - th_config = determine_thread_config(prob_m, prob_n, prob_k);
1477 + exec_cfg =
1478 + determine_thread_config(prob_m, prob_n, prob_k, num_bits, group_size,
1479 + has_act_order, is_k_full, max_shared_mem);
1480 }
1481
1286 - TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1287 - "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1288 - ", thread_n = " + str(th_config.thread_n) +
1289 - ", num_threads = " + str(th_config.num_threads) +
1290 - " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1291 - str(prob_n) + "]");
1292 -
1293 - int num_threads = th_config.num_threads;
1294 - thread_k = th_config.thread_k;
1295 - thread_n = th_config.thread_n;
1482 + TORCH_CHECK(exec_cfg.max_m_blocks > 0 &&
1483 + is_valid_config(exec_cfg.tb_cfg, exec_cfg.max_m_blocks,
1484 + prob_m, prob_n, prob_k, num_bits, group_size,
1485 + has_act_order, is_k_full, max_shared_mem),
1486 + "Invalid thread config: max_m_blocks = ", exec_cfg.max_m_blocks,
1487 + ", thread_k = ", exec_cfg.tb_cfg.thread_k,
1488 + ", thread_n = ", exec_cfg.tb_cfg.thread_n,
1489 + ", num_threads = ", exec_cfg.tb_cfg.num_threads, " for MKN = [",
1490 + prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
1491 + ", group_size = ", group_size,
1492 + ", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
1493 + ", max_shared_mem = ", max_shared_mem);
1494 +
1495 + int num_threads = exec_cfg.tb_cfg.num_threads;
1496 + thread_k = exec_cfg.tb_cfg.thread_k;
1497 + thread_n = exec_cfg.tb_cfg.thread_n;
1498
1499 int thread_k_blocks = thread_k / 16;
1500 int thread_n_blocks = thread_n / 16;
@@ -1352,28 +1554,32 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1554 }
1555
1556 // Main loop
1355 - for (int i = 0; i < tot_m_blocks; i += 4) {
1557 + for (int i = 0; i < tot_m_blocks; i += exec_cfg.max_m_blocks) {
1558 int thread_m_blocks = tot_m_blocks - i;
1559 prob_m = tot_m - 16 * i;
1560 int par = 1;
1359 - if (thread_m_blocks > 4) {
1561 + if (thread_m_blocks > exec_cfg.max_m_blocks) {
1562 // Note that parallel > 1 currently only works for inputs without any
1563 // padding
1362 - par = (16 * thread_m_blocks - pad) / 64;
1564 + par = (16 * thread_m_blocks - pad) / (16 * exec_cfg.max_m_blocks);
1565 if (par > max_par)
1566 par = max_par;
1365 - prob_m = 64 * par;
1366 - i += 4 * (par - 1);
1367 - thread_m_blocks = 4;
1567 + prob_m = (16 * exec_cfg.max_m_blocks) * par;
1568 + i += exec_cfg.max_m_blocks * (par - 1);
1569 + thread_m_blocks = exec_cfg.max_m_blocks;
1570 }
1571
1572 // Define kernel configurations
1573 if (false) {
1574 }
1373 - CALL_IF(16, 4, 256)
1374 - CALL_IF(8, 8, 256)
1375 - CALL_IF(8, 4, 128)
1376 - CALL_IF(4, 8, 128)
1575 + CALL_IF(4, 32, 2, 256)
1576 + CALL_IF(4, 16, 4, 256)
1577 + CALL_IF(4, 8, 4, 128)
1578 + CALL_IF(4, 4, 8, 128)
1579 + CALL_IF(8, 32, 2, 256)
1580 + CALL_IF(8, 16, 4, 256)
1581 + CALL_IF(8, 8, 4, 128)
1582 + CALL_IF(8, 4, 8, 128)
1583 else {
1584 TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1585 str(prob_n) + ", " + str(prob_k) + "]" +
@@ -1395,33 +1601,32 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1601 torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1602 torch::Tensor &b_scales, torch::Tensor &g_idx,
1603 torch::Tensor &perm, torch::Tensor &workspace,
1398 - int64_t size_m, int64_t size_n, int64_t size_k,
1399 - bool is_k_full) {
1604 + int64_t num_bits, int64_t size_m, int64_t size_n,
1605 + int64_t size_k, bool is_k_full) {
1606 + // Verify num_bits
1607 + TORCH_CHECK(num_bits == 4 || num_bits == 8,
1608 + "num_bits must be 4 or 8. Got = ", num_bits);
1609 + int pack_factor = 32 / num_bits;
1610 +
1611 // Verify A
1401 - TORCH_CHECK(a.size(0) == size_m,
1402 - "Shape mismatch: a.size(0) = " + str(a.size(0)) +
1403 - ", size_m = " + str(size_m));
1404 - TORCH_CHECK(a.size(1) == size_k,
1405 - "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1406 - ", size_k = " + str(size_k));
1612 + TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
1613 + ", size_m = ", size_m);
1614 + TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1),
1615 + ", size_k = ", size_k);
1616
1617 // Verify B
1409 - TORCH_CHECK(size_k % gptq_marlin::tile_size == 0,
1410 - "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1411 - str(gptq_marlin::tile_size));
1618 + TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1619 + " is not divisible by tile_size = ", gptq_marlin::tile_size);
1620 TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
1413 - "Shape mismatch: b_q_weight.size(0) = " +
1414 - str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1415 - ", tile_size = " + str(gptq_marlin::tile_size));
1416 - TORCH_CHECK(
1417 - b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1418 - "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1419 - " is not divisible by tile_size = " + str(gptq_marlin::tile_size));
1420 - int actual_size_n = (b_q_weight.size(1) / gptq_marlin::tile_size) *
1421 - gptq_marlin::pack_factor_4bit;
1422 - TORCH_CHECK(size_n == actual_size_n,
1423 - "size_n = " + str(size_n) +
1424 - ", actual_size_n = " + str(actual_size_n));
1621 + "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1622 + ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1623 + TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1624 + "b_q_weight.size(1) = ", b_q_weight.size(1),
1625 + " is not divisible by tile_size = ", gptq_marlin::tile_size);
1626 + int actual_size_n =
1627 + (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
1628 + TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
1629 + ", actual_size_n = ", actual_size_n);
1630
1631 // Verify device and strides
1632 TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
@@ -1457,9 +1662,9 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1662 // Verify g_idx and perm
1663 TORCH_CHECK((g_idx.size(0) == 0 && perm.size(0) == 0) ||
1664 (g_idx.size(0) == size_k && perm.size(0) == size_k),
1460 - "Unexpected g_idx.size(0) = " + str(g_idx.size(0)) +
1461 - " and perm.size(0) = " + str(perm.size(0)) +
1462 - ", where size_k = " + str(size_k));
1665 + "Unexpected g_idx.size(0) = ", g_idx.size(0),
1666 + " and perm.size(0) = ", perm.size(0),
1667 + ", where size_k = ", size_k);
1668
1669 // Detect groupsize and act_order
1670 int num_groups = -1;
@@ -1475,9 +1680,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1680 if (has_act_order) {
1681 if (is_k_full) {
1682 TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
1478 - TORCH_CHECK(size_k % num_groups == 0,
1479 - "size_k = " + str(size_k) +
1480 - ", is not divisible by num_groups = " + str(num_groups));
1683 + TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
1684 + ", is not divisible by num_groups = ", num_groups);
1685 group_size = size_k / num_groups;
1686 } else {
1687 group_size = 0;
@@ -1485,10 +1689,9 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1689
1690 } else {
1691 if (num_groups > 1) {
1488 - TORCH_CHECK(size_k % num_groups == 0,
1489 - "size_k = " + str(size_k) +
1490 - ", is not divisible by b_scales.size(0) = " +
1491 - str(b_scales.size(0)));
1692 + TORCH_CHECK(
1693 + size_k % num_groups == 0, "size_k = ", size_k,
1694 + ", is not divisible by b_scales.size(0) = ", b_scales.size(0));
1695 group_size = size_k / num_groups;
1696 } else {
1697 group_size = -1;
@@ -1496,23 +1699,22 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1699 }
1700
1701 // Verify workspace size
1499 - TORCH_CHECK(size_n % gptq_marlin::min_thread_n == 0,
1500 - "size_n = " + str(size_n) +
1501 - ", is not divisible by min_thread_n = " +
1502 - str(gptq_marlin::min_thread_n));
1702 + TORCH_CHECK(
1703 + size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1704 + ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1705 int min_workspace_size =
1706 (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
1707 TORCH_CHECK(workspace.numel() >= min_workspace_size,
1506 - "workspace.numel = " + str(workspace.numel()) +
1507 - " is below min_workspace_size = " + str(min_workspace_size));
1708 + "workspace.numel = ", workspace.numel(),
1709 + " is below min_workspace_size = ", min_workspace_size);
1710
1711 int dev = a.get_device();
1510 - gptq_marlin::marlin_cuda(
1712 + gptq_marlin::marlin_mm_f16i4(
1713 a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), b_scales.data_ptr(),
1714 g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(), size_m, size_n,
1513 - size_k, workspace.data_ptr(), has_act_order, is_k_full, num_groups,
1514 - group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n,
1515 - sms, gptq_marlin::max_par);
1715 + size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1716 + num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1717 + thread_k, thread_n, sms, gptq_marlin::max_par);
1718
1719 return c;
1720 }
csrc/quantization/gptq_marlin/gptq_marlin.cuh
+2 -6
@@ -24,8 +24,6 @@ static constexpr int min_thread_k = 64;
24 static constexpr int tile_size = 16;
25 static constexpr int max_par = 16;
26
27 -static constexpr int pack_factor_4bit = 8; // We have 8 4-bit vals inside a 32 bit
28 -
27 template <typename T, int n>
28 struct Vec {
29 T elems[n];
@@ -51,13 +49,11 @@ __device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool
49 "r"(smem), "l"(glob_ptr), "n"(BYTES));
50 }
51
54 -__device__ inline void cp_async4_stream(void* smem_ptr, const void* glob_ptr) {
52 +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
53 const int BYTES = 16;
54 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55 asm volatile("{\n"
58 - " .reg .b64 p;\n"
59 - " createpolicy.fractional.L2::evict_first.b64 p, 1.0;"
60 - " cp.async.cg.shared.global.L2::cache_hint [%0], [%1], %2, p;\n"
56 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
57 "}\n" ::"r"(smem),
58 "l"(glob_ptr), "n"(BYTES));
59 }
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+90 -62
@@ -11,7 +11,7 @@ static constexpr int tile_n_size = tile_k_size * 4;
11
12 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
13
14 -template <int const num_threads, bool const has_perm>
14 +template <int const num_threads, int const num_bits, bool const has_perm>
15 __global__ void
16 marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
17 uint32_t const *__restrict__ perm_ptr,
@@ -20,7 +20,8 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
20 } // namespace gptq_marlin
21
22 torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
23 - int64_t size_k, int64_t size_n) {
23 + int64_t size_k, int64_t size_n,
24 + int64_t num_bits) {
25 TORCH_CHECK_NOT_IMPLEMENTED(
26 false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
27 return torch::empty({1, 1});
@@ -28,11 +29,13 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
29
30 #else
31
31 -template <int const num_threads, bool const has_perm>
32 +template <int const num_threads, int const num_bits, bool const has_perm>
33 __global__ void
34 marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
35 uint32_t const *__restrict__ perm_ptr,
36 uint32_t *__restrict__ out_ptr, int size_k, int size_n) {
37 + constexpr int pack_factor = 32 / num_bits;
38 +
39 int k_tiles = size_k / tile_k_size;
40 int n_tiles = size_n / tile_n_size;
41 int block_k_tiles = div_ceil(k_tiles, gridDim.x);
@@ -64,9 +67,10 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
67 sh_pipe_ptr += perm_size;
68 }
69
70 + constexpr int tile_ints = tile_k_size / pack_factor;
71 +
72 constexpr int stage_n_threads = tile_n_size / 4;
68 - constexpr int stage_k_threads =
69 - has_perm ? tile_k_size : tile_k_size / pack_factor_4bit;
73 + constexpr int stage_k_threads = has_perm ? tile_k_size : tile_ints;
74 constexpr int stage_size = stage_k_threads * stage_n_threads;
75
76 auto load_perm_to_shared = [&](int k_tile_id) {
@@ -99,9 +103,9 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
103 reinterpret_cast<uint32_t const *>(sh_perm_ptr);
104
105 int src_k = sh_perm_int_ptr[k_id];
102 - int src_k_packed = src_k / pack_factor_4bit;
106 + int src_k_packed = src_k / pack_factor;
107
104 - cp_async4_stream(
108 + cp_async4(
109 &sh_ptr[k_id * stage_n_threads + n_id],
110 reinterpret_cast<int4 const *>(&(
111 b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)])));
@@ -113,12 +117,12 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
117 int n_id = threadIdx.x % stage_n_threads;
118
119 int first_k = k_tile_id * tile_k_size;
116 - int first_k_packed = first_k / pack_factor_4bit;
120 + int first_k_packed = first_k / pack_factor;
121
118 - cp_async4_stream(&sh_ptr[k_id * stage_n_threads + n_id],
119 - reinterpret_cast<int4 const *>(
120 - &(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
121 - first_n + (n_id * 4)])));
122 + cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
123 + reinterpret_cast<int4 const *>(
124 + &(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
125 + first_n + (n_id * 4)])));
126 }
127 }
128
@@ -145,26 +149,27 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
149 int cur_n = warp_id * 16 + tc_col;
150
151 constexpr int sh_stride = 64;
152 + constexpr uint32_t mask = (1 << num_bits) - 1;
153
154 int4 *sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
155 uint32_t *sh_stage_int_ptr = reinterpret_cast<uint32_t *>(sh_stage_ptr);
156
157 uint32_t *sh_perm_int_ptr = reinterpret_cast<uint32_t *>(sh_perm_ptr);
158
154 - uint32_t vals[pack_factor_4bit];
159 + uint32_t vals[8];
160
161 if constexpr (has_perm) {
162 for (int i = 0; i < 4; i++) {
163 int k_idx = tc_row + tc_offsets[i];
164
165 uint32_t src_k = sh_perm_int_ptr[k_idx];
161 - uint32_t src_k_pos = src_k % pack_factor_4bit;
166 + uint32_t src_k_pos = src_k % pack_factor;
167
168 uint32_t b1_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n];
164 - uint32_t b1_cur_val = (b1_val >> (src_k_pos * 4)) & 0xf;
169 + uint32_t b1_cur_val = (b1_val >> (src_k_pos * num_bits)) & mask;
170
171 uint32_t b2_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n + 8];
167 - uint32_t b2_cur_val = (b2_val >> (src_k_pos * 4)) & 0xf;
172 + uint32_t b2_cur_val = (b2_val >> (src_k_pos * num_bits)) & mask;
173
174 vals[i] = b1_cur_val;
175 vals[4 + i] = b2_cur_val;
@@ -172,41 +177,56 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
177
178 } else {
179
175 - uint32_t b1_val_1 = sh_stage_int_ptr[cur_n];
176 - uint32_t b1_val_2 = sh_stage_int_ptr[sh_stride + cur_n];
177 -
178 - uint32_t b2_val_1 = sh_stage_int_ptr[cur_n + 8];
179 - uint32_t b2_val_2 = sh_stage_int_ptr[sh_stride + cur_n + 8];
180 + uint32_t b1_vals[tile_ints];
181 + uint32_t b2_vals[tile_ints];
182
183 #pragma unroll
182 - for (int i = 0; i < 2; i++) {
183 - int cur_elem = tc_row + tc_offsets[i];
184 - vals[i] = (b1_val_1 >> (cur_elem * 4)) & 0xf;
185 - vals[4 + i] = (b2_val_1 >> (cur_elem * 4)) & 0xf;
184 + for (int i = 0; i < tile_ints; i++) {
185 + b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
186 + b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
187 }
188
189 #pragma unroll
189 - for (int i = 2; i < 4; i++) {
190 - int cur_elem = tc_row + tc_offsets[i] - 8;
191 - vals[i] = (b1_val_2 >> (cur_elem * 4)) & 0xf;
192 - vals[4 + i] = (b2_val_2 >> (cur_elem * 4)) & 0xf;
190 + for (int i = 0; i < 4; i++) {
191 + int cur_elem = tc_row + tc_offsets[i];
192 + int cur_int = cur_elem / pack_factor;
193 + int cur_pos = cur_elem % pack_factor;
194 +
195 + vals[i] = (b1_vals[cur_int] >> (cur_pos * num_bits)) & mask;
196 + vals[4 + i] = (b2_vals[cur_int] >> (cur_pos * num_bits)) & mask;
197 }
198 }
199
200 + constexpr int tile_size = tile_k_size * tile_n_size / pack_factor;
201 + int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
202 +
203 // Result of:
204 // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
198 - constexpr int pack_idx[pack_factor_4bit] = {0, 2, 4, 6, 1, 3, 5, 7};
205 + if constexpr (num_bits == 4) {
206 + constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
207
200 - uint32_t res = 0;
208 + uint32_t res = 0;
209 #pragma unroll
202 - for (int i = 0; i < pack_factor_4bit; i++) {
203 - res |= vals[pack_idx[i]] << (i * 4);
204 - }
210 + for (int i = 0; i < 8; i++) {
211 + res |= vals[pack_idx[i]] << (i * 4);
212 + }
213
206 - constexpr int tile_size = tile_k_size * tile_n_size / pack_factor_4bit;
207 - int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
214 + out_ptr[out_offset + th_id * 4 + warp_id] = res;
215
209 - out_ptr[out_offset + th_id * 4 + warp_id] = res;
216 + } else {
217 + constexpr int pack_idx[4] = {0, 2, 1, 3};
218 +
219 + uint32_t res1 = 0;
220 + uint32_t res2 = 0;
221 +#pragma unroll
222 + for (int i = 0; i < 4; i++) {
223 + res1 |= vals[pack_idx[i]] << (i * 8);
224 + res2 |= vals[4 + pack_idx[i]] << (i * 8);
225 + }
226 +
227 + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1;
228 + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2;
229 + }
230 };
231
232 auto start_pipes = [&](int k_tile_id, int n_tile_id) {
@@ -242,19 +262,35 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
262
263 } // namespace gptq_marlin
264
265 +#define CALL_IF(NUM_BITS, HAS_PERM) \
266 + else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
267 + cudaFuncSetAttribute( \
268 + gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
269 + NUM_BITS, HAS_PERM>, \
270 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
271 + gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
272 + HAS_PERM> \
273 + <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
274 + b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
275 + }
276 +
277 torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
246 - int64_t size_k, int64_t size_n) {
278 + int64_t size_k, int64_t size_n,
279 + int64_t num_bits) {
280 // Verify compatibility with marlin tile of 16x64
281 TORCH_CHECK(size_k % gptq_marlin::tile_k_size == 0, "size_k = ", size_k,
282 " is not divisible by tile_k_size = ", gptq_marlin::tile_k_size);
283 TORCH_CHECK(size_n % gptq_marlin::tile_n_size == 0, "size_n = ", size_n,
284 " is not divisible by tile_n_size = ", gptq_marlin::tile_n_size);
285
286 + TORCH_CHECK(num_bits == 4 || num_bits == 8,
287 + "num_bits must be 4 or 8. Got = ", num_bits);
288 + int const pack_factor = 32 / num_bits;
289 +
290 // Verify B
254 - TORCH_CHECK((size_k / gptq_marlin::pack_factor_4bit) == b_q_weight.size(0),
291 + TORCH_CHECK((size_k / pack_factor) == b_q_weight.size(0),
292 "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
256 - ", size_k = ", size_k,
257 - ", pack_factor_4bit = ", gptq_marlin::pack_factor_4bit);
293 + ", size_k = ", size_k, ", pack_factor = ", pack_factor);
294 TORCH_CHECK(b_q_weight.size(1) == size_n,
295 "b_q_weight.size(1) = ", b_q_weight.size(1),
296 " is not size_n = ", size_n);
@@ -273,10 +309,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
309 auto options = torch::TensorOptions()
310 .dtype(b_q_weight.dtype())
311 .device(b_q_weight.device());
276 - torch::Tensor out = torch::empty(
277 - {size_k / gptq_marlin::tile_size,
278 - size_n * gptq_marlin::tile_size / gptq_marlin::pack_factor_4bit},
279 - options);
312 + torch::Tensor out =
313 + torch::empty({size_k / gptq_marlin::tile_size,
314 + size_n * gptq_marlin::tile_size / pack_factor},
315 + options);
316
317 // Detect if there is act_order
318 bool has_perm = perm.size(0) != 0;
@@ -299,23 +335,15 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
335 cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
336 TORCH_CHECK(max_shared_mem > 0);
337
302 - if (has_perm) {
303 - cudaFuncSetAttribute(
304 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, true>,
305 - cudaFuncAttributeMaxDynamicSharedMemorySize,
306 - max_shared_mem);
307 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, true>
308 - <<<blocks, gptq_marlin::repack_threads, max_shared_mem,
309 - stream>>>(b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n);
310 -
311 - } else {
312 - cudaFuncSetAttribute(
313 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, false>,
314 - cudaFuncAttributeMaxDynamicSharedMemorySize,
315 - max_shared_mem);
316 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, false>
317 - <<<blocks, gptq_marlin::repack_threads, max_shared_mem,
318 - stream>>>(b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n);
338 + if (false) {
339 + }
340 + CALL_IF(4, false)
341 + CALL_IF(4, true)
342 + CALL_IF(8, false)
343 + CALL_IF(8, true)
344 + else {
345 + TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits,
346 + ", has_perm = ", has_perm);
347 }
348
349 return out;
tests/models/test_gptq_marlin.py
+9 -4
@@ -39,6 +39,13 @@ MODELS = [
39 ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-64g-actorder_True"),
40 # act_order==True, group_size=32
41 ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-32g-actorder_True"),
42 +
43 + # 8-bit, act_order==True, group_size=channelwise
44 + ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
45 + # 8-bit, act_order==True, group_size=128
46 + ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-128g-actorder_True"),
47 + # 8-bit, act_order==True, group_size=32
48 + ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-32g-actorder_True"),
49 ]
50
51
@@ -65,8 +72,7 @@ def test_models(
72 dtype=dtype,
73 quantization="marlin",
74 max_model_len=MAX_MODEL_LEN,
68 - tensor_parallel_size=1,
69 - disable_custom_all_reduce=True)
75 + tensor_parallel_size=1)
76
77 gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
78 example_prompts, max_tokens, num_logprobs)
@@ -78,8 +84,7 @@ def test_models(
84 dtype=dtype,
85 quantization="gptq",
86 max_model_len=MAX_MODEL_LEN,
81 - tensor_parallel_size=1,
82 - disable_custom_all_reduce=True)
87 + tensor_parallel_size=1)
88 gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
89 max_tokens,
90 num_logprobs)
vllm/_custom_ops.py
+8 -6
@@ -169,18 +169,20 @@ def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
169
170 # gptq_marlin
171 def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
172 - size_k: int, size_n: int) -> torch.Tensor:
173 - return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n)
172 + size_k: int, size_n: int,
173 + num_bits: int) -> torch.Tensor:
174 + return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n,
175 + num_bits)
176
177
178 def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
179 b_scales: torch.Tensor, g_idx: torch.Tensor,
178 - perm: torch.Tensor, workspace: torch.Tensor, size_m: int,
179 - size_n: int, size_k: int,
180 + perm: torch.Tensor, workspace: torch.Tensor,
181 + num_bits: int, size_m: int, size_n: int, size_k: int,
182 is_k_full: bool) -> torch.Tensor:
183 return vllm_ops.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
182 - workspace, size_m, size_n, size_k,
183 - is_k_full)
184 + workspace, num_bits, size_m, size_n,
185 + size_k, is_k_full)
186
187
188 # fp8
vllm/model_executor/layers/quantization/gptq_marlin.py
+64 -70
@@ -2,7 +2,6 @@ import enum
2 from enum import Enum
3 from typing import Any, Dict, List, Optional
4
5 -import numpy
5 import torch
6 from torch.nn.parameter import Parameter
7
@@ -17,41 +16,13 @@ GPTQ_MARLIN_MIN_THREAD_N = 64
16 GPTQ_MARLIN_MIN_THREAD_K = 128
17 GPTQ_MARLIN_MAX_PARALLEL = 16
18
20 -GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4]
19 +GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
20 GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
21 GPTQ_MARLIN_SUPPORTED_SYM = [True]
22
23
25 -# Precompute permutations for Marlin weight and scale shuffling
26 -#
27 -# Marlin works on [16,64] tiles. The goal of the permutations
28 -# is to reorder the weight data so that it is compatible
29 -# with the tensor-core format that is described here:
30 -# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
31 -#
32 -# As a result of this reordering, the vector loads inside the
33 -# kernel will get the data as it is needed for tensor-core
34 -# (without the need to use ldmatrix instructions)
35 -def _get_perms():
36 - perm = []
37 - for i in range(32):
38 - perm1 = []
39 - col = i // 4
40 - for block in [0, 1]:
41 - for row in [
42 - 2 * (i % 4),
43 - 2 * (i % 4) + 1,
44 - 2 * (i % 4 + 4),
45 - 2 * (i % 4 + 4) + 1,
46 - ]:
47 - perm1.append(16 * row + col + 8 * block)
48 - for j in range(4):
49 - perm.extend([p + 256 * j for p in perm1])
50 -
51 - perm = numpy.array(perm)
52 - interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
53 - perm = perm.reshape((-1, 8))[:, interleave].ravel() # type: ignore
54 - perm = torch.from_numpy(perm)
24 +# Permutations for Marlin scale shuffling
25 +def get_scale_perms(num_bits):
26 scale_perm = []
27 for i in range(8):
28 scale_perm.extend([i + 8 * j for j in range(8)])
@@ -59,23 +30,21 @@ def _get_perms():
30 for i in range(4):
31 scale_perm_single.extend(
32 [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
62 - return perm, scale_perm, scale_perm_single
63 -
64 -
65 -_perm, _scale_perm, _scale_perm_single = _get_perms()
33 + return scale_perm, scale_perm_single
34
35
36 def get_pack_factor(num_bits):
69 - assert num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS, (
70 - f"Unsupported num_bits = {num_bits}")
37 + assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
38 + ), f"Unsupported num_bits = {num_bits}"
39 return 32 // num_bits
40
41
74 -def marlin_permute_scales(s, size_k, size_n, group_size):
42 +def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
43 + scale_perm, scale_perm_single = get_scale_perms(num_bits)
44 if group_size < size_k and group_size != -1:
76 - s = s.reshape((-1, len(_scale_perm)))[:, _scale_perm]
45 + s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
46 else:
78 - s = s.reshape((-1, len(_scale_perm_single)))[:, _scale_perm_single]
47 + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
48 s = s.reshape((-1, size_n)).contiguous()
49
50 return s
@@ -279,13 +248,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
248 requires_grad=False,
249 )
250 set_weight_attrs(
282 - qweight, {
251 + qweight,
252 + {
253 **extra_weight_attrs,
254 "input_dim": 0,
255 "output_dim": 1,
256 "packed_dim": 0,
257 "pack_factor": self.quant_config.pack_factor,
288 - })
258 + },
259 + )
260
261 # Activation order
262 g_idx = Parameter(
@@ -296,10 +267,13 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
267 requires_grad=False,
268 )
269 # Ignore warning from fused linear layers such as QKVParallelLinear.
299 - set_weight_attrs(g_idx, {
300 - **extra_weight_attrs, "input_dim": 0,
301 - "ignore_warning": True
302 - })
270 + set_weight_attrs(
271 + g_idx,
272 + {
273 + **extra_weight_attrs, "input_dim": 0,
274 + "ignore_warning": True
275 + },
276 + )
277
278 g_idx_sort_indices = Parameter(
279 torch.empty(
@@ -320,29 +294,34 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
294 requires_grad=False,
295 )
296 set_weight_attrs(
323 - scales, {
297 + scales,
298 + {
299 **extra_weight_attrs,
300 "input_dim": scales_and_zp_input_dim,
301 "output_dim": 1,
327 - })
302 + },
303 + )
304
305 # Quantized zero-points
306 qzeros = Parameter(
331 - torch.empty(scales_and_zp_size,
332 - output_size_per_partition //
333 - self.quant_config.pack_factor,
334 - dtype=torch.int32,
335 - device="meta"),
307 + torch.empty(
308 + scales_and_zp_size,
309 + output_size_per_partition // self.quant_config.pack_factor,
310 + dtype=torch.int32,
311 + device="meta",
312 + ),
313 requires_grad=False,
314 )
315 set_weight_attrs(
339 - qzeros, {
316 + qzeros,
317 + {
318 **extra_weight_attrs,
319 "input_dim": scales_and_zp_input_dim,
320 "output_dim": 1,
321 "packed_dim": 1,
322 "pack_factor": self.quant_config.pack_factor,
345 - })
323 + },
324 + )
325
326 # Allocate marlin workspace
327 max_workspace_size = (
@@ -405,13 +384,14 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
384
385 else:
386 # Reset g_idx related tensors
408 - layer.g_idx = Parameter(torch.empty(0,
409 - dtype=torch.int,
410 - device=cur_device),
411 - requires_grad=False)
412 - layer.g_idx_sort_indices = Parameter(torch.empty(
413 - 0, dtype=torch.int, device=cur_device),
414 - requires_grad=False)
387 + layer.g_idx = Parameter(
388 + torch.empty(0, dtype=torch.int, device=cur_device),
389 + requires_grad=False,
390 + )
391 + layer.g_idx_sort_indices = Parameter(
392 + torch.empty(0, dtype=torch.int, device=cur_device),
393 + requires_grad=False,
394 + )
395
396 # Repack weights
397 marlin_qweight = ops.gptq_marlin_repack(
@@ -419,6 +399,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
399 layer.g_idx_sort_indices,
400 part_size_k,
401 part_size_n,
402 + self.quant_config.weight_bits,
403 )
404 replace_tensor("qweight", marlin_qweight)
405
@@ -428,15 +409,28 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
409 if self.quant_config.desc_act:
410 scales_size_k = full_size_k
411
431 - marlin_scales = marlin_permute_scales(layer.scales, scales_size_k,
432 - scales_size_n,
433 - self.quant_config.group_size)
412 + marlin_scales = marlin_permute_scales(
413 + layer.scales,
414 + scales_size_k,
415 + scales_size_n,
416 + self.quant_config.group_size,
417 + self.quant_config.weight_bits,
418 + )
419 replace_tensor("scales", marlin_scales)
420
436 - output = ops.gptq_marlin_gemm(reshaped_x, layer.qweight, layer.scales,
437 - layer.g_idx, layer.g_idx_sort_indices,
438 - layer.workspace, size_m, part_size_n,
439 - part_size_k, layer.is_k_full)
421 + output = ops.gptq_marlin_gemm(
422 + reshaped_x,
423 + layer.qweight,
424 + layer.scales,
425 + layer.g_idx,
426 + layer.g_idx_sort_indices,
427 + layer.workspace,
428 + self.quant_config.weight_bits,
429 + size_m,
430 + part_size_n,
431 + part_size_k,
432 + layer.is_k_full,
433 + )
434
435 if bias is not None:
436 output.add_(bias) # In-place add