A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Support running GPTQ 8-bit models in Marlin (#4533)
alexm-nm committed
May 2, 2024 at 12:56 UTC
7038e8b80303bf6128acbe508dec910183a1be56
7 files changed
+553
-324
csrc/ops.h
+3
-1
@@ -132,6 +132,7 @@ torch::Tensor gptq_marlin_gemm(
132
torch::Tensor &g_idx,
133
torch::Tensor &perm,
134
torch::Tensor &workspace,
135
+ int64_t num_bits,
136
int64_t size_m,
137
int64_t size_n,
138
int64_t size_k,
@@ -141,7 +142,8 @@ torch::Tensor gptq_marlin_repack(
142
torch::Tensor &b_q_weight,
143
torch::Tensor &perm,
144
int64_t size_k,
144
- int64_t size_n);
145
+ int64_t size_n,
146
+ int64_t num_bits);
147
#endif
148
149
void squeezellm_gemm(
csrc/quantization/gptq_marlin/gptq_marlin.cu
+377
-175
@@ -32,7 +32,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
32
int4 *__restrict__ out_int4_ptr, int size_m,
33
int size_k, int block_rows) {}
34
35
-template <const int threads, // number of threads in a threadblock
35
+template <const int num_bits, // number of bits used for weights
36
+ const int threads, // number of threads in a threadblock
37
const int thread_m_blocks, // number of 16x16 blocks in the m
38
// dimension (batchsize) of the threadblock
39
const int thread_n_blocks, // same for n dimension (output)
@@ -62,8 +63,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
63
torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
64
torch::Tensor &b_scales, torch::Tensor &g_idx,
65
torch::Tensor &perm, torch::Tensor &workspace,
65
- int64_t size_m, int64_t size_n, int64_t size_k,
66
- bool is_k_full) {
66
+ int64_t num_bits, int64_t size_m, int64_t size_n,
67
+ int64_t size_k, bool is_k_full) {
68
TORCH_CHECK_NOT_IMPLEMENTED(false,
69
"marlin_gemm(..) requires CUDA_ARCH >= 8.0");
70
return torch::empty({1, 1});
@@ -114,11 +115,21 @@ template <int lut> __device__ inline int lop3(int a, int b, int c) {
115
return res;
116
}
117
118
+// Constructs destination register by taking bytes from 2 sources (based on mask)
119
+template <int start_byte, int mask>
120
+__device__ inline uint32_t prmt(uint32_t a) {
121
+ uint32_t res;
122
+ asm volatile("prmt.b32 %0, %1, %2, %3;\n"
123
+ : "=r"(res)
124
+ : "r"(a), "n"(start_byte), "n"(mask));
125
+ return res;
126
+}
127
+
128
// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
129
// values. We mostly follow the strategy in the link below, with some small
130
// changes:
131
// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
121
-__device__ inline FragB dequant(int q) {
132
+__device__ inline FragB dequant_4bit(int q) {
133
const int LO = 0x000f000f;
134
const int HI = 0x00f000f0;
135
const int EX = 0x64006400;
@@ -139,6 +150,24 @@ __device__ inline FragB dequant(int q) {
150
return frag_b;
151
}
152
153
+__device__ inline FragB dequant_8bit(int q) {
154
+ static constexpr uint32_t mask_for_elt_01 = 0x5250;
155
+ static constexpr uint32_t mask_for_elt_23 = 0x5351;
156
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
157
+
158
+ uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
159
+ uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
160
+
161
+ static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
162
+
163
+ FragB frag_b;
164
+ frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
165
+ *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
166
+ frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
167
+ *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
168
+ return frag_b;
169
+}
170
+
171
// Multiply dequantized values by the corresponding quantization scale; used
172
// only for grouped quantization.
173
__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
@@ -162,6 +191,13 @@ __device__ inline void scale4(FragB &frag_b, FragS &frag_s_1, FragS &frag_s_2,
191
frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
192
}
193
194
+// Given 2 floats multiply by 2 scales (halves)
195
+__device__ inline void scale_float(float *c, FragS &s) {
196
+ __half *s_ptr = reinterpret_cast<__half *>(&s);
197
+ c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
198
+ c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
199
+}
200
+
201
// Wait until barrier reaches `count`, then lock for current threadblock.
202
__device__ inline void barrier_acquire(int *lock, int count) {
203
if (threadIdx.x == 0) {
@@ -250,7 +286,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
286
}
287
}
288
253
-template <const int threads, // number of threads in a threadblock
289
+template <const int num_bits, // number of bits used for weights
290
+ const int threads, // number of threads in a threadblock
291
const int thread_m_blocks, // number of 16x16 blocks in the m
292
// dimension (batchsize) of the threadblock
293
const int thread_n_blocks, // same for n dimension (output)
@@ -286,6 +323,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
323
// configurations, while requiring as few slow global cross-threadblock
324
// reductions as possible.
325
326
+ constexpr int pack_factor = 32 / num_bits;
327
+
328
// For larger GEMMs we run multiple batchsize 64 versions in parallel for a
329
// better partitioning with less reductions
330
int parallel = 1;
@@ -385,21 +424,25 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
424
constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta);
425
426
// B sizes/strides
388
- int b_gl_stride = 16 * prob_n / 32;
389
- constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
427
+ int b_gl_stride = 16 * prob_n / (pack_factor * 4);
428
+ constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
429
+ constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
430
+ constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
431
+
432
int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
391
- int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
392
- constexpr int b_sh_wr_delta = threads;
393
- constexpr int b_sh_rd_delta = threads;
433
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
434
+ constexpr int b_sh_wr_delta = threads * b_thread_vecs;
435
+ constexpr int b_sh_rd_delta = threads * b_thread_vecs;
436
constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
437
constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
438
439
// Scale sizes/strides without act_order
440
int s_gl_stride = prob_n / 8;
441
constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
400
- constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
401
- ? thread_k_blocks / group_blocks
402
- : 1;
442
+ constexpr int s_tb_groups =
443
+ !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
444
+ ? thread_k_blocks / group_blocks
445
+ : 1;
446
constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
447
int s_gl_rd_delta = s_gl_stride;
448
@@ -425,12 +468,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
468
a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
469
a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
470
428
- int b_gl_rd =
429
- b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
471
+ int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
472
+ (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
473
b_gl_rd += b_sh_stride * slice_col;
474
b_gl_rd += b_gl_rd_delta_o * slice_row;
432
- int b_sh_wr = threadIdx.x;
433
- int b_sh_rd = threadIdx.x;
475
+ int b_sh_wr = threadIdx.x * b_thread_vecs;
476
+ int b_sh_rd = threadIdx.x * b_thread_vecs;
477
478
// For act_order
479
constexpr int k_iter_size = tb_k / b_sh_wr_iters;
@@ -442,8 +485,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
485
// No act_order
486
int s_gl_rd;
487
if constexpr (!has_act_order) {
445
- s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
446
- s_sh_stride * slice_col + threadIdx.x;
488
+ if constexpr (group_blocks == -1) {
489
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
490
+ } else {
491
+ s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
492
+ s_sh_stride * slice_col + threadIdx.x;
493
+ }
494
}
495
int s_sh_wr = threadIdx.x;
496
bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
@@ -511,7 +558,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
558
559
// Register storage for double buffer of shared memory reads.
560
FragA frag_a[2][thread_m_blocks];
514
- I4 frag_b_quant[2];
561
+ I4 frag_b_quant[2][b_thread_vecs];
562
FragC frag_c[thread_m_blocks][4][2];
563
FragS frag_s[2][4]; // No act-order
564
FragS act_frag_s[2][4][4]; // For act-order
@@ -575,7 +622,11 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
622
int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
623
#pragma unroll
624
for (int i = 0; i < b_sh_wr_iters; i++) {
578
- cp_async4_stream(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
625
+#pragma unroll
626
+ for (int j = 0; j < b_thread_vecs; j++) {
627
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
628
+ }
629
+
630
B_ptr[i] += b_gl_rd_delta_o;
631
}
632
@@ -602,15 +653,15 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
653
// Only fetch scales if this tile starts a new group
654
if (pipe % (group_blocks / thread_k_blocks) == 0) {
655
if (s_sh_wr_pred) {
605
- cp_async4_stream(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
656
+ cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
657
}
658
s_gl_rd += s_gl_rd_delta;
659
}
660
} else {
661
for (int i = 0; i < s_tb_groups; i++) {
662
if (s_sh_wr_pred) {
612
- cp_async4_stream(&sh_s_stage[i * s_sh_stride + s_sh_wr],
613
- &scales_ptr[s_gl_rd]);
663
+ cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
664
+ &scales_ptr[s_gl_rd]);
665
}
666
s_gl_rd += s_gl_rd_delta;
667
}
@@ -641,14 +692,24 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
692
for (int i = 0; i < thread_m_blocks; i++)
693
ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
694
int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
644
- frag_b_quant[k % 2] = *reinterpret_cast<I4 *>(
645
- &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
695
+
696
+#pragma unroll
697
+ for (int i = 0; i < b_thread_vecs; i++) {
698
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
699
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
700
+ }
701
};
702
703
bool is_same_group[stages];
704
int same_group_id[stages];
705
706
auto init_same_group = [&](int pipe) {
707
+ if constexpr (!has_act_order) {
708
+ is_same_group[pipe] = false;
709
+ same_group_id[pipe] = 0;
710
+ return;
711
+ }
712
+
713
int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
714
int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
715
@@ -767,10 +828,23 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
828
// dequantization and matmul operations.
829
#pragma unroll
830
for (int j = 0; j < 4; j++) {
770
- int b_quant = frag_b_quant[k % 2][j];
771
- int b_quant_shift = b_quant >> 8;
831
+ FragB frag_b0;
832
+ FragB frag_b1;
833
+ if constexpr (num_bits == 4) {
834
+ int b_quant = frag_b_quant[k % 2][0][j];
835
+ int b_quant_shift = b_quant >> 8;
836
+
837
+ frag_b0 = dequant_4bit(b_quant);
838
+ frag_b1 = dequant_4bit(b_quant_shift);
839
773
- FragB frag_b0 = dequant(b_quant);
840
+ } else {
841
+ int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
842
+ int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
843
+ int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
844
+
845
+ frag_b0 = dequant_8bit(b_quant_0);
846
+ frag_b1 = dequant_8bit(b_quant_1);
847
+ }
848
849
// Apply scale to frag_b0
850
if constexpr (has_act_order) {
@@ -782,8 +856,6 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
856
}
857
}
858
785
- FragB frag_b1 = dequant(b_quant_shift);
786
-
859
// Apply scale to frag_b1
860
if constexpr (has_act_order) {
861
scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
@@ -808,13 +880,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
880
// multiple warps that accumulate their partial sums of the same output
881
// location; which we have to reduce over in the end. We do in shared memory.
882
auto thread_block_reduce = [&]() {
811
- constexpr int red_off = threads / b_sh_stride / 2;
883
+ constexpr int red_off = threads / b_sh_stride_threads / 2;
884
if (red_off >= 1) {
813
- int red_idx = threadIdx.x / b_sh_stride;
814
- constexpr int red_sh_stride = b_sh_stride * 4 * 2;
815
- constexpr int red_sh_delta = b_sh_stride;
816
- int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
817
- (threadIdx.x % b_sh_stride);
885
+ int red_idx = threadIdx.x / b_sh_stride_threads;
886
+ constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
887
+ constexpr int red_sh_delta = b_sh_stride_threads;
888
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
889
+ (threadIdx.x % b_sh_stride_threads);
890
891
// Parallel logarithmic shared memory reduction. We make sure to avoid any
892
// unnecessary read or write iterations, e.g., for two warps we write only
@@ -861,7 +933,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
933
};
934
935
// Since multiple threadblocks may process parts of the same column slice, we
864
- // finally have to globally reduce over the results. As the striped portioning
936
+ // finally have to globally reduce over the results. As the striped partitioning
937
// minimizes the number of such reductions and our outputs are usually rather
938
// small, we perform this reduction serially in L2 cache.
939
auto global_reduce = [&](bool first = false, bool last = false) {
@@ -951,13 +1023,15 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1023
auto write = [&](int idx, float c0, float c1, FragS &s) {
1024
half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1025
954
- // For per-column quantization we finally apply the scale here
955
- if constexpr (!has_act_order && group_blocks == -1) {
1026
+ // For per-column quantization we finally apply the scale here (only for
1027
+ // 4-bit)
1028
+ if constexpr (!has_act_order && group_blocks == -1 && num_bits == 4) {
1029
res = __hmul2(res, s[0]);
1030
}
1031
1032
((half2 *)sh)[idx] = res;
1033
};
1034
+
1035
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1036
#pragma unroll
1037
for (int i = 0; i < thread_m_blocks; i++) {
@@ -1023,6 +1097,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1097
// ensure all shared memory accesses are static. Note that both pipelines
1098
// have even length meaning that the next iteration will always start at
1099
// index 0.
1100
+
1101
#pragma unroll
1102
for (int pipe = 0; pipe < stages;) {
1103
#pragma unroll
@@ -1070,23 +1145,63 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1145
// For per-column scales, we only fetch them here in the final step before
1146
// write-out
1147
if constexpr (!has_act_order && group_blocks == -1) {
1073
- if (last) {
1148
+ if constexpr (num_bits == 8) {
1149
if (s_sh_wr_pred) {
1075
- cp_async4_stream(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1150
+ cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1151
}
1152
cp_async_fence();
1153
+ } else {
1154
+ if (last) {
1155
+ if (s_sh_wr_pred) {
1156
+ cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1157
+ }
1158
+ cp_async_fence();
1159
+ }
1160
}
1161
}
1162
1163
thread_block_reduce();
1164
if constexpr (!has_act_order && group_blocks == -1) {
1083
- if (last) {
1165
+ if constexpr (num_bits == 8) {
1166
cp_async_wait<0>();
1167
__syncthreads();
1168
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1169
reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1170
reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1171
}
1172
+
1173
+ } else {
1174
+ if (last) {
1175
+ cp_async_wait<0>();
1176
+ __syncthreads();
1177
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1178
+ reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1179
+ reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1180
+ }
1181
+ }
1182
+ }
1183
+ }
1184
+
1185
+ // For 8-bit channelwise, we apply the scale before the global reduction
1186
+ // that converts the fp32 results to fp16 (so that we avoid possible
1187
+ // overflow in fp16)
1188
+ if constexpr (!has_act_order && group_blocks == -1 && num_bits == 8) {
1189
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1190
+#pragma unroll
1191
+ for (int i = 0; i < thread_m_blocks; i++) {
1192
+#pragma unroll
1193
+ for (int j = 0; j < 4; j++) {
1194
+ scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1195
+ frag_s[j / 2][2 * (j % 2) + 0]);
1196
+ scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1197
+ frag_s[j / 2][2 * (j % 2) + 0]);
1198
+
1199
+ scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1200
+ frag_s[j / 2][2 * (j % 2) + 1]);
1201
+ scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1202
+ frag_s[j / 2][2 * (j % 2) + 1]);
1203
+ }
1204
+ }
1205
}
1206
}
1207
@@ -1125,28 +1240,25 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1240
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1241
}
1242
1128
- // if (blockIdx.x == 0 && threadIdx.x == 0) {
1129
- // printf("Move\n");
1130
- // }
1243
start_pipes();
1244
}
1245
}
1246
}
1247
}
1248
1137
-#define __CALL_IF(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1249
+#define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1250
HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1139
- else if (thread_m_blocks == THREAD_M_BLOCKS && \
1251
+ else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1252
thread_n_blocks == THREAD_N_BLOCKS && \
1253
thread_k_blocks == THREAD_K_BLOCKS && \
1254
has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1255
num_threads == NUM_THREADS) { \
1256
cudaFuncSetAttribute( \
1145
- Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1146
- pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1257
+ Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1258
+ THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1259
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1148
- Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1149
- pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1260
+ Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1261
+ THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1262
<<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1263
A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1264
prob_k, locks); \
@@ -1158,28 +1270,92 @@ typedef struct {
1270
int num_threads;
1271
} thread_config_t;
1272
1161
-thread_config_t small_batch_thread_configs[] = {
1273
+typedef struct {
1274
+ int max_m_blocks;
1275
+ thread_config_t tb_cfg;
1276
+} exec_config_t;
1277
+
1278
+thread_config_t thread_configs[] = {
1279
// Ordered by priority
1280
1281
// thread_k, thread_n, num_threads
1165
- {128, 128, 256}, // Default
1166
- {128, 64, 128}, // Reduce N 2X, same K
1167
- {64, 256, 256}, // Reduce K 2X, increase N 2X
1168
- {64, 128, 128}, // Reduce K 2X, same N
1282
+ {64, 256, 256}, // Default (max cache usage)
1283
+ {64, 128, 128}, // Reduce N, reduce warps
1284
+ {128, 64, 128}, // Reduce N more, but increase K
1285
+
1286
};
1287
1171
-thread_config_t large_batch_thread_configs[] = {
1172
- // Ordered by priority
1288
+int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1289
+ int prob_n, int prob_k, int num_bits, int group_size,
1290
+ bool has_act_order, bool is_k_full) {
1291
+ bool cache_scales_chunk = has_act_order && !is_k_full;
1292
1174
- // thread_k, thread_n, num_threads
1175
- {64, 256, 256}, // Default
1176
- {128, 64, 128}, // Reduce N 2X, same K
1177
- {64, 128, 128}, // Reduce N 2X, same K
1178
- // {128, 64, 128}, // Reduce N 4X, increase K 2X
1179
-};
1293
+ int tb_n = th_config.thread_n;
1294
+ int tb_k = th_config.thread_k;
1295
+
1296
+ // Get max scale groups per thread-block
1297
+ int tb_groups;
1298
+ if (group_size == -1) {
1299
+ tb_groups = 1;
1300
+ } else if (group_size == 0) {
1301
+ tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
1302
+ } else {
1303
+ tb_groups = div_ceil(tb_k, group_size);
1304
+ }
1305
+
1306
+ if (cache_scales_chunk) {
1307
+ int load_groups =
1308
+ tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K
1309
+ load_groups = max(load_groups, 32); // We load at least 32 scale groups
1310
+ return load_groups * tb_n * 2;
1311
+
1312
+ } else {
1313
+ int tb_scales = tb_groups * tb_n * 2;
1314
+
1315
+ return tb_scales * pipe_stages;
1316
+ }
1317
+}
1318
+
1319
+bool is_valid_cache_size(thread_config_t const &th_config, int max_m_blocks,
1320
+ int prob_m, int prob_n, int prob_k, int num_bits,
1321
+ int scales_cache_size, int max_shared_mem) {
1322
+ int pack_factor = 32 / num_bits;
1323
+
1324
+ // Get B size
1325
+ int tb_k = th_config.thread_k;
1326
+ int tb_n = th_config.thread_n;
1327
+
1328
+ int b_size = (tb_k * tb_n / pack_factor) * 4;
1329
+
1330
+ // Get A size
1331
+ int m_blocks = div_ceil(prob_m, 16);
1332
+ int tb_max_m = 16;
1333
1181
-bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
1182
- int prob_k) {
1334
+ while (true) {
1335
+ if (m_blocks >= max_m_blocks) {
1336
+ tb_max_m *= max_m_blocks;
1337
+ break;
1338
+ }
1339
+
1340
+ max_m_blocks--;
1341
+ if (max_m_blocks == 0) {
1342
+ TORCH_CHECK(false, "Unexpected m_blocks = ", m_blocks);
1343
+ }
1344
+ }
1345
+
1346
+ int a_size = (tb_max_m * tb_k) * 2;
1347
+
1348
+ float pipe_size = (a_size + b_size) * pipe_stages;
1349
+
1350
+ TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1351
+
1352
+ return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1353
+}
1354
+
1355
+bool is_valid_config(thread_config_t const &th_config, int max_m_blocks,
1356
+ int prob_m, int prob_n, int prob_k, int num_bits,
1357
+ int group_size, bool has_act_order, bool is_k_full,
1358
+ int max_shared_mem) {
1359
// Sanity
1360
if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1361
th_config.num_threads == -1) {
@@ -1201,62 +1377,79 @@ bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
1377
return false;
1378
}
1379
1380
+ // Determine cache for scales
1381
+ int scales_cache_size =
1382
+ get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits,
1383
+ group_size, has_act_order, is_k_full);
1384
+
1385
+ // Check that pipeline fits into cache
1386
+ if (!is_valid_cache_size(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1387
+ num_bits, scales_cache_size, max_shared_mem)) {
1388
+ return false;
1389
+ }
1390
+
1391
return true;
1392
}
1393
1207
-thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1208
-
1209
- // TODO: Enable if needed after some more testing
1210
- if (prob_m <= 0) {
1211
- for (auto th_config : small_batch_thread_configs) {
1212
- if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1213
- return th_config;
1394
+exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1395
+ int num_bits, int group_size,
1396
+ bool has_act_order, bool is_k_full,
1397
+ int max_shared_mem) {
1398
+ int max_m_blocks = 4;
1399
+ while (max_m_blocks > 0) {
1400
+ for (auto th_config : thread_configs) {
1401
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1402
+ num_bits, group_size, has_act_order, is_k_full,
1403
+ max_shared_mem)) {
1404
+ return exec_config_t{max_m_blocks, th_config};
1405
}
1406
}
1407
1217
- } else {
1218
- for (auto th_config : large_batch_thread_configs) {
1219
- if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1220
- return th_config;
1221
- }
1222
- }
1408
+ printf("WARNING: Marlin kernel is reducing max_m_blocks due to small SM "
1409
+ "GPU cache. This may "
1410
+ "hurt performance. Consider upgrading your GPU.\n");
1411
+
1412
+ max_m_blocks--; // Process less M blocks per invocation to reduce cache
1413
+ // usage
1414
}
1415
1225
- return thread_config_t{-1, -1, -1};
1416
+ return exec_config_t{0, {-1, -1, -1}};
1417
}
1418
1228
-#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1229
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1230
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1231
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1232
- __CALL_IF(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1419
+#define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1420
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1421
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1422
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1423
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1424
\
1234
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1235
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1236
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1237
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1425
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1426
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1427
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1428
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1429
\
1239
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1240
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1241
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1242
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1430
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1431
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1432
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1433
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1434
\
1244
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1245
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1246
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1247
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1435
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1436
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1437
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1438
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1439
\
1249
- __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1250
- __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1251
- __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1252
- __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1253
-
1254
-void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1255
- void *perm, void *a_tmp, int prob_m, int prob_n, int prob_k,
1256
- void *workspace, bool has_act_order, bool is_k_full,
1257
- int num_groups, int group_size, int dev = 0,
1258
- cudaStream_t stream = 0, int thread_k = -1, int thread_n = -1,
1259
- int sms = -1, int max_par = 16) {
1440
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1441
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1442
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1443
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1444
+
1445
+void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1446
+ void *g_idx, void *perm, void *a_tmp, int prob_m,
1447
+ int prob_n, int prob_k, void *workspace, int num_bits,
1448
+ bool has_act_order, bool is_k_full, int num_groups,
1449
+ int group_size, int dev, cudaStream_t stream, int thread_k,
1450
+ int thread_n, int sms, int max_par) {
1451
+ TORCH_CHECK(num_bits == 4 || num_bits == 8,
1452
+ "num_bits must be 4 or 8. Got = ", num_bits);
1453
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1454
", ", prob_n, ", ", prob_k, "]");
1455
@@ -1274,25 +1467,34 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1467
TORCH_CHECK(max_shared_mem > 0);
1468
1469
// Set thread config
1277
- thread_config_t th_config;
1470
+ exec_config_t exec_cfg;
1471
if (thread_k != -1 && thread_n != -1) {
1472
// User-defined config
1280
- th_config = thread_config_t{thread_k, thread_n, default_threads};
1473
+ exec_cfg =
1474
+ exec_config_t{4, thread_config_t{thread_k, thread_n, default_threads}};
1475
} else {
1476
// Auto config
1283
- th_config = determine_thread_config(prob_m, prob_n, prob_k);
1477
+ exec_cfg =
1478
+ determine_thread_config(prob_m, prob_n, prob_k, num_bits, group_size,
1479
+ has_act_order, is_k_full, max_shared_mem);
1480
}
1481
1286
- TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1287
- "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1288
- ", thread_n = " + str(th_config.thread_n) +
1289
- ", num_threads = " + str(th_config.num_threads) +
1290
- " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1291
- str(prob_n) + "]");
1292
-
1293
- int num_threads = th_config.num_threads;
1294
- thread_k = th_config.thread_k;
1295
- thread_n = th_config.thread_n;
1482
+ TORCH_CHECK(exec_cfg.max_m_blocks > 0 &&
1483
+ is_valid_config(exec_cfg.tb_cfg, exec_cfg.max_m_blocks,
1484
+ prob_m, prob_n, prob_k, num_bits, group_size,
1485
+ has_act_order, is_k_full, max_shared_mem),
1486
+ "Invalid thread config: max_m_blocks = ", exec_cfg.max_m_blocks,
1487
+ ", thread_k = ", exec_cfg.tb_cfg.thread_k,
1488
+ ", thread_n = ", exec_cfg.tb_cfg.thread_n,
1489
+ ", num_threads = ", exec_cfg.tb_cfg.num_threads, " for MKN = [",
1490
+ prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
1491
+ ", group_size = ", group_size,
1492
+ ", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
1493
+ ", max_shared_mem = ", max_shared_mem);
1494
+
1495
+ int num_threads = exec_cfg.tb_cfg.num_threads;
1496
+ thread_k = exec_cfg.tb_cfg.thread_k;
1497
+ thread_n = exec_cfg.tb_cfg.thread_n;
1498
1499
int thread_k_blocks = thread_k / 16;
1500
int thread_n_blocks = thread_n / 16;
@@ -1352,28 +1554,32 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1554
}
1555
1556
// Main loop
1355
- for (int i = 0; i < tot_m_blocks; i += 4) {
1557
+ for (int i = 0; i < tot_m_blocks; i += exec_cfg.max_m_blocks) {
1558
int thread_m_blocks = tot_m_blocks - i;
1559
prob_m = tot_m - 16 * i;
1560
int par = 1;
1359
- if (thread_m_blocks > 4) {
1561
+ if (thread_m_blocks > exec_cfg.max_m_blocks) {
1562
// Note that parallel > 1 currently only works for inputs without any
1563
// padding
1362
- par = (16 * thread_m_blocks - pad) / 64;
1564
+ par = (16 * thread_m_blocks - pad) / (16 * exec_cfg.max_m_blocks);
1565
if (par > max_par)
1566
par = max_par;
1365
- prob_m = 64 * par;
1366
- i += 4 * (par - 1);
1367
- thread_m_blocks = 4;
1567
+ prob_m = (16 * exec_cfg.max_m_blocks) * par;
1568
+ i += exec_cfg.max_m_blocks * (par - 1);
1569
+ thread_m_blocks = exec_cfg.max_m_blocks;
1570
}
1571
1572
// Define kernel configurations
1573
if (false) {
1574
}
1373
- CALL_IF(16, 4, 256)
1374
- CALL_IF(8, 8, 256)
1375
- CALL_IF(8, 4, 128)
1376
- CALL_IF(4, 8, 128)
1575
+ CALL_IF(4, 32, 2, 256)
1576
+ CALL_IF(4, 16, 4, 256)
1577
+ CALL_IF(4, 8, 4, 128)
1578
+ CALL_IF(4, 4, 8, 128)
1579
+ CALL_IF(8, 32, 2, 256)
1580
+ CALL_IF(8, 16, 4, 256)
1581
+ CALL_IF(8, 8, 4, 128)
1582
+ CALL_IF(8, 4, 8, 128)
1583
else {
1584
TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1585
str(prob_n) + ", " + str(prob_k) + "]" +
@@ -1395,33 +1601,32 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1601
torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1602
torch::Tensor &b_scales, torch::Tensor &g_idx,
1603
torch::Tensor &perm, torch::Tensor &workspace,
1398
- int64_t size_m, int64_t size_n, int64_t size_k,
1399
- bool is_k_full) {
1604
+ int64_t num_bits, int64_t size_m, int64_t size_n,
1605
+ int64_t size_k, bool is_k_full) {
1606
+ // Verify num_bits
1607
+ TORCH_CHECK(num_bits == 4 || num_bits == 8,
1608
+ "num_bits must be 4 or 8. Got = ", num_bits);
1609
+ int pack_factor = 32 / num_bits;
1610
+
1611
// Verify A
1401
- TORCH_CHECK(a.size(0) == size_m,
1402
- "Shape mismatch: a.size(0) = " + str(a.size(0)) +
1403
- ", size_m = " + str(size_m));
1404
- TORCH_CHECK(a.size(1) == size_k,
1405
- "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1406
- ", size_k = " + str(size_k));
1612
+ TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
1613
+ ", size_m = ", size_m);
1614
+ TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1),
1615
+ ", size_k = ", size_k);
1616
1617
// Verify B
1409
- TORCH_CHECK(size_k % gptq_marlin::tile_size == 0,
1410
- "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1411
- str(gptq_marlin::tile_size));
1618
+ TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1619
+ " is not divisible by tile_size = ", gptq_marlin::tile_size);
1620
TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
1413
- "Shape mismatch: b_q_weight.size(0) = " +
1414
- str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1415
- ", tile_size = " + str(gptq_marlin::tile_size));
1416
- TORCH_CHECK(
1417
- b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1418
- "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1419
- " is not divisible by tile_size = " + str(gptq_marlin::tile_size));
1420
- int actual_size_n = (b_q_weight.size(1) / gptq_marlin::tile_size) *
1421
- gptq_marlin::pack_factor_4bit;
1422
- TORCH_CHECK(size_n == actual_size_n,
1423
- "size_n = " + str(size_n) +
1424
- ", actual_size_n = " + str(actual_size_n));
1621
+ "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1622
+ ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1623
+ TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1624
+ "b_q_weight.size(1) = ", b_q_weight.size(1),
1625
+ " is not divisible by tile_size = ", gptq_marlin::tile_size);
1626
+ int actual_size_n =
1627
+ (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
1628
+ TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
1629
+ ", actual_size_n = ", actual_size_n);
1630
1631
// Verify device and strides
1632
TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
@@ -1457,9 +1662,9 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1662
// Verify g_idx and perm
1663
TORCH_CHECK((g_idx.size(0) == 0 && perm.size(0) == 0) ||
1664
(g_idx.size(0) == size_k && perm.size(0) == size_k),
1460
- "Unexpected g_idx.size(0) = " + str(g_idx.size(0)) +
1461
- " and perm.size(0) = " + str(perm.size(0)) +
1462
- ", where size_k = " + str(size_k));
1665
+ "Unexpected g_idx.size(0) = ", g_idx.size(0),
1666
+ " and perm.size(0) = ", perm.size(0),
1667
+ ", where size_k = ", size_k);
1668
1669
// Detect groupsize and act_order
1670
int num_groups = -1;
@@ -1475,9 +1680,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1680
if (has_act_order) {
1681
if (is_k_full) {
1682
TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
1478
- TORCH_CHECK(size_k % num_groups == 0,
1479
- "size_k = " + str(size_k) +
1480
- ", is not divisible by num_groups = " + str(num_groups));
1683
+ TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
1684
+ ", is not divisible by num_groups = ", num_groups);
1685
group_size = size_k / num_groups;
1686
} else {
1687
group_size = 0;
@@ -1485,10 +1689,9 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1689
1690
} else {
1691
if (num_groups > 1) {
1488
- TORCH_CHECK(size_k % num_groups == 0,
1489
- "size_k = " + str(size_k) +
1490
- ", is not divisible by b_scales.size(0) = " +
1491
- str(b_scales.size(0)));
1692
+ TORCH_CHECK(
1693
+ size_k % num_groups == 0, "size_k = ", size_k,
1694
+ ", is not divisible by b_scales.size(0) = ", b_scales.size(0));
1695
group_size = size_k / num_groups;
1696
} else {
1697
group_size = -1;
@@ -1496,23 +1699,22 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1699
}
1700
1701
// Verify workspace size
1499
- TORCH_CHECK(size_n % gptq_marlin::min_thread_n == 0,
1500
- "size_n = " + str(size_n) +
1501
- ", is not divisible by min_thread_n = " +
1502
- str(gptq_marlin::min_thread_n));
1702
+ TORCH_CHECK(
1703
+ size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1704
+ ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1705
int min_workspace_size =
1706
(size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
1707
TORCH_CHECK(workspace.numel() >= min_workspace_size,
1506
- "workspace.numel = " + str(workspace.numel()) +
1507
- " is below min_workspace_size = " + str(min_workspace_size));
1708
+ "workspace.numel = ", workspace.numel(),
1709
+ " is below min_workspace_size = ", min_workspace_size);
1710
1711
int dev = a.get_device();
1510
- gptq_marlin::marlin_cuda(
1712
+ gptq_marlin::marlin_mm_f16i4(
1713
a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), b_scales.data_ptr(),
1714
g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(), size_m, size_n,
1513
- size_k, workspace.data_ptr(), has_act_order, is_k_full, num_groups,
1514
- group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n,
1515
- sms, gptq_marlin::max_par);
1715
+ size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1716
+ num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1717
+ thread_k, thread_n, sms, gptq_marlin::max_par);
1718
1719
return c;
1720
}
csrc/quantization/gptq_marlin/gptq_marlin.cuh
+2
-6
@@ -24,8 +24,6 @@ static constexpr int min_thread_k = 64;
24
static constexpr int tile_size = 16;
25
static constexpr int max_par = 16;
26
27
-static constexpr int pack_factor_4bit = 8; // We have 8 4-bit vals inside a 32 bit
28
-
27
template <typename T, int n>
28
struct Vec {
29
T elems[n];
@@ -51,13 +49,11 @@ __device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool
49
"r"(smem), "l"(glob_ptr), "n"(BYTES));
50
}
51
54
-__device__ inline void cp_async4_stream(void* smem_ptr, const void* glob_ptr) {
52
+__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
53
const int BYTES = 16;
54
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55
asm volatile("{\n"
58
- " .reg .b64 p;\n"
59
- " createpolicy.fractional.L2::evict_first.b64 p, 1.0;"
60
- " cp.async.cg.shared.global.L2::cache_hint [%0], [%1], %2, p;\n"
56
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
57
"}\n" ::"r"(smem),
58
"l"(glob_ptr), "n"(BYTES));
59
}
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+90
-62
@@ -11,7 +11,7 @@ static constexpr int tile_n_size = tile_k_size * 4;
11
12
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
13
14
-template <int const num_threads, bool const has_perm>
14
+template <int const num_threads, int const num_bits, bool const has_perm>
15
__global__ void
16
marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
17
uint32_t const *__restrict__ perm_ptr,
@@ -20,7 +20,8 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
20
} // namespace gptq_marlin
21
22
torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
23
- int64_t size_k, int64_t size_n) {
23
+ int64_t size_k, int64_t size_n,
24
+ int64_t num_bits) {
25
TORCH_CHECK_NOT_IMPLEMENTED(
26
false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
27
return torch::empty({1, 1});
@@ -28,11 +29,13 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
29
30
#else
31
31
-template <int const num_threads, bool const has_perm>
32
+template <int const num_threads, int const num_bits, bool const has_perm>
33
__global__ void
34
marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
35
uint32_t const *__restrict__ perm_ptr,
36
uint32_t *__restrict__ out_ptr, int size_k, int size_n) {
37
+ constexpr int pack_factor = 32 / num_bits;
38
+
39
int k_tiles = size_k / tile_k_size;
40
int n_tiles = size_n / tile_n_size;
41
int block_k_tiles = div_ceil(k_tiles, gridDim.x);
@@ -64,9 +67,10 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
67
sh_pipe_ptr += perm_size;
68
}
69
70
+ constexpr int tile_ints = tile_k_size / pack_factor;
71
+
72
constexpr int stage_n_threads = tile_n_size / 4;
68
- constexpr int stage_k_threads =
69
- has_perm ? tile_k_size : tile_k_size / pack_factor_4bit;
73
+ constexpr int stage_k_threads = has_perm ? tile_k_size : tile_ints;
74
constexpr int stage_size = stage_k_threads * stage_n_threads;
75
76
auto load_perm_to_shared = [&](int k_tile_id) {
@@ -99,9 +103,9 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
103
reinterpret_cast<uint32_t const *>(sh_perm_ptr);
104
105
int src_k = sh_perm_int_ptr[k_id];
102
- int src_k_packed = src_k / pack_factor_4bit;
106
+ int src_k_packed = src_k / pack_factor;
107
104
- cp_async4_stream(
108
+ cp_async4(
109
&sh_ptr[k_id * stage_n_threads + n_id],
110
reinterpret_cast<int4 const *>(&(
111
b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)])));
@@ -113,12 +117,12 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
117
int n_id = threadIdx.x % stage_n_threads;
118
119
int first_k = k_tile_id * tile_k_size;
116
- int first_k_packed = first_k / pack_factor_4bit;
120
+ int first_k_packed = first_k / pack_factor;
121
118
- cp_async4_stream(&sh_ptr[k_id * stage_n_threads + n_id],
119
- reinterpret_cast<int4 const *>(
120
- &(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
121
- first_n + (n_id * 4)])));
122
+ cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
123
+ reinterpret_cast<int4 const *>(
124
+ &(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
125
+ first_n + (n_id * 4)])));
126
}
127
}
128
@@ -145,26 +149,27 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
149
int cur_n = warp_id * 16 + tc_col;
150
151
constexpr int sh_stride = 64;
152
+ constexpr uint32_t mask = (1 << num_bits) - 1;
153
154
int4 *sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
155
uint32_t *sh_stage_int_ptr = reinterpret_cast<uint32_t *>(sh_stage_ptr);
156
157
uint32_t *sh_perm_int_ptr = reinterpret_cast<uint32_t *>(sh_perm_ptr);
158
154
- uint32_t vals[pack_factor_4bit];
159
+ uint32_t vals[8];
160
161
if constexpr (has_perm) {
162
for (int i = 0; i < 4; i++) {
163
int k_idx = tc_row + tc_offsets[i];
164
165
uint32_t src_k = sh_perm_int_ptr[k_idx];
161
- uint32_t src_k_pos = src_k % pack_factor_4bit;
166
+ uint32_t src_k_pos = src_k % pack_factor;
167
168
uint32_t b1_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n];
164
- uint32_t b1_cur_val = (b1_val >> (src_k_pos * 4)) & 0xf;
169
+ uint32_t b1_cur_val = (b1_val >> (src_k_pos * num_bits)) & mask;
170
171
uint32_t b2_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n + 8];
167
- uint32_t b2_cur_val = (b2_val >> (src_k_pos * 4)) & 0xf;
172
+ uint32_t b2_cur_val = (b2_val >> (src_k_pos * num_bits)) & mask;
173
174
vals[i] = b1_cur_val;
175
vals[4 + i] = b2_cur_val;
@@ -172,41 +177,56 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
177
178
} else {
179
175
- uint32_t b1_val_1 = sh_stage_int_ptr[cur_n];
176
- uint32_t b1_val_2 = sh_stage_int_ptr[sh_stride + cur_n];
177
-
178
- uint32_t b2_val_1 = sh_stage_int_ptr[cur_n + 8];
179
- uint32_t b2_val_2 = sh_stage_int_ptr[sh_stride + cur_n + 8];
180
+ uint32_t b1_vals[tile_ints];
181
+ uint32_t b2_vals[tile_ints];
182
183
#pragma unroll
182
- for (int i = 0; i < 2; i++) {
183
- int cur_elem = tc_row + tc_offsets[i];
184
- vals[i] = (b1_val_1 >> (cur_elem * 4)) & 0xf;
185
- vals[4 + i] = (b2_val_1 >> (cur_elem * 4)) & 0xf;
184
+ for (int i = 0; i < tile_ints; i++) {
185
+ b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
186
+ b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
187
}
188
189
#pragma unroll
189
- for (int i = 2; i < 4; i++) {
190
- int cur_elem = tc_row + tc_offsets[i] - 8;
191
- vals[i] = (b1_val_2 >> (cur_elem * 4)) & 0xf;
192
- vals[4 + i] = (b2_val_2 >> (cur_elem * 4)) & 0xf;
190
+ for (int i = 0; i < 4; i++) {
191
+ int cur_elem = tc_row + tc_offsets[i];
192
+ int cur_int = cur_elem / pack_factor;
193
+ int cur_pos = cur_elem % pack_factor;
194
+
195
+ vals[i] = (b1_vals[cur_int] >> (cur_pos * num_bits)) & mask;
196
+ vals[4 + i] = (b2_vals[cur_int] >> (cur_pos * num_bits)) & mask;
197
}
198
}
199
200
+ constexpr int tile_size = tile_k_size * tile_n_size / pack_factor;
201
+ int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
202
+
203
// Result of:
204
// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
198
- constexpr int pack_idx[pack_factor_4bit] = {0, 2, 4, 6, 1, 3, 5, 7};
205
+ if constexpr (num_bits == 4) {
206
+ constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
207
200
- uint32_t res = 0;
208
+ uint32_t res = 0;
209
#pragma unroll
202
- for (int i = 0; i < pack_factor_4bit; i++) {
203
- res |= vals[pack_idx[i]] << (i * 4);
204
- }
210
+ for (int i = 0; i < 8; i++) {
211
+ res |= vals[pack_idx[i]] << (i * 4);
212
+ }
213
206
- constexpr int tile_size = tile_k_size * tile_n_size / pack_factor_4bit;
207
- int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
214
+ out_ptr[out_offset + th_id * 4 + warp_id] = res;
215
209
- out_ptr[out_offset + th_id * 4 + warp_id] = res;
216
+ } else {
217
+ constexpr int pack_idx[4] = {0, 2, 1, 3};
218
+
219
+ uint32_t res1 = 0;
220
+ uint32_t res2 = 0;
221
+#pragma unroll
222
+ for (int i = 0; i < 4; i++) {
223
+ res1 |= vals[pack_idx[i]] << (i * 8);
224
+ res2 |= vals[4 + pack_idx[i]] << (i * 8);
225
+ }
226
+
227
+ out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1;
228
+ out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2;
229
+ }
230
};
231
232
auto start_pipes = [&](int k_tile_id, int n_tile_id) {
@@ -242,19 +262,35 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
262
263
} // namespace gptq_marlin
264
265
+#define CALL_IF(NUM_BITS, HAS_PERM) \
266
+ else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
267
+ cudaFuncSetAttribute( \
268
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
269
+ NUM_BITS, HAS_PERM>, \
270
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
271
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
272
+ HAS_PERM> \
273
+ <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
274
+ b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
275
+ }
276
+
277
torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
246
- int64_t size_k, int64_t size_n) {
278
+ int64_t size_k, int64_t size_n,
279
+ int64_t num_bits) {
280
// Verify compatibility with marlin tile of 16x64
281
TORCH_CHECK(size_k % gptq_marlin::tile_k_size == 0, "size_k = ", size_k,
282
" is not divisible by tile_k_size = ", gptq_marlin::tile_k_size);
283
TORCH_CHECK(size_n % gptq_marlin::tile_n_size == 0, "size_n = ", size_n,
284
" is not divisible by tile_n_size = ", gptq_marlin::tile_n_size);
285
286
+ TORCH_CHECK(num_bits == 4 || num_bits == 8,
287
+ "num_bits must be 4 or 8. Got = ", num_bits);
288
+ int const pack_factor = 32 / num_bits;
289
+
290
// Verify B
254
- TORCH_CHECK((size_k / gptq_marlin::pack_factor_4bit) == b_q_weight.size(0),
291
+ TORCH_CHECK((size_k / pack_factor) == b_q_weight.size(0),
292
"Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
256
- ", size_k = ", size_k,
257
- ", pack_factor_4bit = ", gptq_marlin::pack_factor_4bit);
293
+ ", size_k = ", size_k, ", pack_factor = ", pack_factor);
294
TORCH_CHECK(b_q_weight.size(1) == size_n,
295
"b_q_weight.size(1) = ", b_q_weight.size(1),
296
" is not size_n = ", size_n);
@@ -273,10 +309,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
309
auto options = torch::TensorOptions()
310
.dtype(b_q_weight.dtype())
311
.device(b_q_weight.device());
276
- torch::Tensor out = torch::empty(
277
- {size_k / gptq_marlin::tile_size,
278
- size_n * gptq_marlin::tile_size / gptq_marlin::pack_factor_4bit},
279
- options);
312
+ torch::Tensor out =
313
+ torch::empty({size_k / gptq_marlin::tile_size,
314
+ size_n * gptq_marlin::tile_size / pack_factor},
315
+ options);
316
317
// Detect if there is act_order
318
bool has_perm = perm.size(0) != 0;
@@ -299,23 +335,15 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
335
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
336
TORCH_CHECK(max_shared_mem > 0);
337
302
- if (has_perm) {
303
- cudaFuncSetAttribute(
304
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, true>,
305
- cudaFuncAttributeMaxDynamicSharedMemorySize,
306
- max_shared_mem);
307
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, true>
308
- <<<blocks, gptq_marlin::repack_threads, max_shared_mem,
309
- stream>>>(b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n);
310
-
311
- } else {
312
- cudaFuncSetAttribute(
313
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, false>,
314
- cudaFuncAttributeMaxDynamicSharedMemorySize,
315
- max_shared_mem);
316
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, false>
317
- <<<blocks, gptq_marlin::repack_threads, max_shared_mem,
318
- stream>>>(b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n);
338
+ if (false) {
339
+ }
340
+ CALL_IF(4, false)
341
+ CALL_IF(4, true)
342
+ CALL_IF(8, false)
343
+ CALL_IF(8, true)
344
+ else {
345
+ TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits,
346
+ ", has_perm = ", has_perm);
347
}
348
349
return out;
tests/models/test_gptq_marlin.py
+9
-4
@@ -39,6 +39,13 @@ MODELS = [
39
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-64g-actorder_True"),
40
# act_order==True, group_size=32
41
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-32g-actorder_True"),
42
+
43
+ # 8-bit, act_order==True, group_size=channelwise
44
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
45
+ # 8-bit, act_order==True, group_size=128
46
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-128g-actorder_True"),
47
+ # 8-bit, act_order==True, group_size=32
48
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-32g-actorder_True"),
49
]
50
51
@@ -65,8 +72,7 @@ def test_models(
72
dtype=dtype,
73
quantization="marlin",
74
max_model_len=MAX_MODEL_LEN,
68
- tensor_parallel_size=1,
69
- disable_custom_all_reduce=True)
75
+ tensor_parallel_size=1)
76
77
gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
78
example_prompts, max_tokens, num_logprobs)
@@ -78,8 +84,7 @@ def test_models(
84
dtype=dtype,
85
quantization="gptq",
86
max_model_len=MAX_MODEL_LEN,
81
- tensor_parallel_size=1,
82
- disable_custom_all_reduce=True)
87
+ tensor_parallel_size=1)
88
gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
89
max_tokens,
90
num_logprobs)
vllm/_custom_ops.py
+8
-6
@@ -169,18 +169,20 @@ def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
169
170
# gptq_marlin
171
def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
172
- size_k: int, size_n: int) -> torch.Tensor:
173
- return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n)
172
+ size_k: int, size_n: int,
173
+ num_bits: int) -> torch.Tensor:
174
+ return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n,
175
+ num_bits)
176
177
178
def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
179
b_scales: torch.Tensor, g_idx: torch.Tensor,
178
- perm: torch.Tensor, workspace: torch.Tensor, size_m: int,
179
- size_n: int, size_k: int,
180
+ perm: torch.Tensor, workspace: torch.Tensor,
181
+ num_bits: int, size_m: int, size_n: int, size_k: int,
182
is_k_full: bool) -> torch.Tensor:
183
return vllm_ops.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
182
- workspace, size_m, size_n, size_k,
183
- is_k_full)
184
+ workspace, num_bits, size_m, size_n,
185
+ size_k, is_k_full)
186
187
188
# fp8
vllm/model_executor/layers/quantization/gptq_marlin.py
+64
-70
@@ -2,7 +2,6 @@ import enum
2
from enum import Enum
3
from typing import Any, Dict, List, Optional
4
5
-import numpy
5
import torch
6
from torch.nn.parameter import Parameter
7
@@ -17,41 +16,13 @@ GPTQ_MARLIN_MIN_THREAD_N = 64
16
GPTQ_MARLIN_MIN_THREAD_K = 128
17
GPTQ_MARLIN_MAX_PARALLEL = 16
18
20
-GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4]
19
+GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
20
GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
21
GPTQ_MARLIN_SUPPORTED_SYM = [True]
22
23
25
-# Precompute permutations for Marlin weight and scale shuffling
26
-#
27
-# Marlin works on [16,64] tiles. The goal of the permutations
28
-# is to reorder the weight data so that it is compatible
29
-# with the tensor-core format that is described here:
30
-# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
31
-#
32
-# As a result of this reordering, the vector loads inside the
33
-# kernel will get the data as it is needed for tensor-core
34
-# (without the need to use ldmatrix instructions)
35
-def _get_perms():
36
- perm = []
37
- for i in range(32):
38
- perm1 = []
39
- col = i // 4
40
- for block in [0, 1]:
41
- for row in [
42
- 2 * (i % 4),
43
- 2 * (i % 4) + 1,
44
- 2 * (i % 4 + 4),
45
- 2 * (i % 4 + 4) + 1,
46
- ]:
47
- perm1.append(16 * row + col + 8 * block)
48
- for j in range(4):
49
- perm.extend([p + 256 * j for p in perm1])
50
-
51
- perm = numpy.array(perm)
52
- interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
53
- perm = perm.reshape((-1, 8))[:, interleave].ravel() # type: ignore
54
- perm = torch.from_numpy(perm)
24
+# Permutations for Marlin scale shuffling
25
+def get_scale_perms(num_bits):
26
scale_perm = []
27
for i in range(8):
28
scale_perm.extend([i + 8 * j for j in range(8)])
@@ -59,23 +30,21 @@ def _get_perms():
30
for i in range(4):
31
scale_perm_single.extend(
32
[2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
62
- return perm, scale_perm, scale_perm_single
63
-
64
-
65
-_perm, _scale_perm, _scale_perm_single = _get_perms()
33
+ return scale_perm, scale_perm_single
34
35
36
def get_pack_factor(num_bits):
69
- assert num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS, (
70
- f"Unsupported num_bits = {num_bits}")
37
+ assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
38
+ ), f"Unsupported num_bits = {num_bits}"
39
return 32 // num_bits
40
41
74
-def marlin_permute_scales(s, size_k, size_n, group_size):
42
+def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
43
+ scale_perm, scale_perm_single = get_scale_perms(num_bits)
44
if group_size < size_k and group_size != -1:
76
- s = s.reshape((-1, len(_scale_perm)))[:, _scale_perm]
45
+ s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
46
else:
78
- s = s.reshape((-1, len(_scale_perm_single)))[:, _scale_perm_single]
47
+ s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
48
s = s.reshape((-1, size_n)).contiguous()
49
50
return s
@@ -279,13 +248,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
248
requires_grad=False,
249
)
250
set_weight_attrs(
282
- qweight, {
251
+ qweight,
252
+ {
253
**extra_weight_attrs,
254
"input_dim": 0,
255
"output_dim": 1,
256
"packed_dim": 0,
257
"pack_factor": self.quant_config.pack_factor,
288
- })
258
+ },
259
+ )
260
261
# Activation order
262
g_idx = Parameter(
@@ -296,10 +267,13 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
267
requires_grad=False,
268
)
269
# Ignore warning from fused linear layers such as QKVParallelLinear.
299
- set_weight_attrs(g_idx, {
300
- **extra_weight_attrs, "input_dim": 0,
301
- "ignore_warning": True
302
- })
270
+ set_weight_attrs(
271
+ g_idx,
272
+ {
273
+ **extra_weight_attrs, "input_dim": 0,
274
+ "ignore_warning": True
275
+ },
276
+ )
277
278
g_idx_sort_indices = Parameter(
279
torch.empty(
@@ -320,29 +294,34 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
294
requires_grad=False,
295
)
296
set_weight_attrs(
323
- scales, {
297
+ scales,
298
+ {
299
**extra_weight_attrs,
300
"input_dim": scales_and_zp_input_dim,
301
"output_dim": 1,
327
- })
302
+ },
303
+ )
304
305
# Quantized zero-points
306
qzeros = Parameter(
331
- torch.empty(scales_and_zp_size,
332
- output_size_per_partition //
333
- self.quant_config.pack_factor,
334
- dtype=torch.int32,
335
- device="meta"),
307
+ torch.empty(
308
+ scales_and_zp_size,
309
+ output_size_per_partition // self.quant_config.pack_factor,
310
+ dtype=torch.int32,
311
+ device="meta",
312
+ ),
313
requires_grad=False,
314
)
315
set_weight_attrs(
339
- qzeros, {
316
+ qzeros,
317
+ {
318
**extra_weight_attrs,
319
"input_dim": scales_and_zp_input_dim,
320
"output_dim": 1,
321
"packed_dim": 1,
322
"pack_factor": self.quant_config.pack_factor,
345
- })
323
+ },
324
+ )
325
326
# Allocate marlin workspace
327
max_workspace_size = (
@@ -405,13 +384,14 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
384
385
else:
386
# Reset g_idx related tensors
408
- layer.g_idx = Parameter(torch.empty(0,
409
- dtype=torch.int,
410
- device=cur_device),
411
- requires_grad=False)
412
- layer.g_idx_sort_indices = Parameter(torch.empty(
413
- 0, dtype=torch.int, device=cur_device),
414
- requires_grad=False)
387
+ layer.g_idx = Parameter(
388
+ torch.empty(0, dtype=torch.int, device=cur_device),
389
+ requires_grad=False,
390
+ )
391
+ layer.g_idx_sort_indices = Parameter(
392
+ torch.empty(0, dtype=torch.int, device=cur_device),
393
+ requires_grad=False,
394
+ )
395
396
# Repack weights
397
marlin_qweight = ops.gptq_marlin_repack(
@@ -419,6 +399,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
399
layer.g_idx_sort_indices,
400
part_size_k,
401
part_size_n,
402
+ self.quant_config.weight_bits,
403
)
404
replace_tensor("qweight", marlin_qweight)
405
@@ -428,15 +409,28 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
409
if self.quant_config.desc_act:
410
scales_size_k = full_size_k
411
431
- marlin_scales = marlin_permute_scales(layer.scales, scales_size_k,
432
- scales_size_n,
433
- self.quant_config.group_size)
412
+ marlin_scales = marlin_permute_scales(
413
+ layer.scales,
414
+ scales_size_k,
415
+ scales_size_n,
416
+ self.quant_config.group_size,
417
+ self.quant_config.weight_bits,
418
+ )
419
replace_tensor("scales", marlin_scales)
420
436
- output = ops.gptq_marlin_gemm(reshaped_x, layer.qweight, layer.scales,
437
- layer.g_idx, layer.g_idx_sort_indices,
438
- layer.workspace, size_m, part_size_n,
439
- part_size_k, layer.is_k_full)
421
+ output = ops.gptq_marlin_gemm(
422
+ reshaped_x,
423
+ layer.qweight,
424
+ layer.scales,
425
+ layer.g_idx,
426
+ layer.g_idx_sort_indices,
427
+ layer.workspace,
428
+ self.quant_config.weight_bits,
429
+ size_m,
430
+ part_size_n,
431
+ part_size_k,
432
+ layer.is_k_full,
433
+ )
434
435
if bias is not None:
436
output.add_(bias) # In-place add