A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Marlin Expansion: Support AutoGPTQ Models with Marlin (#3922)
Co-authored-by: alexm <alexm@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Robert Shaw committed
Apr 29, 2024 at 12:35 UTC
73c8d677e57e42374bcfb2271b8f1cf7f2c0a486
14 files changed
+2626
-104
CMakeLists.txt
+2
@@ -177,6 +177,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
177
"csrc/quantization/aqlm/gemm_kernels.cu"
178
"csrc/quantization/awq/gemm_kernels.cu"
179
"csrc/quantization/marlin/marlin_cuda_kernel.cu"
180
+ "csrc/quantization/gptq_marlin/gptq_marlin.cu"
181
+ "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
182
"csrc/custom_all_reduce.cu")
183
endif()
184
csrc/ops.h
+18
@@ -124,6 +124,24 @@ torch::Tensor marlin_gemm(
124
int64_t size_m,
125
int64_t size_n,
126
int64_t size_k);
127
+
128
+torch::Tensor gptq_marlin_gemm(
129
+ torch::Tensor &a,
130
+ torch::Tensor &b_q_weight,
131
+ torch::Tensor &b_scales,
132
+ torch::Tensor &g_idx,
133
+ torch::Tensor &perm,
134
+ torch::Tensor &workspace,
135
+ int64_t size_m,
136
+ int64_t size_n,
137
+ int64_t size_k,
138
+ bool is_k_full);
139
+
140
+torch::Tensor gptq_marlin_repack(
141
+ torch::Tensor &b_q_weight,
142
+ torch::Tensor &perm,
143
+ int64_t size_k,
144
+ int64_t size_n);
145
#endif
146
147
void squeezellm_gemm(
csrc/pybind.cpp
+2
@@ -67,6 +67,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
67
ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
68
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69
ops.def("marlin_gemm", &marlin_gemm, "Marlin Optimized Quantized GEMM for GPTQ");
70
+ ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
71
+ ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
72
ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
73
#endif
74
csrc/quantization/gptq_marlin/gptq_marlin.cu
new
+1520
@@ -0,0 +1,1520 @@
1
+/*
2
+ * Modified by Neural Magic
3
+ * Copyright (C) Marlin.2024 Elias Frantar
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * http://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+
18
+/*
19
+ * Adapted from https://github.com/IST-DASLab/marlin
20
+ */
21
+
22
+#include "gptq_marlin.cuh"
23
+
24
+template <typename T> inline std::string str(T x) { return std::to_string(x); }
25
+
26
+namespace gptq_marlin {
27
+
28
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
29
+
30
+__global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
31
+ int const *__restrict__ perm_int_ptr,
32
+ int4 *__restrict__ out_int4_ptr, int size_m,
33
+ int size_k, int block_rows) {}
34
+
35
+template <const int threads, // number of threads in a threadblock
36
+ const int thread_m_blocks, // number of 16x16 blocks in the m
37
+ // dimension (batchsize) of the threadblock
38
+ const int thread_n_blocks, // same for n dimension (output)
39
+ const int thread_k_blocks, // same for k dimension (reduction)
40
+ const int stages, // number of stages for the async global->shared
41
+ // fetch pipeline
42
+ const bool has_act_order, // whether act_order is enabled
43
+ const int group_blocks = -1 // number of consecutive 16x16 blocks with
44
+ // a separate quantization scale
45
+ >
46
+__global__ void
47
+Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
48
+ const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
49
+ int4 *__restrict__ C, // fp16 output buffer of shape mxn
50
+ const int4 *__restrict__ scales_ptr, // fp16 quantization scales of shape
51
+ // (k/groupsize)xn
52
+ const int *__restrict__ g_idx, // int32 group indices of shape k
53
+ int num_groups, // number of scale groups per output channel
54
+ int prob_m, // batch dimension m
55
+ int prob_n, // output dimension n
56
+ int prob_k, // reduction dimension k
57
+ int *locks // extra global storage for barrier synchronization
58
+) {}
59
+
60
+} // namespace gptq_marlin
61
+
62
+torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
63
+ torch::Tensor &b_scales, torch::Tensor &g_idx,
64
+ torch::Tensor &perm, torch::Tensor &workspace,
65
+ int64_t size_m, int64_t size_n, int64_t size_k,
66
+ bool is_k_full) {
67
+ TORCH_CHECK_NOT_IMPLEMENTED(false,
68
+ "marlin_gemm(..) requires CUDA_ARCH >= 8.0");
69
+ return torch::empty({1, 1});
70
+}
71
+
72
+#else
73
+
74
+// Matrix fragments for tensor core instructions; their precise layout is
75
+// documented here:
76
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
77
+using FragA = Vec<half2, 4>;
78
+using FragB = Vec<half2, 2>;
79
+using FragC = Vec<float, 4>;
80
+using FragS = Vec<half2, 1>; // quantization scales
81
+
82
+// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
83
+// output/accumulation.
84
+__device__ inline void mma(const FragA &a_frag, const FragB &frag_b,
85
+ FragC &frag_c) {
86
+ const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
87
+ const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
88
+ float *c = reinterpret_cast<float *>(&frag_c);
89
+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
90
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
91
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
92
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
93
+ "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
94
+}
95
+
96
+// Instruction for loading a full 16x16 matrix fragment of operand A from shared
97
+// memory, directly in tensor core layout.
98
+__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
99
+ uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
100
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
101
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
102
+ : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
103
+ : "r"(smem));
104
+}
105
+
106
+// Lookup-table based 3-input logical operation; explicitly used for
107
+// dequantization as the compiler does not seem to automatically recognize it in
108
+// all cases.
109
+template <int lut> __device__ inline int lop3(int a, int b, int c) {
110
+ int res;
111
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
112
+ : "=r"(res)
113
+ : "r"(a), "r"(b), "r"(c), "n"(lut));
114
+ return res;
115
+}
116
+
117
+// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
118
+// values. We mostly follow the strategy in the link below, with some small
119
+// changes:
120
+// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
121
+__device__ inline FragB dequant(int q) {
122
+ const int LO = 0x000f000f;
123
+ const int HI = 0x00f000f0;
124
+ const int EX = 0x64006400;
125
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
126
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
127
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
128
+ // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
129
+ // directly into `SUB` and `ADD`.
130
+ const int SUB = 0x64086408;
131
+ const int MUL = 0x2c002c00;
132
+ const int ADD = 0xd480d480;
133
+ FragB frag_b;
134
+ frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
135
+ *reinterpret_cast<const half2 *>(&SUB));
136
+ frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
137
+ *reinterpret_cast<const half2 *>(&MUL),
138
+ *reinterpret_cast<const half2 *>(&ADD));
139
+ return frag_b;
140
+}
141
+
142
+// Multiply dequantized values by the corresponding quantization scale; used
143
+// only for grouped quantization.
144
+__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
145
+ half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
146
+ frag_b[0] = __hmul2(frag_b[0], s);
147
+ frag_b[1] = __hmul2(frag_b[1], s);
148
+}
149
+
150
+// Same as above, but for act_order (each K is multiplied individually)
151
+__device__ inline void scale4(FragB &frag_b, FragS &frag_s_1, FragS &frag_s_2,
152
+ FragS &frag_s_3, FragS &frag_s_4, int i) {
153
+ __half2 s_val_1_2;
154
+ s_val_1_2.x = reinterpret_cast<__half *>(&frag_s_1)[i];
155
+ s_val_1_2.y = reinterpret_cast<__half *>(&frag_s_2)[i];
156
+
157
+ __half2 s_val_3_4;
158
+ s_val_3_4.x = reinterpret_cast<__half *>(&frag_s_3)[i];
159
+ s_val_3_4.y = reinterpret_cast<__half *>(&frag_s_4)[i];
160
+
161
+ frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
162
+ frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
163
+}
164
+
165
+// Wait until barrier reaches `count`, then lock for current threadblock.
166
+__device__ inline void barrier_acquire(int *lock, int count) {
167
+ if (threadIdx.x == 0) {
168
+ int state = -1;
169
+ do
170
+ // Guarantee that subsequent writes by this threadblock will be visible
171
+ // globally.
172
+ asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
173
+ : "=r"(state)
174
+ : "l"(lock));
175
+ while (state != count);
176
+ }
177
+ __syncthreads();
178
+}
179
+
180
+// Release barrier and increment visitation count.
181
+__device__ inline void barrier_release(int *lock, bool reset = false) {
182
+ __syncthreads();
183
+ if (threadIdx.x == 0) {
184
+ if (reset) {
185
+ lock[0] = 0;
186
+ return;
187
+ }
188
+ int val = 1;
189
+ // Make sure that all writes since acquiring this barrier are visible
190
+ // globally, while releasing the barrier.
191
+ asm volatile("fence.acq_rel.gpu;\n");
192
+ asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
193
+ :
194
+ : "l"(lock), "r"(val));
195
+ }
196
+}
197
+
198
+// For a given "a" of size [M,K] performs a permutation of the K columns based
199
+// on the given "perm" indices.
200
+__global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
201
+ int const *__restrict__ perm_int_ptr,
202
+ int4 *__restrict__ out_int4_ptr, int size_m,
203
+ int size_k, int block_rows) {
204
+
205
+ int start_row = block_rows * blockIdx.x;
206
+ int finish_row = start_row + block_rows;
207
+ if (finish_row > size_m) {
208
+ finish_row = size_m;
209
+ }
210
+ int cur_block_rows = finish_row - start_row;
211
+
212
+ int row_stride = size_k * sizeof(half) / 16;
213
+
214
+ auto permute_row = [&](int row) {
215
+ int iters = size_k / default_threads;
216
+ int rest = size_k % default_threads;
217
+
218
+ int offset = row * row_stride;
219
+
220
+ half const *a_row_half =
221
+ reinterpret_cast<half const *>(a_int4_ptr + offset);
222
+ half *out_half = reinterpret_cast<half *>(out_int4_ptr + offset);
223
+
224
+ int base_k = 0;
225
+
226
+ for (int i = 0; i < iters; i++) {
227
+ int cur_k = base_k + threadIdx.x;
228
+ int src_pos = perm_int_ptr[cur_k];
229
+
230
+ out_half[cur_k] = a_row_half[src_pos];
231
+
232
+ base_k += default_threads;
233
+ }
234
+
235
+ if (rest) {
236
+ if (threadIdx.x < rest) {
237
+ int cur_k = base_k + threadIdx.x;
238
+ int src_pos = perm_int_ptr[cur_k];
239
+
240
+ out_half[cur_k] = a_row_half[src_pos];
241
+ }
242
+ }
243
+ };
244
+
245
+ for (int i = 0; i < cur_block_rows; i++) {
246
+ int cur_row = start_row + i;
247
+ if (cur_row < size_m) {
248
+ permute_row(cur_row);
249
+ }
250
+ }
251
+}
252
+
253
+template <const int threads, // number of threads in a threadblock
254
+ const int thread_m_blocks, // number of 16x16 blocks in the m
255
+ // dimension (batchsize) of the threadblock
256
+ const int thread_n_blocks, // same for n dimension (output)
257
+ const int thread_k_blocks, // same for k dimension (reduction)
258
+ const int stages, // number of stages for the async global->shared
259
+ // fetch pipeline
260
+ const bool has_act_order, // whether act_order is enabled
261
+ const int group_blocks = -1 // number of consecutive 16x16 blocks with
262
+ // a separate quantization scale
263
+ >
264
+__global__ void
265
+Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
266
+ const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
267
+ int4 *__restrict__ C, // fp16 output buffer of shape mxn
268
+ const int4 *__restrict__ scales_ptr, // fp16 quantization scales of shape
269
+ // (k/groupsize)xn
270
+ const int *__restrict__ g_idx, // int32 group indices of shape k
271
+ int num_groups, // number of scale groups per output channel
272
+ int prob_m, // batch dimension m
273
+ int prob_n, // output dimension n
274
+ int prob_k, // reduction dimension k
275
+ int *locks // extra global storage for barrier synchronization
276
+) {
277
+ // Each threadblock processes one "stripe" of the B matrix with (roughly) the
278
+ // same size, which might involve multiple column "slices" (of width 16 *
279
+ // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM
280
+ // example:
281
+ // 0 1 3
282
+ // 0 2 3
283
+ // 1 2 4
284
+ // While this kind of partitioning makes things somewhat more complicated, it
285
+ // ensures good utilization of all SMs for many kinds of shape and GPU
286
+ // configurations, while requiring as few slow global cross-threadblock
287
+ // reductions as possible.
288
+
289
+ // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
290
+ // better partitioning with less reductions
291
+ int parallel = 1;
292
+ if (prob_m > 16 * thread_m_blocks) {
293
+ parallel = prob_m / (16 * thread_m_blocks);
294
+ prob_m = 16 * thread_m_blocks;
295
+ }
296
+
297
+ int k_tiles = prob_k / 16 / thread_k_blocks;
298
+ int n_tiles = prob_n / 16 / thread_n_blocks;
299
+ int iters = div_ceil(k_tiles * n_tiles * parallel, gridDim.x);
300
+
301
+ if constexpr (!has_act_order && group_blocks != -1) {
302
+ if (group_blocks >= thread_k_blocks) {
303
+ // Ensure that the number of tiles in each stripe is a multiple of the
304
+ // groupsize; this avoids an annoying special case where a stripe starts
305
+ // in the middle of group.
306
+ iters = (group_blocks / thread_k_blocks) *
307
+ div_ceil(iters, (group_blocks / thread_k_blocks));
308
+ }
309
+ }
310
+
311
+ int slice_row = (iters * blockIdx.x) % k_tiles;
312
+ int slice_col_par = (iters * blockIdx.x) / k_tiles;
313
+ int slice_col = slice_col_par;
314
+ int slice_iters; // number of threadblock tiles in the current slice
315
+ int slice_count =
316
+ 0; // total number of active threadblocks in the current slice
317
+ int slice_idx; // index of threadblock in current slice; numbered bottom to
318
+ // top
319
+
320
+ // We can easily implement parallel problem execution by just remapping
321
+ // indices and advancing global pointers
322
+ if (slice_col_par >= n_tiles) {
323
+ A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_k / 8;
324
+ C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
325
+ locks += (slice_col_par / n_tiles) * n_tiles;
326
+ slice_col = slice_col_par % n_tiles;
327
+ }
328
+
329
+ // Compute all information about the current slice which is required for
330
+ // synchronization.
331
+ auto init_slice = [&]() {
332
+ slice_iters =
333
+ iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
334
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
335
+ slice_iters = 0;
336
+ if (slice_iters == 0)
337
+ return;
338
+ if (slice_row + slice_iters > k_tiles)
339
+ slice_iters = k_tiles - slice_row;
340
+ slice_count = 1;
341
+ slice_idx = 0;
342
+ int col_first = iters * div_ceil(k_tiles * slice_col_par, iters);
343
+ if (col_first <= k_tiles * (slice_col_par + 1)) {
344
+ int col_off = col_first - k_tiles * slice_col_par;
345
+ slice_count = div_ceil(k_tiles - col_off, iters);
346
+ if (col_off > 0)
347
+ slice_count++;
348
+ int delta_first = iters * blockIdx.x - col_first;
349
+ if (delta_first < 0 || (col_off == 0 && delta_first == 0))
350
+ slice_idx = slice_count - 1;
351
+ else {
352
+ slice_idx = slice_count - 1 - delta_first / iters;
353
+ if (col_off > 0)
354
+ slice_idx--;
355
+ }
356
+ }
357
+ if (slice_col == n_tiles) {
358
+ A += 16 * thread_m_blocks * prob_k / 8;
359
+ C += 16 * thread_m_blocks * prob_n / 8;
360
+ locks += n_tiles;
361
+ slice_col = 0;
362
+ }
363
+ };
364
+ init_slice();
365
+
366
+ // A sizes/strides
367
+
368
+ // stride of the A matrix in global memory
369
+ int a_gl_stride = prob_k / 8;
370
+ // stride of an A matrix tile in shared memory
371
+ constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
372
+ // delta between subsequent A tiles in global memory
373
+ constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
374
+ // between subsequent accesses within a tile
375
+ int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
376
+ // between shared memory writes
377
+ constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
378
+ // between shared memory tile reads
379
+ constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
380
+ // within a shared memory tile
381
+ constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
382
+ // overall size of a tile
383
+ constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
384
+ // number of shared write iterations for a tile
385
+ constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta);
386
+
387
+ // B sizes/strides
388
+ int b_gl_stride = 16 * prob_n / 32;
389
+ constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
390
+ int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
391
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
392
+ constexpr int b_sh_wr_delta = threads;
393
+ constexpr int b_sh_rd_delta = threads;
394
+ constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
395
+ constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
396
+
397
+ // Scale sizes/strides without act_order
398
+ int s_gl_stride = prob_n / 8;
399
+ constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
400
+ constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
401
+ ? thread_k_blocks / group_blocks
402
+ : 1;
403
+ constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
404
+ int s_gl_rd_delta = s_gl_stride;
405
+
406
+ // Scale size/strides with act_order
407
+ constexpr int tb_k = 16 * thread_k_blocks;
408
+ constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
409
+ // constexpr int act_s_row_stride = 1;
410
+ // int act_s_col_stride = act_s_row_stride * num_groups;
411
+ int act_s_col_stride = 1;
412
+ int act_s_col_warp_stride = act_s_col_stride * 8;
413
+ int tb_n_warps = thread_n_blocks / 4;
414
+ int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
415
+
416
+ // Global A read index of current thread.
417
+ int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
418
+ (threadIdx.x % a_gl_rd_delta_o);
419
+ a_gl_rd += a_gl_rd_delta_o * slice_row;
420
+ // Shared write index of current thread.
421
+ int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
422
+ (threadIdx.x % a_gl_rd_delta_o);
423
+ // Shared read index.
424
+ int a_sh_rd =
425
+ a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
426
+ a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
427
+
428
+ int b_gl_rd =
429
+ b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
430
+ b_gl_rd += b_sh_stride * slice_col;
431
+ b_gl_rd += b_gl_rd_delta_o * slice_row;
432
+ int b_sh_wr = threadIdx.x;
433
+ int b_sh_rd = threadIdx.x;
434
+
435
+ // For act_order
436
+ constexpr int k_iter_size = tb_k / b_sh_wr_iters;
437
+ int slice_k_start = tb_k * slice_row;
438
+ int slice_k_finish = slice_k_start + tb_k * slice_iters;
439
+ int slice_k_start_shared_fetch = slice_k_start;
440
+ int slice_n_offset = act_s_col_tb_stride * slice_col;
441
+
442
+ // No act_order
443
+ int s_gl_rd;
444
+ if constexpr (!has_act_order) {
445
+ s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
446
+ s_sh_stride * slice_col + threadIdx.x;
447
+ }
448
+ int s_sh_wr = threadIdx.x;
449
+ bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
450
+
451
+ // We use a different scale layout for grouped and column-wise quantization as
452
+ // we scale a `half2` tile in column-major layout in the former and in
453
+ // row-major in the latter case.
454
+ int s_sh_rd;
455
+ if constexpr (group_blocks != -1)
456
+ s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
457
+ (threadIdx.x % 32) / 4;
458
+ else
459
+ s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
460
+ (threadIdx.x % 32) % 4;
461
+
462
+ // Precompute which thread should not read memory in which iterations; this is
463
+ // needed if there are more threads than required for a certain tilesize or
464
+ // when the batchsize is not a multiple of 16.
465
+ bool a_sh_wr_pred[a_sh_wr_iters];
466
+#pragma unroll
467
+ for (int i = 0; i < a_sh_wr_iters; i++)
468
+ a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
469
+
470
+ // To ensure that writing and reading A tiles to/from shared memory, the
471
+ // latter in fragment format, is fully bank conflict free, we need to use a
472
+ // rather fancy XOR-based layout. The key here is that neither reads nor
473
+ // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
474
+ // same shared memory banks. Further, it seems (based on NSight-Compute) that
475
+ // each warp must also write a consecutive memory segment?
476
+ auto transform_a = [&](int i) {
477
+ int row = i / a_gl_rd_delta_o;
478
+ return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
479
+ };
480
+ // Since the computation of this remapping is non-trivial and, due to our main
481
+ // loop unrolls, all shared memory accesses are static, we simply precompute
482
+ // both transformed reads and writes.
483
+ int a_sh_wr_trans[a_sh_wr_iters];
484
+#pragma unroll
485
+ for (int i = 0; i < a_sh_wr_iters; i++)
486
+ a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
487
+ int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
488
+#pragma unroll
489
+ for (int i = 0; i < b_sh_wr_iters; i++) {
490
+#pragma unroll
491
+ for (int j = 0; j < thread_m_blocks; j++)
492
+ a_sh_rd_trans[i][j] =
493
+ transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
494
+ }
495
+
496
+ // Since B-accesses have non-constant stride they have to be computed at
497
+ // runtime; we break dependencies between subsequent accesses with a tile by
498
+ // maintining multiple pointers (we have enough registers), a tiny
499
+ // optimization.
500
+ const int4 *B_ptr[b_sh_wr_iters];
501
+#pragma unroll
502
+ for (int i = 0; i < b_sh_wr_iters; i++)
503
+ B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
504
+
505
+ extern __shared__ int4 sh[];
506
+ // Shared memory storage for global fetch pipelines.
507
+ int4 *sh_a = sh;
508
+ int4 *sh_b = sh_a + (stages * a_sh_stage);
509
+ int4 *sh_g_idx = sh_b + (stages * b_sh_stage);
510
+ int4 *sh_s = sh_g_idx + (stages * g_idx_stage);
511
+
512
+ // Register storage for double buffer of shared memory reads.
513
+ FragA frag_a[2][thread_m_blocks];
514
+ I4 frag_b_quant[2];
515
+ FragC frag_c[thread_m_blocks][4][2];
516
+ FragS frag_s[2][4]; // No act-order
517
+ FragS act_frag_s[2][4][4]; // For act-order
518
+
519
+ // Zero accumulators.
520
+ auto zero_accums = [&]() {
521
+#pragma unroll
522
+ for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
523
+ reinterpret_cast<float *>(frag_c)[i] = 0;
524
+ };
525
+
526
+ int sh_first_group_id = -1;
527
+ int sh_num_groups = -1;
528
+ constexpr int sh_max_num_groups = 32;
529
+
530
+ auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
531
+ int last_group_id) {
532
+ sh_first_group_id = first_group_id;
533
+ sh_num_groups = last_group_id - first_group_id + 1;
534
+
535
+ if (sh_num_groups < sh_max_num_groups) {
536
+ sh_num_groups = sh_max_num_groups;
537
+ }
538
+
539
+ if (sh_first_group_id + sh_num_groups > num_groups) {
540
+ sh_num_groups = num_groups - sh_first_group_id;
541
+ }
542
+
543
+ int row_offset = first_group_id * s_gl_stride;
544
+
545
+ if (is_async) {
546
+ for (int i = 0; i < sh_num_groups; i++) {
547
+ if (threadIdx.x < s_sh_stride) {
548
+ cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
549
+ &scales_ptr[row_offset + (i * s_gl_stride) +
550
+ slice_n_offset + threadIdx.x]);
551
+ }
552
+ }
553
+ } else {
554
+ for (int i = 0; i < sh_num_groups; i++) {
555
+ if (threadIdx.x < s_sh_stride) {
556
+ sh_s[(i * s_sh_stride) + threadIdx.x] =
557
+ scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
558
+ threadIdx.x];
559
+ }
560
+ }
561
+ }
562
+ };
563
+ // Asynchronously fetch the next A, B and s tile from global to the next
564
+ // shared memory pipeline location.
565
+ auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
566
+ if (pred) {
567
+ int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
568
+#pragma unroll
569
+ for (int i = 0; i < a_sh_wr_iters; i++) {
570
+ cp_async4_pred(
571
+ &sh_a_stage[a_sh_wr_trans[i]],
572
+ &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
573
+ a_sh_wr_pred[i]);
574
+ }
575
+ int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
576
+#pragma unroll
577
+ for (int i = 0; i < b_sh_wr_iters; i++) {
578
+ cp_async4_stream(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
579
+ B_ptr[i] += b_gl_rd_delta_o;
580
+ }
581
+
582
+ if constexpr (has_act_order) {
583
+ // Fetch g_idx thread-block portion
584
+ int full_pipe = a_off;
585
+ int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
586
+ if (cur_k < prob_k && cur_k < slice_k_finish) {
587
+ int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
588
+
589
+ int4 const *cur_g_idx_stage_ptr =
590
+ reinterpret_cast<int4 const *>(&g_idx[cur_k]);
591
+
592
+ if (threadIdx.x < g_idx_stage) {
593
+ cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
594
+ &cur_g_idx_stage_ptr[threadIdx.x]);
595
+ }
596
+ }
597
+ } else {
598
+ if constexpr (group_blocks != -1) {
599
+ int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
600
+
601
+ if constexpr (group_blocks >= thread_k_blocks) {
602
+ // Only fetch scales if this tile starts a new group
603
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
604
+ if (s_sh_wr_pred) {
605
+ cp_async4_stream(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
606
+ }
607
+ s_gl_rd += s_gl_rd_delta;
608
+ }
609
+ } else {
610
+ for (int i = 0; i < s_tb_groups; i++) {
611
+ if (s_sh_wr_pred) {
612
+ cp_async4_stream(&sh_s_stage[i * s_sh_stride + s_sh_wr],
613
+ &scales_ptr[s_gl_rd]);
614
+ }
615
+ s_gl_rd += s_gl_rd_delta;
616
+ }
617
+ }
618
+ }
619
+ }
620
+ }
621
+ // Insert a fence even when we are winding down the pipeline to ensure that
622
+ // waiting is also correct at this point.
623
+ cp_async_fence();
624
+ };
625
+
626
+ // Wait until the next thread tile has been loaded to shared memory.
627
+ auto wait_for_stage = [&]() {
628
+ // We only have `stages - 2` active fetches since we are double buffering
629
+ // and can only issue the next fetch when it is guaranteed that the previous
630
+ // shared memory load is fully complete (as it may otherwise be
631
+ // overwritten).
632
+ cp_async_wait<stages - 2>();
633
+ __syncthreads();
634
+ };
635
+
636
+ // Load the next sub-tile from the current location in the shared memory pipe
637
+ // into the current register buffer.
638
+ auto fetch_to_registers = [&](int k, int pipe) {
639
+ int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
640
+#pragma unroll
641
+ for (int i = 0; i < thread_m_blocks; i++)
642
+ ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
643
+ int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
644
+ frag_b_quant[k % 2] = *reinterpret_cast<I4 *>(
645
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
646
+ };
647
+
648
+ bool is_same_group[stages];
649
+ int same_group_id[stages];
650
+
651
+ auto init_same_group = [&](int pipe) {
652
+ int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
653
+ int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
654
+
655
+ int group_id_1 = sh_g_idx_int_ptr[0];
656
+ int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
657
+
658
+ is_same_group[pipe] = group_id_1 == group_id_2;
659
+ same_group_id[pipe] = group_id_1;
660
+ };
661
+
662
+ auto fetch_scales_to_registers = [&](int k, int full_pipe) {
663
+ int pipe = full_pipe % stages;
664
+
665
+ if constexpr (!has_act_order) {
666
+ // No act-order case
667
+ if constexpr (group_blocks != -1) {
668
+ if constexpr (group_blocks >= thread_k_blocks) {
669
+ int4 *sh_s_stage =
670
+ sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
671
+ (pipe / (group_blocks / thread_k_blocks)));
672
+ reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
673
+ } else {
674
+ int warp_id = threadIdx.x / 32;
675
+ int n_warps = thread_n_blocks / 4;
676
+
677
+ int warp_row = warp_id / n_warps;
678
+
679
+ int cur_k = warp_row * 16;
680
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
681
+
682
+ int k_blocks = cur_k / 16;
683
+ int cur_group_id = k_blocks / group_blocks;
684
+
685
+ int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
686
+
687
+ reinterpret_cast<int4 *>(&frag_s[k % 2])[0] =
688
+ sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
689
+ }
690
+ }
691
+
692
+ return;
693
+ }
694
+
695
+ // Act-order case
696
+
697
+ // Determine K of the "current" thread-block
698
+ int cur_k = slice_k_start + tb_k * full_pipe;
699
+ if (cur_k >= prob_k || cur_k >= slice_k_finish) {
700
+ return;
701
+ }
702
+
703
+ // Reset (to current thread-block) since we read g_idx portion from the
704
+ // shared memory
705
+ cur_k = 0;
706
+
707
+ // Progress to current iteration
708
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
709
+
710
+ // Determine "position" inside the thread-block (based on warp and
711
+ // thread-id)
712
+ int warp_id = threadIdx.x / 32;
713
+ int n_warps =
714
+ thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
715
+
716
+ int warp_row = warp_id / n_warps;
717
+ int warp_col = warp_id % n_warps;
718
+
719
+ cur_k += warp_row * 16;
720
+
721
+ int th_id = threadIdx.x % 32;
722
+ cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
723
+
724
+ int s_col_shift =
725
+ /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
726
+ (th_id / 4) * act_s_col_stride;
727
+
728
+ if (is_same_group[pipe]) {
729
+ if (k % 2 == 0) {
730
+ *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0]))) =
731
+ sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
732
+ s_col_shift];
733
+ } else {
734
+ *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0]))) =
735
+ *(reinterpret_cast<int4 *>(&(act_frag_s[(k - 1) % 2][0][0])));
736
+ }
737
+
738
+ for (int i = 1; i < 4; i++) {
739
+ *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][i][0]))) =
740
+ *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0])));
741
+ }
742
+ return;
743
+ }
744
+
745
+ int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
746
+ int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
747
+
748
+ constexpr int k_frag_offsets[4] = {0, 1, 8,
749
+ 9}; // Tensor core offsets per thread
750
+
751
+#pragma unroll
752
+ for (int i = 0; i < 4; i++) {
753
+
754
+ int actual_k = cur_k + k_frag_offsets[i];
755
+
756
+ int group_id = sh_g_idx_int_ptr[actual_k];
757
+ int rel_group_id = group_id - sh_first_group_id;
758
+
759
+ *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][i][0]))) =
760
+ sh_s[rel_group_id * s_sh_stride + s_col_shift];
761
+ }
762
+ };
763
+
764
+ // Execute the actual tensor core matmul of a sub-tile.
765
+ auto matmul = [&](int k) {
766
+// We have the m dimension as the inner loop in order to encourage overlapping
767
+// dequantization and matmul operations.
768
+#pragma unroll
769
+ for (int j = 0; j < 4; j++) {
770
+ int b_quant = frag_b_quant[k % 2][j];
771
+ int b_quant_shift = b_quant >> 8;
772
+
773
+ FragB frag_b0 = dequant(b_quant);
774
+
775
+ // Apply scale to frag_b0
776
+ if constexpr (has_act_order) {
777
+ scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
778
+ act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
779
+ } else {
780
+ if constexpr (group_blocks != -1) {
781
+ scale(frag_b0, frag_s[k % 2][j], 0);
782
+ }
783
+ }
784
+
785
+ FragB frag_b1 = dequant(b_quant_shift);
786
+
787
+ // Apply scale to frag_b1
788
+ if constexpr (has_act_order) {
789
+ scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
790
+ act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
791
+
792
+ } else {
793
+ if constexpr (group_blocks != -1) {
794
+ scale(frag_b1, frag_s[k % 2][j], 1);
795
+ }
796
+ }
797
+
798
+#pragma unroll
799
+ for (int i = 0; i < thread_m_blocks; i++) {
800
+ mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
801
+ mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
802
+ }
803
+ }
804
+ };
805
+
806
+ // Since we slice across the k dimension of a tile in order to increase the
807
+ // number of warps while keeping the n dimension of a tile reasonable, we have
808
+ // multiple warps that accumulate their partial sums of the same output
809
+ // location; which we have to reduce over in the end. We do in shared memory.
810
+ auto thread_block_reduce = [&]() {
811
+ constexpr int red_off = threads / b_sh_stride / 2;
812
+ if (red_off >= 1) {
813
+ int red_idx = threadIdx.x / b_sh_stride;
814
+ constexpr int red_sh_stride = b_sh_stride * 4 * 2;
815
+ constexpr int red_sh_delta = b_sh_stride;
816
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
817
+ (threadIdx.x % b_sh_stride);
818
+
819
+ // Parallel logarithmic shared memory reduction. We make sure to avoid any
820
+ // unnecessary read or write iterations, e.g., for two warps we write only
821
+ // once by warp 1 and read only once by warp 0.
822
+
823
+#pragma unroll
824
+ for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
825
+#pragma unroll
826
+ for (int i = red_off; i > 0; i /= 2) {
827
+ if (i <= red_idx && red_idx < 2 * i) {
828
+#pragma unroll
829
+ for (int j = 0; j < 4 * 2; j++) {
830
+ int red_sh_wr =
831
+ red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
832
+ if (i < red_off) {
833
+ float *c_rd = reinterpret_cast<float *>(
834
+ &sh[red_sh_delta * j + red_sh_rd]);
835
+ float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
836
+#pragma unroll
837
+ for (int k = 0; k < 4; k++)
838
+ reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
839
+ c_rd[k] + c_wr[k];
840
+ }
841
+ sh[red_sh_wr] =
842
+ reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
843
+ }
844
+ }
845
+ __syncthreads();
846
+ }
847
+ if (red_idx == 0) {
848
+#pragma unroll
849
+ for (int i = 0; i < 4 * 2; i++) {
850
+ float *c_rd =
851
+ reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
852
+#pragma unroll
853
+ for (int j = 0; j < 4; j++)
854
+ reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
855
+ c_rd[j];
856
+ }
857
+ }
858
+ __syncthreads();
859
+ }
860
+ }
861
+ };
862
+
863
+ // Since multiple threadblocks may process parts of the same column slice, we
864
+ // finally have to globally reduce over the results. As the striped portioning
865
+ // minimizes the number of such reductions and our outputs are usually rather
866
+ // small, we perform this reduction serially in L2 cache.
867
+ auto global_reduce = [&](bool first = false, bool last = false) {
868
+ // We are very careful here to reduce directly in the output buffer to
869
+ // maximize L2 cache utilization in this step. To do this, we write out
870
+ // results in FP16 (but still reduce with FP32 compute).
871
+ constexpr int active_threads = 32 * thread_n_blocks / 4;
872
+ if (threadIdx.x < active_threads) {
873
+ int c_gl_stride = prob_n / 8;
874
+ int c_gl_wr_delta_o = 8 * c_gl_stride;
875
+ int c_gl_wr_delta_i = 4 * (active_threads / 32);
876
+ int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
877
+ 4 * (threadIdx.x / 32) + threadIdx.x % 4;
878
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
879
+ constexpr int c_sh_wr_delta = active_threads;
880
+ int c_sh_wr = threadIdx.x;
881
+
882
+ int row = (threadIdx.x % 32) / 4;
883
+
884
+ if (!first) {
885
+// Interestingly, doing direct global accesses here really seems to mess up the
886
+// compiler and lead to slowdowns, hence we also use async-copies even though
887
+// these fetches are not actually asynchronous.
888
+#pragma unroll
889
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
890
+ cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
891
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
892
+ c_gl_wr_delta_i * (i % 2)],
893
+ i < (thread_m_blocks - 1) * 4 ||
894
+ 8 * (i / 2) + row < prob_m);
895
+ }
896
+ cp_async_fence();
897
+ cp_async_wait<0>();
898
+ }
899
+
900
+#pragma unroll
901
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
902
+ if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
903
+ if (!first) {
904
+ int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
905
+#pragma unroll
906
+ for (int j = 0; j < 2 * 4; j++) {
907
+ reinterpret_cast<float *>(
908
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
909
+ __half2float(reinterpret_cast<__half *>(&c_red)[j]);
910
+ }
911
+ }
912
+ if (!last) {
913
+ int4 c;
914
+#pragma unroll
915
+ for (int j = 0; j < 2 * 4; j++) {
916
+ reinterpret_cast<__half *>(&c)[j] =
917
+ __float2half(reinterpret_cast<float *>(
918
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
919
+ }
920
+ C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
921
+ c;
922
+ }
923
+ }
924
+ }
925
+ }
926
+ };
927
+
928
+ // Write out the reduce final result in the correct layout. We only actually
929
+ // reshuffle matrix fragments in this step, the reduction above is performed
930
+ // in fragment layout.
931
+ auto write_result = [&]() {
932
+ int c_gl_stride = prob_n / 8;
933
+ constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
934
+ int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
935
+ constexpr int c_sh_rd_delta =
936
+ c_sh_stride * (threads / (2 * thread_n_blocks));
937
+
938
+ int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
939
+ (threadIdx.x % (2 * thread_n_blocks));
940
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
941
+ int c_sh_wr =
942
+ (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
943
+ c_sh_wr += 32 * (threadIdx.x / 32);
944
+ int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
945
+ (threadIdx.x % (2 * thread_n_blocks));
946
+
947
+ int c_gl_wr_end = c_gl_stride * prob_m;
948
+
949
+ // We first reorder in shared memory to guarantee the most efficient final
950
+ // global write patterns
951
+ auto write = [&](int idx, float c0, float c1, FragS &s) {
952
+ half2 res = __halves2half2(__float2half(c0), __float2half(c1));
953
+
954
+ // For per-column quantization we finally apply the scale here
955
+ if constexpr (!has_act_order && group_blocks == -1) {
956
+ res = __hmul2(res, s[0]);
957
+ }
958
+
959
+ ((half2 *)sh)[idx] = res;
960
+ };
961
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
962
+#pragma unroll
963
+ for (int i = 0; i < thread_m_blocks; i++) {
964
+#pragma unroll
965
+ for (int j = 0; j < 4; j++) {
966
+ int wr = c_sh_wr + 8 * j;
967
+ write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
968
+ frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
969
+ write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
970
+ frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
971
+ write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
972
+ frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
973
+ write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
974
+ frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
975
+ }
976
+ c_sh_wr += 16 * (4 * c_sh_stride);
977
+ }
978
+ }
979
+ __syncthreads();
980
+
981
+#pragma unroll
982
+ for (int i = 0;
983
+ i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
984
+ i++) {
985
+ if (c_gl_wr < c_gl_wr_end) {
986
+ C[c_gl_wr] = sh[c_sh_rd];
987
+ c_gl_wr += c_gl_wr_delta;
988
+ c_sh_rd += c_sh_rd_delta;
989
+ }
990
+ }
991
+ };
992
+
993
+ // Start global fetch and register load pipelines.
994
+ auto start_pipes = [&]() {
995
+
996
+#pragma unroll
997
+ for (int i = 0; i < stages - 1; i++) {
998
+ if (has_act_order && i == 0) {
999
+ int last_g_idx = slice_k_start + stages * tb_k * 2;
1000
+ if (last_g_idx >= prob_k) {
1001
+ last_g_idx = prob_k - 1;
1002
+ }
1003
+ fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1004
+ }
1005
+ fetch_to_shared(i, i, i < slice_iters);
1006
+ }
1007
+
1008
+ zero_accums();
1009
+ wait_for_stage();
1010
+ init_same_group(0);
1011
+ fetch_to_registers(0, 0);
1012
+ fetch_scales_to_registers(0, 0);
1013
+ a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1014
+ slice_k_start_shared_fetch += tb_k * (stages - 1);
1015
+ };
1016
+ if (slice_iters) {
1017
+ start_pipes();
1018
+ }
1019
+
1020
+ // Main loop.
1021
+ while (slice_iters) {
1022
+ // We unroll over both the global fetch and the register load pipeline to
1023
+ // ensure all shared memory accesses are static. Note that both pipelines
1024
+ // have even length meaning that the next iteration will always start at
1025
+ // index 0.
1026
+#pragma unroll
1027
+ for (int pipe = 0; pipe < stages;) {
1028
+#pragma unroll
1029
+ for (int k = 0; k < b_sh_wr_iters; k++) {
1030
+ fetch_to_registers(k + 1, pipe % stages);
1031
+ fetch_scales_to_registers(k + 1, pipe);
1032
+ if (k == b_sh_wr_iters - 2) {
1033
+ fetch_to_shared((pipe + stages - 1) % stages, pipe,
1034
+ slice_iters >= stages);
1035
+ pipe++;
1036
+ wait_for_stage();
1037
+ init_same_group(pipe % stages);
1038
+ }
1039
+ matmul(k);
1040
+ }
1041
+ slice_iters--;
1042
+ if (slice_iters == 0) {
1043
+ break;
1044
+ }
1045
+ }
1046
+
1047
+ a_gl_rd += a_gl_rd_delta_o * stages;
1048
+ slice_k_start += tb_k * stages;
1049
+ slice_k_start_shared_fetch += tb_k * stages;
1050
+
1051
+ if constexpr (has_act_order) {
1052
+ int first_group_id = g_idx[slice_k_start];
1053
+ int last_g_idx = slice_k_start + stages * tb_k * 2;
1054
+ if (last_g_idx >= prob_k) {
1055
+ last_g_idx = prob_k - 1;
1056
+ }
1057
+ int last_group_id = g_idx[last_g_idx];
1058
+ if (last_group_id >= sh_first_group_id + sh_num_groups) {
1059
+ fetch_scales_to_shared(false, first_group_id, last_group_id);
1060
+ __syncthreads();
1061
+ }
1062
+ }
1063
+
1064
+ // Process results and, if necessary, proceed to the next column slice.
1065
+ // While this pattern may not be the most readable, other ways of writing
1066
+ // the loop seemed to noticeably worse performance after compilation.
1067
+ if (slice_iters == 0) {
1068
+ cp_async_wait<0>();
1069
+ bool last = slice_idx == slice_count - 1;
1070
+ // For per-column scales, we only fetch them here in the final step before
1071
+ // write-out
1072
+ if constexpr (!has_act_order && group_blocks == -1) {
1073
+ if (last) {
1074
+ if (s_sh_wr_pred) {
1075
+ cp_async4_stream(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1076
+ }
1077
+ cp_async_fence();
1078
+ }
1079
+ }
1080
+
1081
+ thread_block_reduce();
1082
+ if constexpr (!has_act_order && group_blocks == -1) {
1083
+ if (last) {
1084
+ cp_async_wait<0>();
1085
+ __syncthreads();
1086
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1087
+ reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1088
+ reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1089
+ }
1090
+ }
1091
+ }
1092
+
1093
+ if (slice_count > 1) { // only globally reduce if there is more than one
1094
+ // block in a slice
1095
+ barrier_acquire(&locks[slice_col], slice_idx);
1096
+ global_reduce(slice_idx == 0, last);
1097
+ barrier_release(&locks[slice_col], last);
1098
+ }
1099
+ if (last) // only the last block in a slice actually writes the result
1100
+ write_result();
1101
+ slice_row = 0;
1102
+ slice_col_par++;
1103
+ slice_col++;
1104
+ init_slice();
1105
+ if (slice_iters) {
1106
+ a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1107
+ (threadIdx.x % a_gl_rd_delta_o);
1108
+#pragma unroll
1109
+ for (int i = 0; i < b_sh_wr_iters; i++)
1110
+ B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1111
+ if (slice_col == 0) {
1112
+#pragma unroll
1113
+ for (int i = 0; i < b_sh_wr_iters; i++)
1114
+ B_ptr[i] -= b_gl_stride;
1115
+ }
1116
+
1117
+ // Update slice k/n for scales loading
1118
+ if constexpr (has_act_order) {
1119
+ slice_k_start = tb_k * slice_row;
1120
+ slice_k_finish = slice_k_start + tb_k * slice_iters;
1121
+ slice_k_start_shared_fetch = slice_k_start;
1122
+ slice_n_offset = act_s_col_tb_stride * slice_col;
1123
+
1124
+ } else {
1125
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1126
+ }
1127
+
1128
+ // if (blockIdx.x == 0 && threadIdx.x == 0) {
1129
+ // printf("Move\n");
1130
+ // }
1131
+ start_pipes();
1132
+ }
1133
+ }
1134
+ }
1135
+}
1136
+
1137
+#define __CALL_IF(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1138
+ HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1139
+ else if (thread_m_blocks == THREAD_M_BLOCKS && \
1140
+ thread_n_blocks == THREAD_N_BLOCKS && \
1141
+ thread_k_blocks == THREAD_K_BLOCKS && \
1142
+ has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1143
+ num_threads == NUM_THREADS) { \
1144
+ cudaFuncSetAttribute( \
1145
+ Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1146
+ pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1147
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1148
+ Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1149
+ pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1150
+ <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1151
+ A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1152
+ prob_k, locks); \
1153
+ }
1154
+
1155
+typedef struct {
1156
+ int thread_k;
1157
+ int thread_n;
1158
+ int num_threads;
1159
+} thread_config_t;
1160
+
1161
+thread_config_t small_batch_thread_configs[] = {
1162
+ // Ordered by priority
1163
+
1164
+ // thread_k, thread_n, num_threads
1165
+ {128, 128, 256}, // Default
1166
+ {128, 64, 128}, // Reduce N 2X, same K
1167
+ {64, 256, 256}, // Reduce K 2X, increase N 2X
1168
+ {64, 128, 128}, // Reduce K 2X, same N
1169
+};
1170
+
1171
+thread_config_t large_batch_thread_configs[] = {
1172
+ // Ordered by priority
1173
+
1174
+ // thread_k, thread_n, num_threads
1175
+ {64, 256, 256}, // Default
1176
+ {128, 64, 128}, // Reduce N 2X, same K
1177
+ {64, 128, 128}, // Reduce N 2X, same K
1178
+ // {128, 64, 128}, // Reduce N 4X, increase K 2X
1179
+};
1180
+
1181
+bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
1182
+ int prob_k) {
1183
+ // Sanity
1184
+ if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1185
+ th_config.num_threads == -1) {
1186
+ return false;
1187
+ }
1188
+
1189
+ // Verify K/N are divisible by thread K/N
1190
+ if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) {
1191
+ return false;
1192
+ }
1193
+
1194
+ // Verify min for thread K/N
1195
+ if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) {
1196
+ return false;
1197
+ }
1198
+
1199
+ // num_threads must be at least 128 (= 4 warps)
1200
+ if (th_config.num_threads < 128) {
1201
+ return false;
1202
+ }
1203
+
1204
+ return true;
1205
+}
1206
+
1207
+thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1208
+
1209
+ // TODO: Enable if needed after some more testing
1210
+ if (prob_m <= 0) {
1211
+ for (auto th_config : small_batch_thread_configs) {
1212
+ if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1213
+ return th_config;
1214
+ }
1215
+ }
1216
+
1217
+ } else {
1218
+ for (auto th_config : large_batch_thread_configs) {
1219
+ if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1220
+ return th_config;
1221
+ }
1222
+ }
1223
+ }
1224
+
1225
+ return thread_config_t{-1, -1, -1};
1226
+}
1227
+
1228
+#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1229
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1230
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1231
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1232
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1233
+ \
1234
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1235
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1236
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1237
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1238
+ \
1239
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1240
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1241
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1242
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1243
+ \
1244
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1245
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1246
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1247
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1248
+ \
1249
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1250
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1251
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1252
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1253
+
1254
+void marlin_cuda(const void *A, const void *B, void *C, void *s, void *g_idx,
1255
+ void *perm, void *a_tmp, int prob_m, int prob_n, int prob_k,
1256
+ void *workspace, bool has_act_order, bool is_k_full,
1257
+ int num_groups, int group_size, int dev = 0,
1258
+ cudaStream_t stream = 0, int thread_k = -1, int thread_n = -1,
1259
+ int sms = -1, int max_par = 16) {
1260
+ TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1261
+ ", ", prob_n, ", ", prob_k, "]");
1262
+
1263
+ int tot_m = prob_m;
1264
+ int tot_m_blocks = div_ceil(tot_m, 16);
1265
+ int pad = 16 * tot_m_blocks - tot_m;
1266
+
1267
+ if (sms == -1) {
1268
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1269
+ }
1270
+
1271
+ int max_shared_mem = 0;
1272
+ cudaDeviceGetAttribute(&max_shared_mem,
1273
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1274
+ TORCH_CHECK(max_shared_mem > 0);
1275
+
1276
+ // Set thread config
1277
+ thread_config_t th_config;
1278
+ if (thread_k != -1 && thread_n != -1) {
1279
+ // User-defined config
1280
+ th_config = thread_config_t{thread_k, thread_n, default_threads};
1281
+ } else {
1282
+ // Auto config
1283
+ th_config = determine_thread_config(prob_m, prob_n, prob_k);
1284
+ }
1285
+
1286
+ TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1287
+ "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1288
+ ", thread_n = " + str(th_config.thread_n) +
1289
+ ", num_threads = " + str(th_config.num_threads) +
1290
+ " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1291
+ str(prob_n) + "]");
1292
+
1293
+ int num_threads = th_config.num_threads;
1294
+ thread_k = th_config.thread_k;
1295
+ thread_n = th_config.thread_n;
1296
+
1297
+ int thread_k_blocks = thread_k / 16;
1298
+ int thread_n_blocks = thread_n / 16;
1299
+
1300
+ int blocks = sms;
1301
+
1302
+ TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
1303
+ " is not divisible by thread_n = ", thread_n);
1304
+ TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
1305
+ " is not divisible by thread_k = ", thread_k);
1306
+
1307
+ int group_blocks = 0;
1308
+ if (has_act_order) {
1309
+ if (is_k_full) {
1310
+ TORCH_CHECK(group_size != -1);
1311
+ group_blocks = group_size / 16;
1312
+ TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1313
+ " is not divisible by group_blocks = ", group_blocks);
1314
+ } else {
1315
+ TORCH_CHECK(group_size == 0);
1316
+ group_blocks = 0;
1317
+ }
1318
+
1319
+ } else {
1320
+ if (group_size == -1) {
1321
+ group_blocks = -1;
1322
+ } else {
1323
+ group_blocks = group_size / 16;
1324
+ TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1325
+ " is not divisible by group_blocks = ", group_blocks);
1326
+ }
1327
+ }
1328
+
1329
+ const int4 *A_ptr = (const int4 *)A;
1330
+ const int4 *B_ptr = (const int4 *)B;
1331
+ int4 *C_ptr = (int4 *)C;
1332
+ const int4 *s_ptr = (const int4 *)s;
1333
+ const int *g_idx_ptr = (const int *)g_idx;
1334
+ const int *perm_ptr = (const int *)perm;
1335
+ int4 *a_tmp_ptr = (int4 *)a_tmp;
1336
+
1337
+ int *locks = (int *)workspace;
1338
+
1339
+ if (has_act_order) {
1340
+ // Permute A columns
1341
+ int block_rows = div_ceil(prob_m, blocks);
1342
+ permute_cols_kernel<<<blocks, default_threads, 0, stream>>>(
1343
+ A_ptr, perm_ptr, a_tmp_ptr, prob_m, prob_k, block_rows);
1344
+ A_ptr = a_tmp_ptr;
1345
+ }
1346
+
1347
+ // If we have a full K, then we can run the non-act-order version of Marlin
1348
+ // (since the weight rows are reordered by increasing group ids, and by having
1349
+ // a full K, we have full original groups)
1350
+ if (is_k_full) {
1351
+ has_act_order = false;
1352
+ }
1353
+
1354
+ // Main loop
1355
+ for (int i = 0; i < tot_m_blocks; i += 4) {
1356
+ int thread_m_blocks = tot_m_blocks - i;
1357
+ prob_m = tot_m - 16 * i;
1358
+ int par = 1;
1359
+ if (thread_m_blocks > 4) {
1360
+ // Note that parallel > 1 currently only works for inputs without any
1361
+ // padding
1362
+ par = (16 * thread_m_blocks - pad) / 64;
1363
+ if (par > max_par)
1364
+ par = max_par;
1365
+ prob_m = 64 * par;
1366
+ i += 4 * (par - 1);
1367
+ thread_m_blocks = 4;
1368
+ }
1369
+
1370
+ // Define kernel configurations
1371
+ if (false) {
1372
+ }
1373
+ CALL_IF(16, 4, 256)
1374
+ CALL_IF(8, 8, 256)
1375
+ CALL_IF(8, 4, 128)
1376
+ CALL_IF(4, 8, 128)
1377
+ else {
1378
+ TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1379
+ str(prob_n) + ", " + str(prob_k) + "]" +
1380
+ ", has_act_order = " + str(has_act_order) +
1381
+ ", num_groups = " + str(num_groups) +
1382
+ ", group_size = " + str(group_size) +
1383
+ ", thread_m_blocks = " + str(thread_m_blocks) +
1384
+ ", thread_n_blocks = " + str(thread_n_blocks) +
1385
+ ", thread_k_blocks = " + str(thread_k_blocks));
1386
+ }
1387
+
1388
+ A_ptr += 16 * thread_m_blocks * (prob_k / 8) * par;
1389
+ C_ptr += 16 * thread_m_blocks * (prob_n / 8) * par;
1390
+ }
1391
+}
1392
+
1393
+} // namespace gptq_marlin
1394
+
1395
+torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1396
+ torch::Tensor &b_scales, torch::Tensor &g_idx,
1397
+ torch::Tensor &perm, torch::Tensor &workspace,
1398
+ int64_t size_m, int64_t size_n, int64_t size_k,
1399
+ bool is_k_full) {
1400
+ // Verify A
1401
+ TORCH_CHECK(a.size(0) == size_m,
1402
+ "Shape mismatch: a.size(0) = " + str(a.size(0)) +
1403
+ ", size_m = " + str(size_m));
1404
+ TORCH_CHECK(a.size(1) == size_k,
1405
+ "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1406
+ ", size_k = " + str(size_k));
1407
+
1408
+ // Verify B
1409
+ TORCH_CHECK(size_k % gptq_marlin::tile_size == 0,
1410
+ "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1411
+ str(gptq_marlin::tile_size));
1412
+ TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
1413
+ "Shape mismatch: b_q_weight.size(0) = " +
1414
+ str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1415
+ ", tile_size = " + str(gptq_marlin::tile_size));
1416
+ TORCH_CHECK(
1417
+ b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1418
+ "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1419
+ " is not divisible by tile_size = " + str(gptq_marlin::tile_size));
1420
+ int actual_size_n = (b_q_weight.size(1) / gptq_marlin::tile_size) *
1421
+ gptq_marlin::pack_factor_4bit;
1422
+ TORCH_CHECK(size_n == actual_size_n,
1423
+ "size_n = " + str(size_n) +
1424
+ ", actual_size_n = " + str(actual_size_n));
1425
+
1426
+ // Verify device and strides
1427
+ TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1428
+ TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1429
+
1430
+ TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
1431
+ TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
1432
+
1433
+ TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
1434
+ TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
1435
+
1436
+ TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
1437
+ TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
1438
+
1439
+ TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
1440
+ TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
1441
+
1442
+ // Alloc buffers
1443
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1444
+ auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1445
+ torch::Tensor c = torch::empty({size_m, size_n}, options);
1446
+ torch::Tensor a_tmp = torch::empty({size_m, size_k}, options);
1447
+
1448
+ // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
1449
+ // auto -1)
1450
+ int thread_k = -1;
1451
+ // thread_n: `n` size of a thread_tile in `weights` (can usually be left as
1452
+ // auto -1)
1453
+ int thread_n = -1;
1454
+ // sms: number of SMs to use for the kernel (can usually be left as auto -1)
1455
+ int sms = -1;
1456
+
1457
+ // Verify g_idx and perm
1458
+ TORCH_CHECK((g_idx.size(0) == 0 && perm.size(0) == 0) ||
1459
+ (g_idx.size(0) == size_k && perm.size(0) == size_k),
1460
+ "Unexpected g_idx.size(0) = " + str(g_idx.size(0)) +
1461
+ " and perm.size(0) = " + str(perm.size(0)) +
1462
+ ", where size_k = " + str(size_k));
1463
+
1464
+ // Detect groupsize and act_order
1465
+ int num_groups = -1;
1466
+ int group_size = -1;
1467
+ bool has_act_order = g_idx.size(0) != 0;
1468
+
1469
+ int b_rank = b_scales.sizes().size();
1470
+ TORCH_CHECK(b_rank == 2, "b_scales rank = ", b_rank, " is not 2");
1471
+ TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1),
1472
+ " is not size_n = ", size_n);
1473
+ num_groups = b_scales.size(0);
1474
+
1475
+ if (has_act_order) {
1476
+ if (is_k_full) {
1477
+ TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
1478
+ TORCH_CHECK(size_k % num_groups == 0,
1479
+ "size_k = " + str(size_k) +
1480
+ ", is not divisible by num_groups = " + str(num_groups));
1481
+ group_size = size_k / num_groups;
1482
+ } else {
1483
+ group_size = 0;
1484
+ }
1485
+
1486
+ } else {
1487
+ if (num_groups > 1) {
1488
+ TORCH_CHECK(size_k % num_groups == 0,
1489
+ "size_k = " + str(size_k) +
1490
+ ", is not divisible by b_scales.size(0) = " +
1491
+ str(b_scales.size(0)));
1492
+ group_size = size_k / num_groups;
1493
+ } else {
1494
+ group_size = -1;
1495
+ }
1496
+ }
1497
+
1498
+ // Verify workspace size
1499
+ TORCH_CHECK(size_n % gptq_marlin::min_thread_n == 0,
1500
+ "size_n = " + str(size_n) +
1501
+ ", is not divisible by min_thread_n = " +
1502
+ str(gptq_marlin::min_thread_n));
1503
+ int min_workspace_size =
1504
+ (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
1505
+ TORCH_CHECK(workspace.numel() >= min_workspace_size,
1506
+ "workspace.numel = " + str(workspace.numel()) +
1507
+ " is below min_workspace_size = " + str(min_workspace_size));
1508
+
1509
+ int dev = a.get_device();
1510
+ gptq_marlin::marlin_cuda(
1511
+ a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), b_scales.data_ptr(),
1512
+ g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(), size_m, size_n,
1513
+ size_k, workspace.data_ptr(), has_act_order, is_k_full, num_groups,
1514
+ group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n,
1515
+ sms, gptq_marlin::max_par);
1516
+
1517
+ return c;
1518
+}
1519
+
1520
+#endif
csrc/quantization/gptq_marlin/gptq_marlin.cuh
new
+74
@@ -0,0 +1,74 @@
1
+#pragma once
2
+
3
+#include <torch/extension.h>
4
+
5
+#include <ATen/cuda/CUDAContext.h>
6
+#include <c10/cuda/CUDAGuard.h>
7
+#include <cuda.h>
8
+#include <cuda_fp16.h>
9
+#include <cuda_runtime.h>
10
+#include <iostream>
11
+
12
+namespace gptq_marlin {
13
+
14
+// 8 warps are a good choice since every SM has 4 schedulers and having more than 1 warp per
15
+// schedule allows some more latency hiding. At the same time, we want relatively few warps to have
16
+// many registers per warp and small tiles.
17
+static constexpr int default_threads = 256;
18
+
19
+static constexpr int pipe_stages = 4; // 4 pipeline stages fit into shared memory
20
+
21
+static constexpr int min_thread_n = 64;
22
+static constexpr int min_thread_k = 64;
23
+
24
+static constexpr int tile_size = 16;
25
+static constexpr int max_par = 16;
26
+
27
+static constexpr int pack_factor_4bit = 8; // We have 8 4-bit vals inside a 32 bit
28
+
29
+template <typename T, int n>
30
+struct Vec {
31
+ T elems[n];
32
+ __device__ T& operator[](int i) { return elems[i]; }
33
+};
34
+
35
+using I4 = Vec<int, 4>;
36
+
37
+constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
38
+
39
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
40
+ // No support for async
41
+#else
42
+
43
+__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool pred = true) {
44
+ const int BYTES = 16;
45
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
46
+ asm volatile("{\n"
47
+ " .reg .pred p;\n"
48
+ " setp.ne.b32 p, %0, 0;\n"
49
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
50
+ "}\n" ::"r"((int)pred),
51
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
52
+}
53
+
54
+__device__ inline void cp_async4_stream(void* smem_ptr, const void* glob_ptr) {
55
+ const int BYTES = 16;
56
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
57
+ asm volatile("{\n"
58
+ " .reg .b64 p;\n"
59
+ " createpolicy.fractional.L2::evict_first.b64 p, 1.0;"
60
+ " cp.async.cg.shared.global.L2::cache_hint [%0], [%1], %2, p;\n"
61
+ "}\n" ::"r"(smem),
62
+ "l"(glob_ptr), "n"(BYTES));
63
+}
64
+
65
+__device__ inline void cp_async_fence() { asm volatile("cp.async.commit_group;\n" ::); }
66
+
67
+template <int n>
68
+__device__ inline void cp_async_wait() {
69
+ asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
70
+}
71
+
72
+#endif
73
+
74
+} // namespace gptq_marlin
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
new
+324
@@ -0,0 +1,324 @@
1
+#include "gptq_marlin.cuh"
2
+
3
+namespace gptq_marlin {
4
+
5
+static constexpr int repack_stages = 8;
6
+
7
+static constexpr int repack_threads = 256;
8
+
9
+static constexpr int tile_k_size = tile_size;
10
+static constexpr int tile_n_size = tile_k_size * 4;
11
+
12
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
13
+
14
+template <int const num_threads, bool const has_perm>
15
+__global__ void
16
+marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
17
+ uint32_t const *__restrict__ perm_ptr,
18
+ uint32_t *__restrict__ out_ptr, int size_k, int size_n) {}
19
+
20
+} // namespace gptq_marlin
21
+
22
+torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
23
+ int64_t size_k, int64_t size_n) {
24
+ TORCH_CHECK_NOT_IMPLEMENTED(
25
+ false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
26
+ return torch::empty({1, 1});
27
+}
28
+
29
+#else
30
+
31
+template <int const num_threads, bool const has_perm>
32
+__global__ void
33
+marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
34
+ uint32_t const *__restrict__ perm_ptr,
35
+ uint32_t *__restrict__ out_ptr, int size_k, int size_n) {
36
+ int k_tiles = size_k / tile_k_size;
37
+ int n_tiles = size_n / tile_n_size;
38
+ int block_k_tiles = div_ceil(k_tiles, gridDim.x);
39
+
40
+ int start_k_tile = blockIdx.x * block_k_tiles;
41
+ if (start_k_tile >= k_tiles) {
42
+ return;
43
+ }
44
+
45
+ int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles);
46
+
47
+ // Wait until the next thread tile has been loaded to shared memory.
48
+ auto wait_for_stage = [&]() {
49
+ // We only have `stages - 2` active fetches since we are double buffering
50
+ // and can only issue the next fetch when it is guaranteed that the previous
51
+ // shared memory load is fully complete (as it may otherwise be
52
+ // overwritten).
53
+ cp_async_wait<repack_stages - 2>();
54
+ __syncthreads();
55
+ };
56
+
57
+ extern __shared__ int4 sh[];
58
+
59
+ constexpr int perm_size = tile_k_size / 4;
60
+
61
+ int4 *sh_perm_ptr = sh;
62
+ int4 *sh_pipe_ptr = sh_perm_ptr;
63
+ if constexpr (has_perm) {
64
+ sh_pipe_ptr += perm_size;
65
+ }
66
+
67
+ constexpr int stage_n_threads = tile_n_size / 4;
68
+ constexpr int stage_k_threads =
69
+ has_perm ? tile_k_size : tile_k_size / pack_factor_4bit;
70
+ constexpr int stage_size = stage_k_threads * stage_n_threads;
71
+
72
+ auto load_perm_to_shared = [&](int k_tile_id) {
73
+ int first_k_int4 = (k_tile_id * tile_k_size) / 4;
74
+
75
+ int4 const *perm_int4_ptr = reinterpret_cast<int4 const *>(perm_ptr);
76
+
77
+ if (threadIdx.x < perm_size) {
78
+ sh_perm_ptr[threadIdx.x] = perm_int4_ptr[first_k_int4 + threadIdx.x];
79
+ }
80
+ __syncthreads();
81
+ };
82
+
83
+ auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) {
84
+ if (n_tile_id >= n_tiles) {
85
+ cp_async_fence();
86
+ return;
87
+ }
88
+
89
+ int first_n = n_tile_id * tile_n_size;
90
+
91
+ int4 *sh_ptr = sh_pipe_ptr + stage_size * pipe;
92
+
93
+ if constexpr (has_perm) {
94
+ if (threadIdx.x < stage_size) {
95
+ int k_id = threadIdx.x / stage_n_threads;
96
+ int n_id = threadIdx.x % stage_n_threads;
97
+
98
+ uint32_t const *sh_perm_int_ptr =
99
+ reinterpret_cast<uint32_t const *>(sh_perm_ptr);
100
+
101
+ int src_k = sh_perm_int_ptr[k_id];
102
+ int src_k_packed = src_k / pack_factor_4bit;
103
+
104
+ cp_async4_stream(
105
+ &sh_ptr[k_id * stage_n_threads + n_id],
106
+ reinterpret_cast<int4 const *>(&(
107
+ b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)])));
108
+ }
109
+
110
+ } else {
111
+ if (threadIdx.x < stage_size) {
112
+ int k_id = threadIdx.x / stage_n_threads;
113
+ int n_id = threadIdx.x % stage_n_threads;
114
+
115
+ int first_k = k_tile_id * tile_k_size;
116
+ int first_k_packed = first_k / pack_factor_4bit;
117
+
118
+ cp_async4_stream(&sh_ptr[k_id * stage_n_threads + n_id],
119
+ reinterpret_cast<int4 const *>(
120
+ &(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
121
+ first_n + (n_id * 4)])));
122
+ }
123
+ }
124
+
125
+ cp_async_fence();
126
+ };
127
+
128
+ auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) {
129
+ if (n_tile_id >= n_tiles) {
130
+ return;
131
+ }
132
+
133
+ int warp_id = threadIdx.x / 32;
134
+ int th_id = threadIdx.x % 32;
135
+
136
+ if (warp_id >= 4) {
137
+ return;
138
+ }
139
+
140
+ int tc_col = th_id / 4;
141
+ int tc_row = (th_id % 4) * 2;
142
+
143
+ constexpr int tc_offsets[4] = {0, 1, 8, 9};
144
+
145
+ int cur_n = warp_id * 16 + tc_col;
146
+
147
+ constexpr int sh_stride = 64;
148
+
149
+ int4 *sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
150
+ uint32_t *sh_stage_int_ptr = reinterpret_cast<uint32_t *>(sh_stage_ptr);
151
+
152
+ uint32_t *sh_perm_int_ptr = reinterpret_cast<uint32_t *>(sh_perm_ptr);
153
+
154
+ uint32_t vals[pack_factor_4bit];
155
+
156
+ if constexpr (has_perm) {
157
+ for (int i = 0; i < 4; i++) {
158
+ int k_idx = tc_row + tc_offsets[i];
159
+
160
+ uint32_t src_k = sh_perm_int_ptr[k_idx];
161
+ uint32_t src_k_pos = src_k % pack_factor_4bit;
162
+
163
+ uint32_t b1_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n];
164
+ uint32_t b1_cur_val = (b1_val >> (src_k_pos * 4)) & 0xf;
165
+
166
+ uint32_t b2_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n + 8];
167
+ uint32_t b2_cur_val = (b2_val >> (src_k_pos * 4)) & 0xf;
168
+
169
+ vals[i] = b1_cur_val;
170
+ vals[4 + i] = b2_cur_val;
171
+ }
172
+
173
+ } else {
174
+
175
+ uint32_t b1_val_1 = sh_stage_int_ptr[cur_n];
176
+ uint32_t b1_val_2 = sh_stage_int_ptr[sh_stride + cur_n];
177
+
178
+ uint32_t b2_val_1 = sh_stage_int_ptr[cur_n + 8];
179
+ uint32_t b2_val_2 = sh_stage_int_ptr[sh_stride + cur_n + 8];
180
+
181
+#pragma unroll
182
+ for (int i = 0; i < 2; i++) {
183
+ int cur_elem = tc_row + tc_offsets[i];
184
+ vals[i] = (b1_val_1 >> (cur_elem * 4)) & 0xf;
185
+ vals[4 + i] = (b2_val_1 >> (cur_elem * 4)) & 0xf;
186
+ }
187
+
188
+#pragma unroll
189
+ for (int i = 2; i < 4; i++) {
190
+ int cur_elem = tc_row + tc_offsets[i] - 8;
191
+ vals[i] = (b1_val_2 >> (cur_elem * 4)) & 0xf;
192
+ vals[4 + i] = (b2_val_2 >> (cur_elem * 4)) & 0xf;
193
+ }
194
+ }
195
+
196
+ // Result of:
197
+ // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
198
+ constexpr int pack_idx[pack_factor_4bit] = {0, 2, 4, 6, 1, 3, 5, 7};
199
+
200
+ uint32_t res = 0;
201
+#pragma unroll
202
+ for (int i = 0; i < pack_factor_4bit; i++) {
203
+ res |= vals[pack_idx[i]] << (i * 4);
204
+ }
205
+
206
+ constexpr int tile_size = tile_k_size * tile_n_size / pack_factor_4bit;
207
+ int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
208
+
209
+ out_ptr[out_offset + th_id * 4 + warp_id] = res;
210
+ };
211
+
212
+ auto start_pipes = [&](int k_tile_id, int n_tile_id) {
213
+#pragma unroll
214
+ for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
215
+ fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
216
+ }
217
+
218
+ wait_for_stage();
219
+ };
220
+#pragma unroll
221
+ for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
222
+ int n_tile_id = 0;
223
+
224
+ if constexpr (has_perm) {
225
+ load_perm_to_shared(k_tile_id);
226
+ }
227
+
228
+ start_pipes(k_tile_id, n_tile_id);
229
+
230
+ while (n_tile_id < n_tiles) {
231
+#pragma unroll
232
+ for (int pipe = 0; pipe < repack_stages; pipe++) {
233
+ fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
234
+ n_tile_id + pipe + repack_stages - 1);
235
+ repack_tile(pipe, k_tile_id, n_tile_id + pipe);
236
+ wait_for_stage();
237
+ }
238
+ n_tile_id += repack_stages;
239
+ }
240
+ }
241
+}
242
+
243
+} // namespace gptq_marlin
244
+
245
+torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
246
+ int64_t size_k, int64_t size_n) {
247
+ // Verify compatibility with marlin tile of 16x64
248
+ TORCH_CHECK(size_k % gptq_marlin::tile_k_size == 0, "size_k = ", size_k,
249
+ " is not divisible by tile_k_size = ", gptq_marlin::tile_k_size);
250
+ TORCH_CHECK(size_n % gptq_marlin::tile_n_size == 0, "size_n = ", size_n,
251
+ " is not divisible by tile_n_size = ", gptq_marlin::tile_n_size);
252
+
253
+ // Verify B
254
+ TORCH_CHECK((size_k / gptq_marlin::pack_factor_4bit) == b_q_weight.size(0),
255
+ "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
256
+ ", size_k = ", size_k,
257
+ ", pack_factor_4bit = ", gptq_marlin::pack_factor_4bit);
258
+ TORCH_CHECK(b_q_weight.size(1) == size_n,
259
+ "b_q_weight.size(1) = ", b_q_weight.size(1),
260
+ " is not size_n = ", size_n);
261
+
262
+ // Verify device and strides
263
+ TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
264
+ TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
265
+ TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt");
266
+
267
+ TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
268
+ TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
269
+ TORCH_CHECK(perm.dtype() == at::kInt, "perm type is not at::kInt");
270
+
271
+ // Alloc buffers
272
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight));
273
+ auto options = torch::TensorOptions()
274
+ .dtype(b_q_weight.dtype())
275
+ .device(b_q_weight.device());
276
+ torch::Tensor out = torch::empty(
277
+ {size_k / gptq_marlin::tile_size,
278
+ size_n * gptq_marlin::tile_size / gptq_marlin::pack_factor_4bit},
279
+ options);
280
+
281
+ // Detect if there is act_order
282
+ bool has_perm = perm.size(0) != 0;
283
+
284
+ // Get ptrs
285
+ uint32_t const *b_q_weight_ptr =
286
+ reinterpret_cast<uint32_t const *>(b_q_weight.data_ptr());
287
+ uint32_t const *perm_ptr =
288
+ reinterpret_cast<uint32_t const *>(perm.data_ptr());
289
+ uint32_t *out_ptr = reinterpret_cast<uint32_t *>(out.data_ptr());
290
+
291
+ // Get dev info
292
+ int dev = b_q_weight.get_device();
293
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
294
+ int blocks;
295
+ cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
296
+
297
+ int max_shared_mem = 0;
298
+ cudaDeviceGetAttribute(&max_shared_mem,
299
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
300
+ TORCH_CHECK(max_shared_mem > 0);
301
+
302
+ if (has_perm) {
303
+ cudaFuncSetAttribute(
304
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, true>,
305
+ cudaFuncAttributeMaxDynamicSharedMemorySize,
306
+ max_shared_mem);
307
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, true>
308
+ <<<blocks, gptq_marlin::repack_threads, max_shared_mem,
309
+ stream>>>(b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n);
310
+
311
+ } else {
312
+ cudaFuncSetAttribute(
313
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, false>,
314
+ cudaFuncAttributeMaxDynamicSharedMemorySize,
315
+ max_shared_mem);
316
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, false>
317
+ <<<blocks, gptq_marlin::repack_threads, max_shared_mem,
318
+ stream>>>(b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n);
319
+ }
320
+
321
+ return out;
322
+}
323
+
324
+#endif
tests/models/test_gptq_marlin.py
new
+93
@@ -0,0 +1,93 @@
1
+"""Compares the outputs of gptq vs gptq_marlin
2
+Note: GPTQ and Marlin do not have bitwise correctness.
3
+As a result, in this test, we just confirm that the top selected tokens of the
4
+Marlin/GPTQ models are in the top 3 selections of each other.
5
+Note: Marlin internally uses locks to synchronize the threads. This can
6
+result in very slight nondeterminism for Marlin. As a result, we re-run the test
7
+up to 3 times to see if we pass.
8
+Note: This test currently fails running with --forked with the following:
9
+ RuntimeError: Cannot re-initialize CUDA in forked subprocess.
10
+ To use CUDA with multiprocessing, you must use the 'spawn' start method
11
+Run `pytest tests/models/test_gptq_marlin.py`.
12
+"""
13
+import os
14
+
15
+import pytest
16
+import torch
17
+
18
+from tests.models.utils import check_logprobs_close
19
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
20
+
21
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
22
+
23
+MAX_MODEL_LEN = 1024
24
+
25
+capability = torch.cuda.get_device_capability()
26
+capability = capability[0] * 10 + capability[1]
27
+gptq_marlin_not_supported = (
28
+ capability < QUANTIZATION_METHODS["gptq_marlin"].get_min_capability())
29
+
30
+MODELS = [
31
+ # act_order==False, group_size=channelwise
32
+ ("robertgshaw2/zephyr-7b-beta-channelwise-gptq", "main"),
33
+ # act_order==False, group_size=128
34
+ ("TheBloke/Llama-2-7B-GPTQ", "main"),
35
+
36
+ # act_order==True, group_size=128
37
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "main"),
38
+ # act_order==True, group_size=64
39
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-64g-actorder_True"),
40
+ # act_order==True, group_size=32
41
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-32g-actorder_True"),
42
+]
43
+
44
+
45
+@pytest.mark.flaky(reruns=2)
46
+@pytest.mark.skipif(gptq_marlin_not_supported,
47
+ reason="gptq_marlin is not supported on this GPU type.")
48
+@pytest.mark.parametrize("model", MODELS)
49
+@pytest.mark.parametrize("dtype", ["half"])
50
+@pytest.mark.parametrize("max_tokens", [32])
51
+@pytest.mark.parametrize("num_logprobs", [5])
52
+def test_models(
53
+ vllm_runner,
54
+ example_prompts,
55
+ model,
56
+ dtype: str,
57
+ max_tokens: int,
58
+ num_logprobs: int,
59
+) -> None:
60
+ model_name, revision = model
61
+
62
+ # Run marlin.
63
+ gptq_marlin_model = vllm_runner(model_name=model_name,
64
+ revision=revision,
65
+ dtype=dtype,
66
+ quantization="marlin",
67
+ max_model_len=MAX_MODEL_LEN,
68
+ tensor_parallel_size=1,
69
+ disable_custom_all_reduce=True)
70
+
71
+ gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
72
+ example_prompts, max_tokens, num_logprobs)
73
+ del gptq_marlin_model
74
+
75
+ # Run gptq.
76
+ gptq_model = vllm_runner(model_name=model_name,
77
+ revision=revision,
78
+ dtype=dtype,
79
+ quantization="gptq",
80
+ max_model_len=MAX_MODEL_LEN,
81
+ tensor_parallel_size=1,
82
+ disable_custom_all_reduce=True)
83
+ gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
84
+ max_tokens,
85
+ num_logprobs)
86
+ del gptq_model
87
+
88
+ check_logprobs_close(
89
+ outputs_0_lst=gptq_outputs,
90
+ outputs_1_lst=gptq_marlin_outputs,
91
+ name_0="gptq",
92
+ name_1="gptq_marlin",
93
+ )
tests/models/test_marlin.py
+13
-32
@@ -10,12 +10,12 @@ up to 3 times to see if we pass.
10
11
Run `pytest tests/models/test_marlin.py`.
12
"""
13
-
13
from dataclasses import dataclass
14
15
import pytest
16
import torch
17
18
+from tests.models.utils import check_logprobs_close
19
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
20
21
capability = torch.cuda.get_device_capability()
@@ -55,43 +55,24 @@ def test_models(
55
max_tokens: int,
56
num_logprobs: int,
57
) -> None:
58
- marlin_model = vllm_runner(model_pair.model_marlin, dtype=dtype)
58
+ marlin_model = vllm_runner(model_pair.model_marlin,
59
+ dtype=dtype,
60
+ quantization="marlin")
61
marlin_outputs = marlin_model.generate_greedy_logprobs(
62
example_prompts, max_tokens, num_logprobs)
61
-
62
- # Note: not sure why, but deleting just the model on Ada Lovelace
63
- # does not free the GPU memory. On Ampere, deleting the just model
64
- # frees the memory.
63
del marlin_model
64
67
- gptq_model = vllm_runner(model_pair.model_gptq, dtype=dtype)
65
+ gptq_model = vllm_runner(model_pair.model_gptq,
66
+ dtype=dtype,
67
+ quantization="gptq")
68
gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
69
max_tokens,
70
num_logprobs)
71
-
72
- # Note: not sure why, but deleting just the model on Ada Lovelace
73
- # does not free the GPU memory. On Ampere, deleting the just model
74
- # frees the memory.
71
del gptq_model
72
77
- # loop through the prompts
78
- for prompt_idx in range(len(example_prompts)):
79
- gptq_output_ids, gptq_output_str, gptq_logprobs = gptq_outputs[
80
- prompt_idx]
81
- marlin_output_ids, marlin_output_str, marlin_logprobs = marlin_outputs[
82
- prompt_idx]
83
-
84
- for idx, (gptq_output_id, marlin_output_id) in enumerate(
85
- zip(gptq_output_ids, marlin_output_ids)):
86
- # If sequence is not an exact match,
87
- if marlin_output_id != gptq_output_id:
88
- # Each predicted token must be in top 5 of the other's
89
- assert gptq_output_id in marlin_logprobs[idx], (
90
- f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\n"
91
- f"Marlin:\t{marlin_output_str!r}")
92
- assert marlin_output_id in gptq_logprobs[idx], (
93
- f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\n"
94
- f"Marlin:\t{marlin_output_str!r}")
95
-
96
- # Break out since sequences will now diverge.
97
- break
73
+ check_logprobs_close(
74
+ outputs_0_lst=gptq_outputs,
75
+ outputs_1_lst=marlin_outputs,
76
+ name_0="gptq",
77
+ name_1="marlin",
78
+ )
tests/models/utils.py
new
+29
@@ -0,0 +1,29 @@
1
+def check_logprobs_close(outputs_0_lst, outputs_1_lst, name_0, name_1):
2
+ """Compare the logprobs of two sequences generated by different models,
3
+ which should be similar but not necessarily equal.
4
+ """
5
+ # Loop through responses to each prompt.
6
+ for prompt_idx, (outputs_0,
7
+ outputs_1) in enumerate(zip(outputs_0_lst,
8
+ outputs_1_lst)):
9
+ output_ids_0, output_str_0, logprobs_0 = outputs_0
10
+ output_ids_1, output_str_1, logprobs_1 = outputs_1
11
+
12
+ # Loop through generated tokens.
13
+ for idx, (output_id_0,
14
+ output_id_1) in enumerate(zip(output_ids_0, output_ids_1)):
15
+
16
+ # If generated tokens don't match, then
17
+ if output_id_0 != output_id_1:
18
+ # Each predicted token must be in top N logprobs of the other
19
+ assert output_id_0 in logprobs_1[idx], (
20
+ f"Test{prompt_idx}:"
21
+ f"\n{name_0}:\t{output_str_0!r}"
22
+ f"\n{name_1}:\t{output_str_1!r}")
23
+ assert output_id_1 in logprobs_0[idx], (
24
+ f"Test{prompt_idx}:"
25
+ f"\n{name_0}:\t{output_str_0!r}"
26
+ f"\n{name_1}:\t{output_str_1!r}")
27
+
28
+ # Break out since sequences will now diverge.
29
+ break
tests/quantization/test_autogptq_marlin_configs.py
deleted
-64
@@ -1,64 +0,0 @@
1
-"""Tests whether Marlin models can be loaded from the autogptq config.
2
-
3
-Run `pytest tests/quantization/test_autogptq_marlin_configs.py --forked`.
4
-"""
5
-
6
-from dataclasses import dataclass
7
-
8
-import pytest
9
-
10
-from vllm.config import ModelConfig
11
-
12
-
13
-@dataclass
14
-class ModelPair:
15
- model_marlin: str
16
- model_gptq: str
17
-
18
-
19
-# Model Id // Expected Kernel
20
-MODELS_QUANT_TYPE = [
21
- # compat: autogptq <=0.7.1 is_marlin_format: bool
22
- ("neuralmagic/TinyLlama-1.1B-Chat-v1.0-marlin", "marlin"),
23
- ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq"),
24
- # compat: autogptq >=0.8.0 use checkpoint_format: str
25
- ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-Marlin-4bit", "marlin"),
26
- ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq")
27
-]
28
-
29
-
30
-@pytest.mark.parametrize("model_quant_type", MODELS_QUANT_TYPE)
31
-def test_auto_gptq(model_quant_type: str, ) -> None:
32
- model_path, quant_type = model_quant_type
33
-
34
- model_config_no_quant_arg = ModelConfig(
35
- model_path,
36
- model_path,
37
- tokenizer_mode="auto",
38
- trust_remote_code=False,
39
- seed=0,
40
- dtype="float16",
41
- revision=None,
42
- quantization=None # case 1
43
- )
44
-
45
- model_config_quant_arg = ModelConfig(
46
- model_path,
47
- model_path,
48
- tokenizer_mode="auto",
49
- trust_remote_code=False,
50
- seed=0,
51
- dtype="float16",
52
- revision=None,
53
- quantization="gptq" # case 2
54
- )
55
-
56
- assert model_config_no_quant_arg.quantization == quant_type, (
57
- f"Expected quant_type == {quant_type} for {model_path}, "
58
- f"but found {model_config_no_quant_arg.quantization} "
59
- "for no --quantization None case")
60
-
61
- assert model_config_quant_arg.quantization == quant_type, (
62
- f"Expected quant_type == {quant_type} for {model_path}, "
63
- f"but found {model_config_quant_arg.quantization} "
64
- "for --quantization gptq case")
tests/quantization/test_configs.py
new
+73
@@ -0,0 +1,73 @@
1
+"""Tests whether Marlin models can be loaded from the autogptq config.
2
+
3
+Run `pytest tests/quantization/test_configs.py --forked`.
4
+"""
5
+
6
+from dataclasses import dataclass
7
+
8
+import pytest
9
+
10
+from vllm.config import ModelConfig
11
+
12
+
13
+@dataclass
14
+class ModelPair:
15
+ model_marlin: str
16
+ model_gptq: str
17
+
18
+
19
+# Model Id // Quantization Arg // Expected Type
20
+MODEL_ARG_EXPTYPES = [
21
+ # AUTOGPTQ
22
+ # compat: autogptq <=0.7.1 is_marlin_format: bool
23
+ # Model Serialized in Marlin Format should always use Marlin kernel.
24
+ ("neuralmagic/TinyLlama-1.1B-Chat-v1.0-marlin", None, "marlin"),
25
+ ("neuralmagic/TinyLlama-1.1B-Chat-v1.0-marlin", "marlin", "marlin"),
26
+ ("neuralmagic/TinyLlama-1.1B-Chat-v1.0-marlin", "gptq", "marlin"),
27
+ ("neuralmagic/TinyLlama-1.1B-Chat-v1.0-marlin", "awq", "ERROR"),
28
+ # Model Serialized in Exllama Format.
29
+ ("TheBloke/Llama-2-7B-Chat-GPTQ", None, "gptq_marlin"),
30
+ ("TheBloke/Llama-2-7B-Chat-GPTQ", "marlin", "gptq_marlin"),
31
+ ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "gptq"),
32
+ ("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
33
+ # compat: autogptq >=0.8.0 use checkpoint_format: str
34
+ # Model Serialized in Marlin Format should always use Marlin kernel.
35
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-Marlin-4bit", None, "marlin"),
36
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-Marlin-4bit", "marlin", "marlin"),
37
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-Marlin-4bit", "gptq", "marlin"),
38
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-Marlin-4bit", "awq", "ERROR"),
39
+ # Model Serialized in Exllama Format.
40
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "gptq_marlin"),
41
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "marlin", "gptq_marlin"),
42
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "gptq"),
43
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
44
+
45
+ # AUTOAWQ
46
+ ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, "awq"),
47
+ ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
48
+ ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", "ERROR"),
49
+ ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
50
+]
51
+
52
+
53
+@pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)
54
+def test_auto_gptq(model_arg_exptype: str) -> None:
55
+ model_path, quantization_arg, expected_type = model_arg_exptype
56
+
57
+ try:
58
+ model_config = ModelConfig(model_path,
59
+ model_path,
60
+ tokenizer_mode="auto",
61
+ trust_remote_code=False,
62
+ seed=0,
63
+ dtype="float16",
64
+ revision=None,
65
+ quantization=quantization_arg)
66
+ found_quantization_type = model_config.quantization
67
+ except ValueError:
68
+ found_quantization_type = "ERROR"
69
+
70
+ assert found_quantization_type == expected_type, (
71
+ f"Expected quant_type == {expected_type} for {model_path}, "
72
+ f"but found {found_quantization_type} "
73
+ f"for no --quantization {quantization_arg} case")
vllm/config.py
+31
-8
@@ -9,11 +9,14 @@ from packaging.version import Version
9
from transformers import PretrainedConfig
10
11
from vllm.logger import init_logger
12
-from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
+from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
13
+ get_quantization_config)
14
from vllm.transformers_utils.config import get_config, get_hf_text_config
15
from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
16
is_neuron)
17
18
+GPTQMarlinConfig = get_quantization_config("gptq_marlin")
19
+
20
if TYPE_CHECKING:
21
from ray.util.placement_group import PlacementGroup
22
@@ -138,14 +141,34 @@ class ModelConfig:
141
is_format_marlin = (quant_cfg.get("checkpoint_format") == "marlin"
142
or quant_cfg.get("is_marlin_format", False))
143
141
- # Use marlin if the GPTQ model is serialized in marlin format.
142
- if quant_method == "gptq" and is_format_marlin:
143
- logger.info("The model is serialized in Marlin format. "
144
+ # Check which LinearMethod the GPTQ model should use.
145
+ if quant_method == "gptq":
146
+ # If serialized in Marlin format, use MarlinLinearMethod.
147
+ # TODO (@robertgshaw): migrate under GPTQMarlinLinearMethod.
148
+ if is_format_marlin:
149
+ logger.info("The model is serialized in Marlin format. "
150
+ "Using Marlin kernel.")
151
+ quant_method = "marlin"
152
+ if self.quantization == "gptq":
153
+ self.quantization = quant_method
154
+
155
+ # If convertible to Marlin format, use GPTQMarlinLinearMethod
156
+ # unless the user explicitly specified GPTQLinearMethod.
157
+ elif GPTQMarlinConfig.is_marlin_compatible(quant_cfg):
158
+ if self.quantization == "gptq":
159
+ logger.warning(
160
+ "The model is convertible to Marlin format, but "
161
+ "you specified quantization=gptq. Use "
162
+ "quantization=marlin for faster inference.")
163
+ else:
164
+ logger.info(
165
+ "The model is convertible to Marlin format. "
166
"Using Marlin kernel.")
145
- quant_method = "marlin"
146
- if self.quantization == "gptq":
147
- self.quantization = quant_method
167
+ quant_method = "gptq_marlin"
168
+ if self.quantization == "marlin":
169
+ self.quantization = quant_method
170
171
+ # Verify quantization configurations.
172
if self.quantization is None:
173
self.quantization = quant_method
174
elif self.quantization != quant_method:
@@ -165,7 +188,7 @@ class ModelConfig:
188
raise ValueError(
189
f"{self.quantization} quantization is currently not "
190
f"supported in ROCm.")
168
- if self.quantization != "marlin":
191
+ if (self.quantization not in ["marlin", "gptq_marlin"]):
192
logger.warning(
193
"%s quantization is not fully "
194
"optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -6,6 +6,8 @@ from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
7
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
8
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
9
+from vllm.model_executor.layers.quantization.gptq_marlin import (
10
+ GPTQMarlinConfig)
11
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
12
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
13
@@ -15,6 +17,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
17
"fp8": Fp8Config,
18
"gptq": GPTQConfig,
19
"squeezellm": SqueezeLLMConfig,
20
+ "gptq_marlin": GPTQMarlinConfig,
21
"marlin": MarlinConfig,
22
}
23
vllm/model_executor/layers/quantization/gptq_marlin.py
new
+444
@@ -0,0 +1,444 @@
1
+import enum
2
+from enum import Enum
3
+from typing import Any, Dict, List, Optional
4
+
5
+import numpy
6
+import torch
7
+from torch.nn.parameter import Parameter
8
+
9
+from vllm._C import ops
10
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
+ set_weight_attrs)
12
+from vllm.model_executor.layers.quantization.base_config import (
13
+ QuantizationConfig)
14
+
15
+GPTQ_MARLIN_TILE = 16
16
+GPTQ_MARLIN_MIN_THREAD_N = 64
17
+GPTQ_MARLIN_MIN_THREAD_K = 128
18
+GPTQ_MARLIN_MAX_PARALLEL = 16
19
+
20
+GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4]
21
+GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
22
+GPTQ_MARLIN_SUPPORTED_SYM = [True]
23
+
24
+
25
+# Precompute permutations for Marlin weight and scale shuffling
26
+#
27
+# Marlin works on [16,64] tiles. The goal of the permutations
28
+# is to reorder the weight data so that it is compatible
29
+# with the tensor-core format that is described here:
30
+# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
31
+#
32
+# As a result of this reordering, the vector loads inside the
33
+# kernel will get the data as it is needed for tensor-core
34
+# (without the need to use ldmatrix instructions)
35
+def _get_perms():
36
+ perm = []
37
+ for i in range(32):
38
+ perm1 = []
39
+ col = i // 4
40
+ for block in [0, 1]:
41
+ for row in [
42
+ 2 * (i % 4),
43
+ 2 * (i % 4) + 1,
44
+ 2 * (i % 4 + 4),
45
+ 2 * (i % 4 + 4) + 1,
46
+ ]:
47
+ perm1.append(16 * row + col + 8 * block)
48
+ for j in range(4):
49
+ perm.extend([p + 256 * j for p in perm1])
50
+
51
+ perm = numpy.array(perm)
52
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
53
+ perm = perm.reshape((-1, 8))[:, interleave].ravel() # type: ignore
54
+ perm = torch.from_numpy(perm)
55
+ scale_perm = []
56
+ for i in range(8):
57
+ scale_perm.extend([i + 8 * j for j in range(8)])
58
+ scale_perm_single = []
59
+ for i in range(4):
60
+ scale_perm_single.extend(
61
+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
62
+ return perm, scale_perm, scale_perm_single
63
+
64
+
65
+_perm, _scale_perm, _scale_perm_single = _get_perms()
66
+
67
+
68
+def get_pack_factor(num_bits):
69
+ assert num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS, (
70
+ f"Unsupported num_bits = {num_bits}")
71
+ return 32 // num_bits
72
+
73
+
74
+def marlin_permute_scales(s, size_k, size_n, group_size):
75
+ if group_size < size_k and group_size != -1:
76
+ s = s.reshape((-1, len(_scale_perm)))[:, _scale_perm]
77
+ else:
78
+ s = s.reshape((-1, len(_scale_perm_single)))[:, _scale_perm_single]
79
+ s = s.reshape((-1, size_n)).contiguous()
80
+
81
+ return s
82
+
83
+
84
+class GPTQMarlinConfig(QuantizationConfig):
85
+ """Config class for GPTQ Marlin"""
86
+
87
+ def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
88
+ is_sym: bool) -> None:
89
+ if desc_act and group_size == -1:
90
+ # In this case, act_order == True is the same as act_order == False
91
+ # (since we have only one group per output channel)
92
+ desc_act = False
93
+
94
+ self.weight_bits = weight_bits
95
+ self.group_size = group_size
96
+ self.desc_act = desc_act
97
+ self.is_sym = is_sym
98
+
99
+ # Verify
100
+ if self.weight_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
101
+ raise ValueError(
102
+ f"Marlin does not support weight_bits = {self.weight_bits}. "
103
+ f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
104
+ "are supported.")
105
+ if self.group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
106
+ raise ValueError(
107
+ f"Marlin does not support group_size = {self.group_size}. "
108
+ f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
109
+ "are supported.")
110
+ if self.is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
111
+ raise ValueError(
112
+ f"Marlin does not support is_sym = {self.is_sym}. "
113
+ f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
114
+
115
+ # Init
116
+ self.pack_factor = get_pack_factor(weight_bits)
117
+ self.tile_size = GPTQ_MARLIN_TILE
118
+ self.min_thread_n = GPTQ_MARLIN_MIN_THREAD_N
119
+ self.min_thread_k = GPTQ_MARLIN_MIN_THREAD_K
120
+ self.max_parallel = GPTQ_MARLIN_MAX_PARALLEL
121
+
122
+ def __repr__(self) -> str:
123
+ return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
124
+ f"group_size={self.group_size}, "
125
+ f"desc_act={self.desc_act})")
126
+
127
+ @classmethod
128
+ def get_name(cls) -> str:
129
+ return "gptq_marlin"
130
+
131
+ @classmethod
132
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
133
+ return [torch.half]
134
+
135
+ @classmethod
136
+ def get_min_capability(cls) -> int:
137
+ return 80
138
+
139
+ @classmethod
140
+ def get_config_filenames(cls) -> List[str]:
141
+ return ["quantize_config.json"]
142
+
143
+ @classmethod
144
+ def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlinConfig":
145
+ weight_bits = cls.get_from_keys(config, ["bits"])
146
+ group_size = cls.get_from_keys(config, ["group_size"])
147
+ desc_act = cls.get_from_keys(config, ["desc_act"])
148
+ is_sym = cls.get_from_keys(config, ["sym"])
149
+ return cls(weight_bits, group_size, desc_act, is_sym)
150
+
151
+ def get_quant_method(
152
+ self,
153
+ layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
154
+ if isinstance(layer, LinearBase):
155
+ return GPTQMarlinLinearMethod(self)
156
+ return None
157
+
158
+ def get_scaled_act_names(self) -> List[str]:
159
+ return []
160
+
161
+ @classmethod
162
+ def is_marlin_compatible(cls, quant_config: Dict[str, Any]):
163
+ # Extract data from quant config.
164
+ num_bits = quant_config.get("bits", None)
165
+ group_size = quant_config.get("group_size", None)
166
+ sym = quant_config.get("sym", None)
167
+ desc_act = quant_config.get("desc_act", None)
168
+
169
+ # If we cannot find the info needed in the config, cannot convert.
170
+ if (num_bits is None or group_size is None or sym is None
171
+ or desc_act is None):
172
+ return False
173
+
174
+ # If the capability of the device is too low, cannot convert.
175
+ major, minor = torch.cuda.get_device_capability()
176
+ device_capability = major * 10 + minor
177
+ if device_capability < cls.get_min_capability():
178
+ return False
179
+
180
+ # Otherwise, can convert if model satisfies marlin constraints.
181
+ return (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
182
+ and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
183
+ and sym in GPTQ_MARLIN_SUPPORTED_SYM)
184
+
185
+
186
+class GPTQMarlinState(Enum):
187
+ REPACK = enum.auto()
188
+ READY = enum.auto()
189
+
190
+
191
+class GPTQMarlinLinearMethod(LinearMethodBase):
192
+ """Linear method for GPTQ Marlin.
193
+
194
+ Args:
195
+ quant_config: The GPTQ Marlin quantization config.
196
+ """
197
+
198
+ def __init__(self, quant_config: GPTQMarlinConfig) -> None:
199
+ self.quant_config = quant_config
200
+
201
+ def create_weights(
202
+ self,
203
+ layer: torch.nn.Module,
204
+ input_size_per_partition: int,
205
+ output_partition_sizes: List[int],
206
+ input_size: int,
207
+ output_size: int,
208
+ params_dtype: torch.dtype,
209
+ **extra_weight_attrs,
210
+ ) -> None:
211
+ del output_size
212
+
213
+ # Normalize group_size
214
+ if self.quant_config.group_size != -1:
215
+ group_size = self.quant_config.group_size
216
+ else:
217
+ group_size = input_size
218
+
219
+ # Validate dtype
220
+ if params_dtype != torch.float16:
221
+ raise ValueError(
222
+ f"The params dtype must be float16, but got {params_dtype}")
223
+
224
+ # Validate output_size_per_partition
225
+ output_size_per_partition = sum(output_partition_sizes)
226
+ if output_size_per_partition % self.quant_config.min_thread_n != 0:
227
+ raise ValueError(
228
+ f"Weight output_size_per_partition = "
229
+ f"{output_size_per_partition} is not divisible by "
230
+ f" min_thread_n = {self.quant_config.min_thread_n}.")
231
+
232
+ # Validate input_size_per_partition
233
+ if input_size_per_partition % self.quant_config.min_thread_k != 0:
234
+ raise ValueError(
235
+ f"Weight input_size_per_partition = "
236
+ f"{input_size_per_partition} is not divisible "
237
+ f"by min_thread_k = {self.quant_config.min_thread_k}.")
238
+
239
+ if (group_size < input_size
240
+ and input_size_per_partition % group_size != 0):
241
+ raise ValueError(
242
+ f"Weight input_size_per_partition = {input_size_per_partition}"
243
+ f" is not divisible by group_size = {group_size}.")
244
+
245
+ # Detect sharding of scales/zp
246
+
247
+ # By default, no sharding over "input dim"
248
+ scales_and_zp_size = input_size // group_size
249
+ scales_and_zp_input_dim = None
250
+
251
+ if self.quant_config.desc_act:
252
+ # Act-order case
253
+ assert self.quant_config.group_size != -1
254
+
255
+ is_k_full = input_size_per_partition == input_size
256
+
257
+ else:
258
+ # No act-order case
259
+
260
+ # K is always full due to full alignment with
261
+ # group-size and shard of scales/zp
262
+ is_k_full = True
263
+
264
+ # If this is a row-parallel case, then shard scales/zp
265
+ if (input_size != input_size_per_partition
266
+ and self.quant_config.group_size != -1):
267
+ scales_and_zp_size = input_size_per_partition // group_size
268
+ scales_and_zp_input_dim = 0
269
+
270
+ # Init buffers
271
+
272
+ # Quantized weights
273
+ qweight = Parameter(
274
+ torch.empty(
275
+ input_size_per_partition // self.quant_config.pack_factor,
276
+ output_size_per_partition,
277
+ dtype=torch.int32,
278
+ ),
279
+ requires_grad=False,
280
+ )
281
+ set_weight_attrs(
282
+ qweight, {
283
+ **extra_weight_attrs,
284
+ "input_dim": 0,
285
+ "output_dim": 1,
286
+ "packed_dim": 0,
287
+ "pack_factor": self.quant_config.pack_factor,
288
+ })
289
+
290
+ # Activation order
291
+ g_idx = Parameter(
292
+ torch.empty(
293
+ input_size_per_partition,
294
+ dtype=torch.int32,
295
+ ),
296
+ requires_grad=False,
297
+ )
298
+ # Ignore warning from fused linear layers such as QKVParallelLinear.
299
+ set_weight_attrs(g_idx, {
300
+ **extra_weight_attrs, "input_dim": 0,
301
+ "ignore_warning": True
302
+ })
303
+
304
+ g_idx_sort_indices = Parameter(
305
+ torch.empty(
306
+ g_idx.shape,
307
+ dtype=torch.int32,
308
+ ),
309
+ requires_grad=False,
310
+ )
311
+ set_weight_attrs(g_idx_sort_indices, extra_weight_attrs)
312
+
313
+ # Scales
314
+ scales = Parameter(
315
+ torch.empty(
316
+ scales_and_zp_size,
317
+ output_size_per_partition,
318
+ dtype=params_dtype,
319
+ ),
320
+ requires_grad=False,
321
+ )
322
+ set_weight_attrs(
323
+ scales, {
324
+ **extra_weight_attrs,
325
+ "input_dim": scales_and_zp_input_dim,
326
+ "output_dim": 1,
327
+ })
328
+
329
+ # Quantized zero-points
330
+ qzeros = Parameter(
331
+ torch.empty(scales_and_zp_size,
332
+ output_size_per_partition //
333
+ self.quant_config.pack_factor,
334
+ dtype=torch.int32,
335
+ device="meta"),
336
+ requires_grad=False,
337
+ )
338
+ set_weight_attrs(
339
+ qzeros, {
340
+ **extra_weight_attrs,
341
+ "input_dim": scales_and_zp_input_dim,
342
+ "output_dim": 1,
343
+ "packed_dim": 1,
344
+ "pack_factor": self.quant_config.pack_factor,
345
+ })
346
+
347
+ # Allocate marlin workspace
348
+ max_workspace_size = (
349
+ output_size_per_partition //
350
+ self.quant_config.min_thread_n) * self.quant_config.max_parallel
351
+ workspace = torch.zeros(max_workspace_size,
352
+ dtype=torch.int,
353
+ requires_grad=False)
354
+
355
+ layer.register_parameter("qweight", qweight)
356
+ layer.register_parameter("g_idx", g_idx)
357
+ layer.register_parameter("g_idx_sort_indices", g_idx_sort_indices)
358
+ layer.register_parameter("scales", scales)
359
+ layer.register_parameter("qzeros", qzeros)
360
+ layer.workspace = workspace
361
+ layer.input_size_per_partition = input_size_per_partition
362
+ layer.output_size_per_partition = output_size_per_partition
363
+ layer.input_size = input_size
364
+ layer.is_k_full = is_k_full
365
+ layer.marlin_state = GPTQMarlinState.REPACK
366
+
367
+ def apply(
368
+ self,
369
+ layer: torch.nn.Module,
370
+ x: torch.Tensor,
371
+ bias: Optional[torch.Tensor] = None,
372
+ ) -> torch.Tensor:
373
+ reshaped_x = x.reshape(-1, x.shape[-1])
374
+
375
+ size_m = reshaped_x.shape[0]
376
+ part_size_n = layer.output_size_per_partition
377
+ part_size_k = layer.input_size_per_partition
378
+ full_size_k = layer.input_size
379
+
380
+ out_shape = x.shape[:-1] + (part_size_n, )
381
+
382
+ if layer.marlin_state == GPTQMarlinState.REPACK:
383
+ layer.marlin_state = GPTQMarlinState.READY
384
+
385
+ # Newly generated tensors need to replace existing tensors that are
386
+ # already registered as parameters by vLLM (and won't be freed)
387
+ def replace_tensor(name, new_t):
388
+ # It is important to use resize_() here since it ensures
389
+ # the same buffer is reused
390
+ getattr(layer, name).resize_(new_t.shape)
391
+ getattr(layer, name).copy_(new_t)
392
+ del new_t
393
+
394
+ cur_device = layer.qweight.device
395
+
396
+ # Process act_order
397
+ if self.quant_config.desc_act:
398
+ # Get sorting based on g_idx
399
+ g_idx_sort_indices = torch.argsort(layer.g_idx).to(torch.int)
400
+
401
+ sorted_g_idx = layer.g_idx[g_idx_sort_indices]
402
+
403
+ replace_tensor("g_idx", sorted_g_idx)
404
+ replace_tensor("g_idx_sort_indices", g_idx_sort_indices)
405
+
406
+ else:
407
+ # Reset g_idx related tensors
408
+ layer.g_idx = Parameter(torch.empty(0,
409
+ dtype=torch.int,
410
+ device=cur_device),
411
+ requires_grad=False)
412
+ layer.g_idx_sort_indices = Parameter(torch.empty(
413
+ 0, dtype=torch.int, device=cur_device),
414
+ requires_grad=False)
415
+
416
+ # Repack weights
417
+ marlin_qweight = ops.gptq_marlin_repack(
418
+ layer.qweight,
419
+ layer.g_idx_sort_indices,
420
+ part_size_k,
421
+ part_size_n,
422
+ )
423
+ replace_tensor("qweight", marlin_qweight)
424
+
425
+ # Permute scales
426
+ scales_size_k = part_size_k
427
+ scales_size_n = part_size_n
428
+ if self.quant_config.desc_act:
429
+ scales_size_k = full_size_k
430
+
431
+ marlin_scales = marlin_permute_scales(layer.scales, scales_size_k,
432
+ scales_size_n,
433
+ self.quant_config.group_size)
434
+ replace_tensor("scales", marlin_scales)
435
+
436
+ output = ops.gptq_marlin_gemm(reshaped_x, layer.qweight, layer.scales,
437
+ layer.g_idx, layer.g_idx_sort_indices,
438
+ layer.workspace, size_m, part_size_n,
439
+ part_size_k, layer.is_k_full)
440
+
441
+ if bias is not None:
442
+ output.add_(bias) # In-place add
443
+
444
+ return output.reshape(out_shape)