A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Kernel][Core] Add AWQ support to the Marlin kernel (#6612)
Alexander Matveev committed
Jul 21, 2024 at 19:41 UTC
396d92d5e064d131422886482cb1b75e5b6538d9
21 files changed
+1602
-284
CMakeLists.txt
+1
@@ -172,6 +172,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
172
"csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
173
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
174
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
175
+ "csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
176
"csrc/quantization/fp8/fp8_marlin.cu"
177
"csrc/custom_all_reduce.cu"
178
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
csrc/ops.h
+8
-4
@@ -89,15 +89,19 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
89
int64_t size_k);
90
91
torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
92
- torch::Tensor& b_scales, torch::Tensor& g_idx,
93
- torch::Tensor& perm, torch::Tensor& workspace,
94
- int64_t num_bits, int64_t size_m, int64_t size_n,
95
- int64_t size_k, bool is_k_full);
92
+ torch::Tensor& b_scales, torch::Tensor& b_zeros,
93
+ torch::Tensor& g_idx, torch::Tensor& perm,
94
+ torch::Tensor& workspace, int64_t num_bits,
95
+ int64_t size_m, int64_t size_n, int64_t size_k,
96
+ bool is_k_full, bool has_zp);
97
98
torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
99
int64_t size_k, int64_t size_n,
100
int64_t num_bits);
101
102
+torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
103
+ int64_t size_n, int64_t num_bits);
104
+
105
torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
106
torch::Tensor& b_scales, torch::Tensor& workspace,
107
int64_t num_bits, int64_t size_m, int64_t size_n,
csrc/quantization/fp8/fp8_marlin.cu
+15
-18
@@ -19,10 +19,10 @@
19
* Adapted from https://github.com/IST-DASLab/marlin
20
*/
21
22
-#include "../gptq_marlin/gptq_marlin.cuh"
23
-#include "../gptq_marlin/gptq_marlin_dtypes.cuh"
22
+#include "../gptq_marlin/marlin.cuh"
23
+#include "../gptq_marlin/marlin_dtypes.cuh"
24
25
-using namespace gptq_marlin;
25
+using namespace marlin;
26
27
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
28
static_assert(std::is_same<scalar_t, half>::value || \
@@ -1224,16 +1224,15 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1224
", size_k = ", size_k);
1225
1226
// Verify B
1227
- TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1228
- " is not divisible by tile_size = ", gptq_marlin::tile_size);
1229
- TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
1227
+ TORCH_CHECK(size_k % marlin::tile_size == 0, "size_k = ", size_k,
1228
+ " is not divisible by tile_size = ", marlin::tile_size);
1229
+ TORCH_CHECK((size_k / marlin::tile_size) == b_q_weight.size(0),
1230
"Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1231
- ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1232
- TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1231
+ ", size_k = ", size_k, ", tile_size = ", marlin::tile_size);
1232
+ TORCH_CHECK(b_q_weight.size(1) % marlin::tile_size == 0,
1233
"b_q_weight.size(1) = ", b_q_weight.size(1),
1234
- " is not divisible by tile_size = ", gptq_marlin::tile_size);
1235
- int actual_size_n =
1236
- (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
1234
+ " is not divisible by tile_size = ", marlin::tile_size);
1235
+ int actual_size_n = (b_q_weight.size(1) / marlin::tile_size) * pack_factor;
1236
TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
1237
", actual_size_n = ", actual_size_n);
1238
@@ -1274,11 +1273,9 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1273
num_groups = b_scales.size(0);
1274
1275
// Verify workspace size
1277
- TORCH_CHECK(
1278
- size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1279
- ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1280
- int min_workspace_size =
1281
- (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
1276
+ TORCH_CHECK(size_n % marlin::min_thread_n == 0, "size_n = ", size_n,
1277
+ ", is not divisible by min_thread_n = ", marlin::min_thread_n);
1278
+ int min_workspace_size = (size_n / marlin::min_thread_n) * marlin::max_par;
1279
TORCH_CHECK(workspace.numel() >= min_workspace_size,
1280
"workspace.numel = ", workspace.numel(),
1281
" is below min_workspace_size = ", min_workspace_size);
@@ -1290,14 +1287,14 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1287
b_scales.data_ptr<at::Half>(), size_m, size_n, size_k,
1288
workspace.data_ptr(), num_bits, num_groups, group_size, dev,
1289
at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1293
- gptq_marlin::max_par);
1290
+ marlin::max_par);
1291
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
1292
fp8_marlin::marlin_mm_f16i4<nv_bfloat16>(
1293
a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
1294
c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(), size_m,
1295
size_n, size_k, workspace.data_ptr(), num_bits, num_groups, group_size,
1296
dev, at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1300
- gptq_marlin::max_par);
1297
+ marlin::max_par);
1298
} else {
1299
TORCH_CHECK(false, "fp8_marlin_gemm only supports bfloat16 and float16");
1300
}
csrc/quantization/gptq_marlin/awq_marlin_repack.cu
new
+269
@@ -0,0 +1,269 @@
1
+#include "marlin.cuh"
2
+
3
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4
+
5
+namespace marlin {
6
+
7
+template <int const num_threads, int const num_bits, bool const has_perm>
8
+__global__ void awq_marlin_repack_kernel(
9
+ uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
10
+ int size_k, int size_n) {}
11
+
12
+} // namespace marlin
13
+
14
+torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
15
+ int64_t size_k, int64_t size_n,
16
+ int64_t num_bits) {
17
+ TORCH_CHECK_NOT_IMPLEMENTED(
18
+ false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
19
+ return torch::empty({1, 1});
20
+}
21
+
22
+#else
23
+
24
+namespace marlin {
25
+
26
+template <int const num_threads, int const num_bits>
27
+__global__ void awq_marlin_repack_kernel(
28
+ uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
29
+ int size_k, int size_n) {
30
+ constexpr int pack_factor = 32 / num_bits;
31
+
32
+ int k_tiles = size_k / tile_k_size;
33
+ int n_tiles = size_n / tile_n_size;
34
+ int block_k_tiles = div_ceil(k_tiles, gridDim.x);
35
+
36
+ int start_k_tile = blockIdx.x * block_k_tiles;
37
+ if (start_k_tile >= k_tiles) {
38
+ return;
39
+ }
40
+
41
+ int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles);
42
+
43
+ // Wait until the next thread tile has been loaded to shared memory.
44
+ auto wait_for_stage = [&]() {
45
+ // We only have `stages - 2` active fetches since we are double buffering
46
+ // and can only issue the next fetch when it is guaranteed that the previous
47
+ // shared memory load is fully complete (as it may otherwise be
48
+ // overwritten).
49
+ cp_async_wait<repack_stages - 2>();
50
+ __syncthreads();
51
+ };
52
+
53
+ extern __shared__ int4 sh[];
54
+
55
+ constexpr int tile_n_ints = tile_n_size / pack_factor;
56
+
57
+ constexpr int stage_n_threads = tile_n_ints / 4;
58
+ constexpr int stage_k_threads = tile_k_size;
59
+ constexpr int stage_size = stage_k_threads * stage_n_threads;
60
+
61
+ auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) {
62
+ if (n_tile_id >= n_tiles) {
63
+ cp_async_fence();
64
+ return;
65
+ }
66
+
67
+ int first_n = n_tile_id * tile_n_size;
68
+ int first_n_packed = first_n / pack_factor;
69
+
70
+ int4* sh_ptr = sh + stage_size * pipe;
71
+
72
+ if (threadIdx.x < stage_size) {
73
+ int k_id = threadIdx.x / stage_n_threads;
74
+ int n_id = threadIdx.x % stage_n_threads;
75
+
76
+ int first_k = k_tile_id * tile_k_size;
77
+
78
+ cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
79
+ reinterpret_cast<int4 const*>(
80
+ &(b_q_weight_ptr[(first_k + k_id) * (size_n / pack_factor) +
81
+ first_n_packed + (n_id * 4)])));
82
+ }
83
+
84
+ cp_async_fence();
85
+ };
86
+
87
+ auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) {
88
+ if (n_tile_id >= n_tiles) {
89
+ return;
90
+ }
91
+
92
+ int warp_id = threadIdx.x / 32;
93
+ int th_id = threadIdx.x % 32;
94
+
95
+ if (warp_id >= 4) {
96
+ return;
97
+ }
98
+
99
+ int tc_col = th_id / 4;
100
+ int tc_row = (th_id % 4) * 2;
101
+
102
+ constexpr int tc_offsets[4] = {0, 1, 8, 9};
103
+
104
+ int cur_n = warp_id * 16 + tc_col;
105
+ int cur_n_packed = cur_n / pack_factor;
106
+ int cur_n_pos = cur_n % pack_factor;
107
+
108
+ constexpr int sh_stride = tile_n_ints;
109
+ constexpr uint32_t mask = (1 << num_bits) - 1;
110
+
111
+ int4* sh_stage_ptr = sh + stage_size * pipe;
112
+ uint32_t* sh_stage_int_ptr = reinterpret_cast<uint32_t*>(sh_stage_ptr);
113
+
114
+ // Undo interleaving
115
+ int cur_n_pos_unpacked;
116
+ if constexpr (num_bits == 4) {
117
+ constexpr int undo_pack[8] = {0, 4, 1, 5, 2, 6, 3, 7};
118
+ cur_n_pos_unpacked = undo_pack[cur_n_pos];
119
+ } else {
120
+ constexpr int undo_pack[4] = {0, 2, 1, 3};
121
+ cur_n_pos_unpacked = undo_pack[cur_n_pos];
122
+ }
123
+
124
+ uint32_t vals[8];
125
+ #pragma unroll
126
+ for (int i = 0; i < 4; i++) {
127
+ int cur_elem = tc_row + tc_offsets[i];
128
+
129
+ int packed_src_0 = sh_stage_int_ptr[cur_n_packed + sh_stride * cur_elem];
130
+ int packed_src_1 = sh_stage_int_ptr[cur_n_packed + (8 / pack_factor) +
131
+ sh_stride * cur_elem];
132
+
133
+ vals[i] = (packed_src_0 >> (cur_n_pos_unpacked * num_bits)) & mask;
134
+ vals[4 + i] = (packed_src_1 >> (cur_n_pos_unpacked * num_bits)) & mask;
135
+ }
136
+
137
+ constexpr int tile_size = tile_k_size * tile_n_size / pack_factor;
138
+ int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
139
+
140
+ // Result of:
141
+ // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
142
+ if constexpr (num_bits == 4) {
143
+ constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
144
+
145
+ uint32_t res = 0;
146
+ #pragma unroll
147
+ for (int i = 0; i < 8; i++) {
148
+ res |= vals[pack_idx[i]] << (i * 4);
149
+ }
150
+
151
+ out_ptr[out_offset + th_id * 4 + warp_id] = res;
152
+
153
+ } else {
154
+ constexpr int pack_idx[4] = {0, 2, 1, 3};
155
+
156
+ uint32_t res1 = 0;
157
+ uint32_t res2 = 0;
158
+ #pragma unroll
159
+ for (int i = 0; i < 4; i++) {
160
+ res1 |= vals[pack_idx[i]] << (i * 8);
161
+ res2 |= vals[4 + pack_idx[i]] << (i * 8);
162
+ }
163
+
164
+ out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1;
165
+ out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2;
166
+ }
167
+ };
168
+
169
+ auto start_pipes = [&](int k_tile_id, int n_tile_id) {
170
+ #pragma unroll
171
+ for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
172
+ fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
173
+ }
174
+
175
+ wait_for_stage();
176
+ };
177
+ #pragma unroll
178
+ for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
179
+ int n_tile_id = 0;
180
+
181
+ start_pipes(k_tile_id, n_tile_id);
182
+
183
+ while (n_tile_id < n_tiles) {
184
+ #pragma unroll
185
+ for (int pipe = 0; pipe < repack_stages; pipe++) {
186
+ fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
187
+ n_tile_id + pipe + repack_stages - 1);
188
+ repack_tile(pipe, k_tile_id, n_tile_id + pipe);
189
+ wait_for_stage();
190
+ }
191
+ n_tile_id += repack_stages;
192
+ }
193
+ }
194
+}
195
+
196
+} // namespace marlin
197
+
198
+ #define CALL_IF(NUM_BITS) \
199
+ else if (num_bits == NUM_BITS) { \
200
+ cudaFuncSetAttribute( \
201
+ marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS>, \
202
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
203
+ marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS> \
204
+ <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
205
+ b_q_weight_ptr, out_ptr, size_k, size_n); \
206
+ }
207
+
208
+torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
209
+ int64_t size_n, int64_t num_bits) {
210
+ // Verify compatibility with marlin tile of 16x64
211
+ TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k,
212
+ " is not divisible by tile_k_size = ", marlin::tile_k_size);
213
+ TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n,
214
+ " is not divisible by tile_n_size = ", marlin::tile_n_size);
215
+
216
+ TORCH_CHECK(num_bits == 4 || num_bits == 8,
217
+ "num_bits must be 4 or 8. Got = ", num_bits);
218
+ int const pack_factor = 32 / num_bits;
219
+
220
+ // Verify B
221
+ TORCH_CHECK(b_q_weight.size(0) == size_k,
222
+ "b_q_weight.size(0) = ", b_q_weight.size(0),
223
+ " is not size_k = ", size_k);
224
+ TORCH_CHECK((size_n / pack_factor) == b_q_weight.size(1),
225
+ "Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1),
226
+ ", size_n = ", size_n, ", pack_factor = ", pack_factor);
227
+
228
+ // Verify device and strides
229
+ TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
230
+ TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
231
+ TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt");
232
+
233
+ // Alloc buffers
234
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight));
235
+ auto options = torch::TensorOptions()
236
+ .dtype(b_q_weight.dtype())
237
+ .device(b_q_weight.device());
238
+ torch::Tensor out = torch::empty(
239
+ {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
240
+ options);
241
+
242
+ // Get ptrs
243
+ uint32_t const* b_q_weight_ptr =
244
+ reinterpret_cast<uint32_t const*>(b_q_weight.data_ptr());
245
+ uint32_t* out_ptr = reinterpret_cast<uint32_t*>(out.data_ptr());
246
+
247
+ // Get dev info
248
+ int dev = b_q_weight.get_device();
249
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
250
+ int blocks;
251
+ cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
252
+
253
+ int max_shared_mem = 0;
254
+ cudaDeviceGetAttribute(&max_shared_mem,
255
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
256
+ TORCH_CHECK(max_shared_mem > 0);
257
+
258
+ if (false) {
259
+ }
260
+ CALL_IF(4)
261
+ CALL_IF(8)
262
+ else {
263
+ TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits);
264
+ }
265
+
266
+ return out;
267
+}
268
+
269
+#endif
csrc/quantization/gptq_marlin/gptq_marlin.cu
+425
-100
@@ -19,8 +19,8 @@
19
* Adapted from https://github.com/IST-DASLab/marlin
20
*/
21
22
-#include "gptq_marlin.cuh"
23
-#include "gptq_marlin_dtypes.cuh"
22
+#include "marlin.cuh"
23
+#include "marlin_dtypes.cuh"
24
25
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
26
static_assert(std::is_same<scalar_t, half>::value || \
@@ -32,7 +32,7 @@ inline std::string str(T x) {
32
return std::to_string(x);
33
}
34
35
-namespace gptq_marlin {
35
+namespace marlin {
36
37
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
38
@@ -72,10 +72,11 @@ __global__ void Marlin(
72
} // namespace gptq_marlin
73
74
torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
75
- torch::Tensor& b_scales, torch::Tensor& g_idx,
76
- torch::Tensor& perm, torch::Tensor& workspace,
77
- int64_t num_bits, int64_t size_m, int64_t size_n,
78
- int64_t size_k, bool is_k_full) {
75
+ torch::Tensor& b_scales, torch::Tensor& b_zeros,
76
+ torch::Tensor& g_idx, torch::Tensor& perm,
77
+ torch::Tensor& workspace, int64_t num_bits,
78
+ int64_t size_m, int64_t size_n, int64_t size_k,
79
+ bool is_k_full) {
80
TORCH_CHECK_NOT_IMPLEMENTED(false,
81
"marlin_gemm(..) requires CUDA_ARCH >= 8.0");
82
return torch::empty({1, 1});
@@ -264,6 +265,114 @@ dequant_8bit<nv_bfloat16>(int q) {
265
return frag_b;
266
}
267
268
+// Zero-point dequantizers
269
+
270
+template <typename scalar_t>
271
+__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit_zp(int q) {
272
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
273
+}
274
+
275
+template <>
276
+__device__ inline typename ScalarType<half>::FragB dequant_4bit_zp<half>(
277
+ int q) {
278
+ const int LO = 0x000f000f;
279
+ const int HI = 0x00f000f0;
280
+ const int EX = 0x64006400;
281
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
282
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
283
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
284
+
285
+ const int SUB = 0x64006400;
286
+ const int MUL = 0x2c002c00;
287
+ const int ADD = 0xd400d400;
288
+ typename ScalarType<half>::FragB frag_b;
289
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
290
+ *reinterpret_cast<const half2*>(&SUB));
291
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
292
+ *reinterpret_cast<const half2*>(&MUL),
293
+ *reinterpret_cast<const half2*>(&ADD));
294
+ return frag_b;
295
+}
296
+
297
+template <>
298
+__device__ inline typename ScalarType<nv_bfloat16>::FragB
299
+dequant_4bit_zp<nv_bfloat16>(int q) {
300
+ static constexpr uint32_t MASK = 0x000f000f;
301
+ static constexpr uint32_t EX = 0x43004300;
302
+
303
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
304
+
305
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
306
+ q >>= 4;
307
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
308
+
309
+ typename ScalarType<nv_bfloat16>::FragB frag_b;
310
+ static constexpr uint32_t MUL = 0x3F803F80;
311
+ static constexpr uint32_t ADD = 0xC300C300;
312
+
313
+ frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
314
+ *reinterpret_cast<const nv_bfloat162*>(&MUL),
315
+ *reinterpret_cast<const nv_bfloat162*>(&ADD));
316
+ frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
317
+ *reinterpret_cast<const nv_bfloat162*>(&MUL),
318
+ *reinterpret_cast<const nv_bfloat162*>(&ADD));
319
+ return frag_b;
320
+}
321
+
322
+template <typename scalar_t>
323
+__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit_zp(int q) {
324
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
325
+}
326
+
327
+template <>
328
+__device__ inline typename ScalarType<half>::FragB dequant_8bit_zp<half>(
329
+ int q) {
330
+ static constexpr uint32_t mask_for_elt_01 = 0x5250;
331
+ static constexpr uint32_t mask_for_elt_23 = 0x5351;
332
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
333
+
334
+ uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
335
+ uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
336
+
337
+ static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64006400;
338
+
339
+ typename ScalarType<half>::FragB frag_b;
340
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
341
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
342
+ frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
343
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
344
+ return frag_b;
345
+}
346
+
347
+template <>
348
+__device__ inline typename ScalarType<nv_bfloat16>::FragB
349
+dequant_8bit_zp<nv_bfloat16>(int q) {
350
+ typename ScalarType<nv_bfloat16>::FragB frag_b;
351
+
352
+ float fp32_intermediates[4];
353
+ uint32_t* fp32_intermediates_casted =
354
+ reinterpret_cast<uint32_t*>(fp32_intermediates);
355
+
356
+ static constexpr uint32_t fp32_base = 0x4B000000;
357
+ fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
358
+ fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652);
359
+ fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
360
+ fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653);
361
+
362
+ fp32_intermediates[0] -= 8388608.f;
363
+ fp32_intermediates[1] -= 8388608.f;
364
+ fp32_intermediates[2] -= 8388608.f;
365
+ fp32_intermediates[3] -= 8388608.f;
366
+
367
+ uint32_t* bf16_result_ptr = reinterpret_cast<uint32_t*>(&frag_b);
368
+ bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0],
369
+ fp32_intermediates_casted[1], 0x7632);
370
+ bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2],
371
+ fp32_intermediates_casted[3], 0x7632);
372
+
373
+ return frag_b;
374
+}
375
+
376
// Multiply dequantized values by the corresponding quantization scale; used
377
// only for grouped quantization.
378
template <typename scalar_t>
@@ -277,6 +386,17 @@ __device__ inline void scale(typename ScalarType<scalar_t>::FragB& frag_b,
386
frag_b[1] = __hmul2(frag_b[1], s);
387
}
388
389
+template <typename scalar_t>
390
+__device__ inline void sub_zp(typename ScalarType<scalar_t>::FragB& frag_b,
391
+ typename ScalarType<scalar_t>::scalar_t2& frag_zp,
392
+ int i) {
393
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
394
+ scalar_t2 zp =
395
+ ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t*>(&frag_zp)[i]);
396
+ frag_b[0] = __hsub2(frag_b[0], zp);
397
+ frag_b[1] = __hsub2(frag_b[1], zp);
398
+}
399
+
400
// Same as above, but for act_order (each K is multiplied individually)
401
template <typename scalar_t>
402
__device__ inline void scale4(typename ScalarType<scalar_t>::FragB& frag_b,
@@ -404,6 +524,7 @@ template <typename scalar_t, // compute dtype, half or nv_float16
524
const int stages, // number of stages for the async global->shared
525
// fetch pipeline
526
const bool has_act_order, // whether act_order is enabled
527
+ const bool has_zp, // whether zero-points are enabled
528
const int group_blocks = -1 // number of consecutive 16x16 blocks
529
// with a separate quantization scale
530
>
@@ -413,6 +534,8 @@ __global__ void Marlin(
534
int4* __restrict__ C, // fp16 output buffer of shape mxn
535
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
536
// (k/groupsize)xn
537
+ const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
538
+ // (k/groupsize)x(n/pack_factor)
539
const int* __restrict__ g_idx, // int32 group indices of shape k
540
int num_groups, // number of scale groups per output channel
541
int prob_m, // batch dimension m
@@ -437,6 +560,7 @@ __global__ void Marlin(
560
using FragB = typename ScalarType<scalar_t>::FragB;
561
using FragC = typename ScalarType<scalar_t>::FragC;
562
using FragS = typename ScalarType<scalar_t>::FragS;
563
+ using FragZP = typename ScalarType<scalar_t>::FragZP;
564
565
constexpr int pack_factor = 32 / num_bits;
566
@@ -566,6 +690,13 @@ __global__ void Marlin(
690
int tb_n_warps = thread_n_blocks / 4;
691
int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
692
693
+ // Zero-points sizes/strides
694
+ int zp_gl_stride = (prob_n / pack_factor) / 4;
695
+ constexpr int zp_sh_stride = ((16 * thread_n_blocks) / pack_factor) / 4;
696
+ constexpr int zp_tb_groups = s_tb_groups;
697
+ constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0;
698
+ int zp_gl_rd_delta = zp_gl_stride;
699
+
700
// Global A read index of current thread.
701
int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
702
(threadIdx.x % a_gl_rd_delta_o);
@@ -605,6 +736,19 @@ __global__ void Marlin(
736
int s_sh_wr = threadIdx.x;
737
bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
738
739
+ // Zero-points
740
+ int zp_gl_rd;
741
+ if constexpr (has_zp) {
742
+ if constexpr (group_blocks == -1) {
743
+ zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
744
+ } else {
745
+ zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
746
+ zp_sh_stride * slice_col + threadIdx.x;
747
+ }
748
+ }
749
+ int zp_sh_wr = threadIdx.x;
750
+ bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride;
751
+
752
// We use a different scale layout for grouped and column-wise quantization as
753
// we scale a `half2` tile in column-major layout in the former and in
754
// row-major in the latter case.
@@ -616,6 +760,18 @@ __global__ void Marlin(
760
s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
761
(threadIdx.x % 32) % 4;
762
763
+ // Zero-points have the same read layout as the scales
764
+ // (without column-wise case)
765
+ constexpr int num_col_threads = 8;
766
+ constexpr int num_row_threads = 4;
767
+ constexpr int num_ints_per_thread = 8 / pack_factor;
768
+ int zp_sh_rd;
769
+ if constexpr (has_zp) {
770
+ zp_sh_rd = num_ints_per_thread * num_col_threads *
771
+ ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
772
+ num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
773
+ }
774
+
775
// Precompute which thread should not read memory in which iterations; this is
776
// needed if there are more threads than required for a certain tilesize or
777
// when the batchsize is not a multiple of 16.
@@ -664,14 +820,17 @@ __global__ void Marlin(
820
int4* sh_a = sh;
821
int4* sh_b = sh_a + (stages * a_sh_stage);
822
int4* sh_g_idx = sh_b + (stages * b_sh_stage);
667
- int4* sh_s = sh_g_idx + (stages * g_idx_stage);
823
+ int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
824
+ int4* sh_s = sh_zp + (stages * zp_sh_stage);
825
826
// Register storage for double buffer of shared memory reads.
827
FragA frag_a[2][thread_m_blocks];
828
I4 frag_b_quant[2][b_thread_vecs];
829
FragC frag_c[thread_m_blocks][4][2];
673
- FragS frag_s[2][4]; // No act-order
674
- FragS act_frag_s[2][4][4]; // For act-order
830
+ FragS frag_s[2][4]; // No act-order
831
+ FragS act_frag_s[2][4][4]; // For act-order
832
+ int frag_qzp[2][num_ints_per_thread]; // Zero-points
833
+ FragZP frag_zp; // Zero-points in fp16
834
835
// Zero accumulators.
836
auto zero_accums = [&]() {
@@ -777,6 +936,28 @@ __global__ void Marlin(
936
}
937
}
938
}
939
+
940
+ if constexpr (has_zp && group_blocks != -1) {
941
+ int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
942
+
943
+ if constexpr (group_blocks >= thread_k_blocks) {
944
+ // Only fetch zero-points if this tile starts a new group
945
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
946
+ if (zp_sh_wr_pred) {
947
+ cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
948
+ }
949
+ zp_gl_rd += zp_gl_rd_delta;
950
+ }
951
+ } else {
952
+ for (int i = 0; i < zp_tb_groups; i++) {
953
+ if (zp_sh_wr_pred) {
954
+ cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr],
955
+ &zp_ptr[zp_gl_rd]);
956
+ }
957
+ zp_gl_rd += zp_gl_rd_delta;
958
+ }
959
+ }
960
+ }
961
}
962
}
963
// Insert a fence even when we are winding down the pipeline to ensure that
@@ -784,6 +965,12 @@ __global__ void Marlin(
965
cp_async_fence();
966
};
967
968
+ auto fetch_zp_to_shared = [&]() {
969
+ if (zp_sh_wr_pred) {
970
+ cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]);
971
+ }
972
+ };
973
+
974
// Wait until the next thread tile has been loaded to shared memory.
975
auto wait_for_stage = [&]() {
976
// We only have `stages - 2` active fetches since we are double buffering
@@ -932,8 +1119,73 @@ __global__ void Marlin(
1119
}
1120
};
1121
1122
+ auto fetch_zp_to_registers = [&](int k, int full_pipe) {
1123
+ if constexpr (!has_zp) {
1124
+ return;
1125
+ }
1126
+
1127
+ int pipe = full_pipe % stages;
1128
+
1129
+ if constexpr (group_blocks == -1) {
1130
+ for (int i = 0; i < num_ints_per_thread; i++) {
1131
+ frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
1132
+ }
1133
+
1134
+ } else if constexpr (group_blocks >= thread_k_blocks) {
1135
+ int4* sh_zp_stage =
1136
+ sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1137
+ (pipe / (group_blocks / thread_k_blocks)));
1138
+ for (int i = 0; i < num_ints_per_thread; i++) {
1139
+ frag_qzp[k % 2][i] =
1140
+ (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1141
+ }
1142
+ } else {
1143
+ int warp_id = threadIdx.x / 32;
1144
+ int n_warps = thread_n_blocks / 4;
1145
+
1146
+ int warp_row = warp_id / n_warps;
1147
+
1148
+ int cur_k = warp_row * 16;
1149
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
1150
+
1151
+ int k_blocks = cur_k / 16;
1152
+ int cur_group_id = k_blocks / group_blocks;
1153
+
1154
+ int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1155
+
1156
+ sh_zp_stage += cur_group_id * zp_sh_stride;
1157
+
1158
+ for (int i = 0; i < num_ints_per_thread; i++) {
1159
+ frag_qzp[k % 2][i] =
1160
+ (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1161
+ }
1162
+ }
1163
+ };
1164
+
1165
// Execute the actual tensor core matmul of a sub-tile.
1166
auto matmul = [&](int k) {
1167
+ if constexpr (has_zp) {
1168
+ FragB frag_zp_0;
1169
+ FragB frag_zp_1;
1170
+ if constexpr (num_bits == 4) {
1171
+ int zp_quant = frag_qzp[k % 2][0];
1172
+ int zp_quant_shift = zp_quant >> 8;
1173
+ frag_zp_0 = dequant_4bit_zp<scalar_t>(zp_quant);
1174
+ frag_zp_1 = dequant_4bit_zp<scalar_t>(zp_quant_shift);
1175
+
1176
+ } else {
1177
+ int zp_quant_0 = frag_qzp[k % 2][0];
1178
+ int zp_quant_1 = frag_qzp[k % 2][1];
1179
+ frag_zp_0 = dequant_8bit_zp<scalar_t>(zp_quant_0);
1180
+ frag_zp_1 = dequant_8bit_zp<scalar_t>(zp_quant_1);
1181
+ }
1182
+
1183
+ frag_zp[0] = frag_zp_0[0];
1184
+ frag_zp[1] = frag_zp_0[1];
1185
+ frag_zp[2] = frag_zp_1[0];
1186
+ frag_zp[3] = frag_zp_1[1];
1187
+ }
1188
+
1189
// We have the m dimension as the inner loop in order to encourage overlapping
1190
// dequantization and matmul operations.
1191
#pragma unroll
@@ -944,16 +1196,32 @@ __global__ void Marlin(
1196
int b_quant = frag_b_quant[k % 2][0][j];
1197
int b_quant_shift = b_quant >> 8;
1198
947
- frag_b0 = dequant_4bit<scalar_t>(b_quant);
948
- frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
1199
+ if constexpr (has_zp) {
1200
+ frag_b0 = dequant_4bit_zp<scalar_t>(b_quant);
1201
+ frag_b1 = dequant_4bit_zp<scalar_t>(b_quant_shift);
1202
+
1203
+ } else {
1204
+ frag_b0 = dequant_4bit<scalar_t>(b_quant);
1205
+ frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
1206
+ }
1207
1208
} else {
1209
int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
1210
int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
1211
int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
1212
955
- frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
956
- frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
1213
+ if constexpr (has_zp) {
1214
+ frag_b0 = dequant_8bit_zp<scalar_t>(b_quant_0);
1215
+ frag_b1 = dequant_8bit_zp<scalar_t>(b_quant_1);
1216
+ } else {
1217
+ frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
1218
+ frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
1219
+ }
1220
+ }
1221
+
1222
+ // Apply zero-point to frag_b0
1223
+ if constexpr (has_zp) {
1224
+ sub_zp<scalar_t>(frag_b0, frag_zp[j], 0);
1225
}
1226
1227
// Apply scale to frag_b0
@@ -967,6 +1235,11 @@ __global__ void Marlin(
1235
}
1236
}
1237
1238
+ // Apply zero-point to frag_b1
1239
+ if constexpr (has_zp) {
1240
+ sub_zp<scalar_t>(frag_b1, frag_zp[j], 1);
1241
+ }
1242
+
1243
// Apply scale to frag_b1
1244
if constexpr (has_act_order) {
1245
scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j],
@@ -1189,6 +1462,12 @@ __global__ void Marlin(
1462
}
1463
fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1464
}
1465
+
1466
+ if constexpr (has_zp && group_blocks == -1) {
1467
+ if (i == 0) {
1468
+ fetch_zp_to_shared();
1469
+ }
1470
+ }
1471
fetch_to_shared(i, i, i < slice_iters);
1472
}
1473
@@ -1197,6 +1476,7 @@ __global__ void Marlin(
1476
init_same_group(0);
1477
fetch_to_registers(0, 0);
1478
fetch_scales_to_registers(0, 0);
1479
+ fetch_zp_to_registers(0, 0);
1480
a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1481
slice_k_start_shared_fetch += tb_k * (stages - 1);
1482
};
@@ -1217,6 +1497,7 @@ __global__ void Marlin(
1497
for (int k = 0; k < b_sh_wr_iters; k++) {
1498
fetch_to_registers(k + 1, pipe % stages);
1499
fetch_scales_to_registers(k + 1, pipe);
1500
+ fetch_zp_to_registers(k + 1, pipe);
1501
if (k == b_sh_wr_iters - 2) {
1502
fetch_to_shared((pipe + stages - 1) % stages, pipe,
1503
slice_iters >= stages);
@@ -1354,6 +1635,7 @@ __global__ void Marlin(
1635
1636
} else {
1637
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1638
+ zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
1639
}
1640
1641
start_pipes();
@@ -1363,22 +1645,24 @@ __global__ void Marlin(
1645
}
1646
1647
#define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1366
- THREAD_K_BLOCKS, HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1648
+ THREAD_K_BLOCKS, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, \
1649
+ NUM_THREADS) \
1650
else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1651
thread_n_blocks == THREAD_N_BLOCKS && \
1652
thread_k_blocks == THREAD_K_BLOCKS && \
1370
- has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1371
- num_threads == NUM_THREADS) { \
1653
+ has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1654
+ group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1655
cudaFuncSetAttribute( \
1656
Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1657
THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1375
- GROUP_BLOCKS>, \
1658
+ HAS_ZP, GROUP_BLOCKS>, \
1659
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1660
Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1661
THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1379
- GROUP_BLOCKS><<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1380
- A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1381
- prob_k, locks); \
1662
+ HAS_ZP, GROUP_BLOCKS> \
1663
+ <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1664
+ A_ptr, B_ptr, C_ptr, s_ptr, zp_ptr, g_idx_ptr, num_groups, \
1665
+ prob_m, prob_n, prob_k, locks); \
1666
}
1667
1668
typedef struct {
@@ -1548,39 +1832,61 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1832
return exec_config_t{0, {-1, -1, -1}};
1833
}
1834
1551
- #define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1552
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1553
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1554
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1555
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1556
- \
1557
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1558
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1559
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1560
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1561
- \
1562
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1563
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1564
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1565
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1566
- \
1567
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1568
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1569
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1570
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1571
- \
1572
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1573
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1574
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1575
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1835
+ #define GPTQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1836
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1837
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1838
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1839
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1840
+ \
1841
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1842
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1843
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1844
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1845
+ \
1846
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1847
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1848
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1849
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1850
+ \
1851
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1852
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1853
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1854
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1855
+ \
1856
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1857
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1858
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1859
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1860
+
1861
+ #define AWQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1862
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1863
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1864
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1865
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1866
+ \
1867
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1868
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1869
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1870
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1871
+ \
1872
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1873
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1874
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1875
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1876
+ \
1877
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1878
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1879
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1880
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1881
1882
template <typename scalar_t>
1578
-void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
1883
+void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, void* zp,
1884
void* g_idx, void* perm, void* a_tmp, int prob_m,
1885
int prob_n, int prob_k, void* workspace, int num_bits,
1581
- bool has_act_order, bool is_k_full, int num_groups,
1582
- int group_size, int dev, cudaStream_t stream, int thread_k,
1583
- int thread_n, int sms, int max_par) {
1886
+ bool has_act_order, bool is_k_full, bool has_zp,
1887
+ int num_groups, int group_size, int dev,
1888
+ cudaStream_t stream, int thread_k, int thread_n, int sms,
1889
+ int max_par) {
1890
TORCH_CHECK(num_bits == 4 || num_bits == 8,
1891
"num_bits must be 4 or 8. Got = ", num_bits);
1892
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
@@ -1665,6 +1971,7 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
1971
const int4* B_ptr = (const int4*)B;
1972
int4* C_ptr = (int4*)C;
1973
const int4* s_ptr = (const int4*)s;
1974
+ const int4* zp_ptr = (const int4*)zp;
1975
const int* g_idx_ptr = (const int*)g_idx;
1976
const int* perm_ptr = (const int*)perm;
1977
int4* a_tmp_ptr = (int4*)a_tmp;
@@ -1701,28 +2008,33 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
2008
thread_m_blocks = exec_cfg.max_m_blocks;
2009
}
2010
1704
- // Define kernel configurations
2011
if (false) {
2012
}
1707
- CALL_IF(4, 32, 2, 256)
1708
- CALL_IF(4, 16, 4, 256)
1709
- CALL_IF(4, 8, 8, 256)
1710
- CALL_IF(4, 8, 4, 128)
1711
- CALL_IF(4, 4, 8, 128)
1712
- CALL_IF(8, 32, 2, 256)
1713
- CALL_IF(8, 16, 4, 256)
1714
- CALL_IF(8, 8, 8, 256)
1715
- CALL_IF(8, 8, 4, 128)
1716
- CALL_IF(8, 4, 8, 128)
2013
+ GPTQ_CALL_IF(4, 16, 4, 256)
2014
+ GPTQ_CALL_IF(4, 8, 8, 256)
2015
+ GPTQ_CALL_IF(4, 8, 4, 128)
2016
+ GPTQ_CALL_IF(4, 4, 8, 128)
2017
+ GPTQ_CALL_IF(8, 16, 4, 256)
2018
+ GPTQ_CALL_IF(8, 8, 8, 256)
2019
+ GPTQ_CALL_IF(8, 8, 4, 128)
2020
+ GPTQ_CALL_IF(8, 4, 8, 128)
2021
+
2022
+ AWQ_CALL_IF(4, 16, 4, 256)
2023
+ AWQ_CALL_IF(4, 8, 8, 256)
2024
+ AWQ_CALL_IF(4, 8, 4, 128)
2025
+ AWQ_CALL_IF(4, 4, 8, 128)
2026
+ AWQ_CALL_IF(8, 16, 4, 256)
2027
+ AWQ_CALL_IF(8, 8, 8, 256)
2028
+ AWQ_CALL_IF(8, 8, 4, 128)
2029
+ AWQ_CALL_IF(8, 4, 8, 128)
2030
else {
1718
- TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1719
- str(prob_n) + ", " + str(prob_k) + "]" +
1720
- ", has_act_order = " + str(has_act_order) +
1721
- ", num_groups = " + str(num_groups) +
1722
- ", group_size = " + str(group_size) +
1723
- ", thread_m_blocks = " + str(thread_m_blocks) +
1724
- ", thread_n_blocks = " + str(thread_n_blocks) +
1725
- ", thread_k_blocks = " + str(thread_k_blocks));
2031
+ TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
2032
+ ", ", prob_k, "]", ", has_act_order = ", has_act_order,
2033
+ ", num_groups = ", num_groups, ", group_size = ", group_size,
2034
+ ", thread_m_blocks = ", thread_m_blocks,
2035
+ ", thread_n_blocks = ", thread_n_blocks,
2036
+ ", thread_k_blocks = ", thread_k_blocks,
2037
+ ", num_bits = ", num_bits);
2038
}
2039
2040
A_ptr += 16 * thread_m_blocks * (prob_k / 8) * par;
@@ -1733,10 +2045,11 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
2045
} // namespace gptq_marlin
2046
2047
torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1736
- torch::Tensor& b_scales, torch::Tensor& g_idx,
1737
- torch::Tensor& perm, torch::Tensor& workspace,
1738
- int64_t num_bits, int64_t size_m, int64_t size_n,
1739
- int64_t size_k, bool is_k_full) {
2048
+ torch::Tensor& b_scales, torch::Tensor& b_zeros,
2049
+ torch::Tensor& g_idx, torch::Tensor& perm,
2050
+ torch::Tensor& workspace, int64_t num_bits,
2051
+ int64_t size_m, int64_t size_n, int64_t size_k,
2052
+ bool is_k_full, bool has_zp) {
2053
// Verify num_bits
2054
TORCH_CHECK(num_bits == 4 || num_bits == 8,
2055
"num_bits must be 4 or 8. Got = ", num_bits);
@@ -1749,16 +2062,15 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2062
", size_k = ", size_k);
2063
2064
// Verify B
1752
- TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1753
- " is not divisible by tile_size = ", gptq_marlin::tile_size);
1754
- TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
2065
+ TORCH_CHECK(size_k % marlin::tile_size == 0, "size_k = ", size_k,
2066
+ " is not divisible by tile_size = ", marlin::tile_size);
2067
+ TORCH_CHECK((size_k / marlin::tile_size) == b_q_weight.size(0),
2068
"Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1756
- ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1757
- TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
2069
+ ", size_k = ", size_k, ", tile_size = ", marlin::tile_size);
2070
+ TORCH_CHECK(b_q_weight.size(1) % marlin::tile_size == 0,
2071
"b_q_weight.size(1) = ", b_q_weight.size(1),
1759
- " is not divisible by tile_size = ", gptq_marlin::tile_size);
1760
- int actual_size_n =
1761
- (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
2072
+ " is not divisible by tile_size = ", marlin::tile_size);
2073
+ int actual_size_n = (b_q_weight.size(1) / marlin::tile_size) * pack_factor;
2074
TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
2075
", actual_size_n = ", actual_size_n);
2076
@@ -1772,6 +2084,9 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2084
TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
2085
TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
2086
2087
+ TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU");
2088
+ TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous");
2089
+
2090
TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
2091
TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
2092
@@ -1805,8 +2120,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2120
int group_size = -1;
2121
bool has_act_order = g_idx.size(0) != 0;
2122
1808
- int b_rank = b_scales.sizes().size();
1809
- TORCH_CHECK(b_rank == 2, "b_scales rank = ", b_rank, " is not 2");
2123
+ int rank = b_scales.sizes().size();
2124
+ TORCH_CHECK(rank == 2, "b_scales rank = ", rank, " is not 2");
2125
TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1),
2126
" is not size_n = ", size_n);
2127
num_groups = b_scales.size(0);
@@ -1832,34 +2147,44 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2147
}
2148
}
2149
2150
+ // Verify b_zeros
2151
+ if (has_zp) {
2152
+ int rank = b_zeros.sizes().size();
2153
+ TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2");
2154
+ TORCH_CHECK(b_zeros.size(0) == num_groups,
2155
+ "b_zeros dim 0 = ", b_zeros.size(0),
2156
+ " is not num_groups = ", num_groups);
2157
+ TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2158
+ "b_zeros dim 1 = ", b_scales.size(1),
2159
+ " is not size_n / pack_factor = ", size_n / pack_factor);
2160
+ }
2161
+
2162
// Verify workspace size
1836
- TORCH_CHECK(
1837
- size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1838
- ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1839
- int min_workspace_size =
1840
- (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
2163
+ TORCH_CHECK(size_n % marlin::min_thread_n == 0, "size_n = ", size_n,
2164
+ ", is not divisible by min_thread_n = ", marlin::min_thread_n);
2165
+ int min_workspace_size = (size_n / marlin::min_thread_n) * marlin::max_par;
2166
TORCH_CHECK(workspace.numel() >= min_workspace_size,
2167
"workspace.numel = ", workspace.numel(),
2168
" is below min_workspace_size = ", min_workspace_size);
2169
2170
int dev = a.get_device();
2171
if (a.scalar_type() == at::ScalarType::Half) {
1847
- gptq_marlin::marlin_mm_f16i4<half>(
2172
+ marlin::marlin_mm_f16i4<half>(
2173
a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
1849
- b_scales.data_ptr<at::Half>(), g_idx.data_ptr(), perm.data_ptr(),
1850
- a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
1851
- workspace.data_ptr(), num_bits, has_act_order, is_k_full, num_groups,
1852
- group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1853
- thread_n, sms, gptq_marlin::max_par);
2174
+ b_scales.data_ptr<at::Half>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2175
+ perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2176
+ workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2177
+ num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2178
+ thread_k, thread_n, sms, marlin::max_par);
2179
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
1855
- gptq_marlin::marlin_mm_f16i4<nv_bfloat16>(
2180
+ marlin::marlin_mm_f16i4<nv_bfloat16>(
2181
a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
2182
c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1858
- g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(),
1859
- size_m, size_n, size_k, workspace.data_ptr(), num_bits, has_act_order,
1860
- is_k_full, num_groups, group_size, dev,
1861
- at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1862
- gptq_marlin::max_par);
2183
+ b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2184
+ a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2185
+ workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2186
+ num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2187
+ thread_k, thread_n, sms, marlin::max_par);
2188
} else {
2189
TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
2190
}
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+27
-33
@@ -1,23 +1,16 @@
1
-#include "gptq_marlin.cuh"
2
-
3
-namespace gptq_marlin {
4
-
5
-static constexpr int repack_stages = 8;
6
-
7
-static constexpr int repack_threads = 256;
8
-
9
-static constexpr int tile_k_size = tile_size;
10
-static constexpr int tile_n_size = tile_k_size * 4;
1
+#include "marlin.cuh"
2
3
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4
5
+namespace marlin {
6
+
7
template <int const num_threads, int const num_bits, bool const has_perm>
15
-__global__ void marlin_repack_kernel(
8
+__global__ void gptq_marlin_repack_kernel(
9
uint32_t const* __restrict__ b_q_weight_ptr,
10
uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
11
int size_k, int size_n) {}
12
20
-} // namespace gptq_marlin
13
+} // namespace marlin
14
15
torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
16
int64_t size_k, int64_t size_n,
@@ -29,8 +22,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
22
23
#else
24
25
+namespace marlin {
26
+
27
template <int const num_threads, int const num_bits, bool const has_perm>
33
-__global__ void marlin_repack_kernel(
28
+__global__ void gptq_marlin_repack_kernel(
29
uint32_t const* __restrict__ b_q_weight_ptr,
30
uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
31
int size_k, int size_n) {
@@ -259,28 +254,28 @@ __global__ void marlin_repack_kernel(
254
}
255
}
256
262
-} // namespace gptq_marlin
263
-
264
- #define CALL_IF(NUM_BITS, HAS_PERM) \
265
- else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
266
- cudaFuncSetAttribute( \
267
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
268
- NUM_BITS, HAS_PERM>, \
269
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
270
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
271
- HAS_PERM> \
272
- <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
273
- b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
257
+} // namespace marlin
258
+
259
+ #define CALL_IF(NUM_BITS, HAS_PERM) \
260
+ else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
261
+ cudaFuncSetAttribute( \
262
+ marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
263
+ HAS_PERM>, \
264
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
265
+ marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
266
+ HAS_PERM> \
267
+ <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
268
+ b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
269
}
270
271
torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
272
int64_t size_k, int64_t size_n,
273
int64_t num_bits) {
274
// Verify compatibility with marlin tile of 16x64
280
- TORCH_CHECK(size_k % gptq_marlin::tile_k_size == 0, "size_k = ", size_k,
281
- " is not divisible by tile_k_size = ", gptq_marlin::tile_k_size);
282
- TORCH_CHECK(size_n % gptq_marlin::tile_n_size == 0, "size_n = ", size_n,
283
- " is not divisible by tile_n_size = ", gptq_marlin::tile_n_size);
275
+ TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k,
276
+ " is not divisible by tile_k_size = ", marlin::tile_k_size);
277
+ TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n,
278
+ " is not divisible by tile_n_size = ", marlin::tile_n_size);
279
280
TORCH_CHECK(num_bits == 4 || num_bits == 8,
281
"num_bits must be 4 or 8. Got = ", num_bits);
@@ -308,10 +303,9 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
303
auto options = torch::TensorOptions()
304
.dtype(b_q_weight.dtype())
305
.device(b_q_weight.device());
311
- torch::Tensor out =
312
- torch::empty({size_k / gptq_marlin::tile_size,
313
- size_n * gptq_marlin::tile_size / pack_factor},
314
- options);
306
+ torch::Tensor out = torch::empty(
307
+ {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
308
+ options);
309
310
// Detect if there is act_order
311
bool has_perm = perm.size(0) != 0;
csrc/quantization/gptq_marlin/marlin.cuh
renamed
+13
-2
@@ -9,7 +9,9 @@
9
#include <cuda_runtime.h>
10
#include <iostream>
11
12
-namespace gptq_marlin {
12
+namespace marlin {
13
+
14
+// Marlin params
15
16
// 8 warps are a good choice since every SM has 4 schedulers and having more
17
// than 1 warp per schedule allows some more latency hiding. At the same time,
@@ -25,6 +27,15 @@ static constexpr int min_thread_k = 64;
27
static constexpr int tile_size = 16;
28
static constexpr int max_par = 16;
29
30
+// Repack params
31
+static constexpr int repack_stages = 8;
32
+
33
+static constexpr int repack_threads = 256;
34
+
35
+static constexpr int tile_k_size = tile_size;
36
+static constexpr int tile_n_size = tile_k_size * 4;
37
+
38
+// Helpers
39
template <typename T, int n>
40
struct Vec {
41
T elems[n];
@@ -73,4 +84,4 @@ __device__ inline void cp_async_wait() {
84
85
#endif
86
76
-} // namespace gptq_marlin
87
+} // namespace marlin
csrc/quantization/gptq_marlin/marlin_dtypes.cuh
renamed
+5
-3
@@ -1,11 +1,11 @@
1
2
#ifndef _data_types_cuh
3
#define _data_types_cuh
4
-#include "gptq_marlin.cuh"
4
+#include "marlin.cuh"
5
#include <cuda_fp16.h>
6
#include <cuda_bf16.h>
7
8
-namespace gptq_marlin {
8
+namespace marlin {
9
10
template <typename scalar_t>
11
class ScalarType {};
@@ -23,6 +23,7 @@ class ScalarType<half> {
23
using FragB = Vec<half2, 2>;
24
using FragC = Vec<float, 4>;
25
using FragS = Vec<half2, 1>;
26
+ using FragZP = Vec<half2, 4>;
27
28
static __device__ float inline num2float(const half x) {
29
return __half2float(x);
@@ -51,6 +52,7 @@ class ScalarType<nv_bfloat16> {
52
using FragB = Vec<nv_bfloat162, 2>;
53
using FragC = Vec<float, 4>;
54
using FragS = Vec<nv_bfloat162, 1>;
55
+ using FragZP = Vec<nv_bfloat162, 4>;
56
57
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
58
static __device__ float inline num2float(const nv_bfloat16 x) {
@@ -72,6 +74,6 @@ class ScalarType<nv_bfloat16> {
74
#endif
75
};
76
75
-} // namespace gptq_marlin
77
+} // namespace marlin
78
79
#endif
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+24
-22
@@ -30,7 +30,7 @@ inline std::string str(T x) {
30
return std::to_string(x);
31
}
32
33
-namespace marlin {
33
+namespace marlin_dense {
34
35
constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
36
@@ -1040,7 +1040,7 @@ void marlin_cuda(const void* A, const void* B, void* C, void* s, int prob_m,
1040
}
1041
}
1042
1043
-} // namespace marlin
1043
+} // namespace marlin_dense
1044
1045
torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1046
torch::Tensor& b_scales, torch::Tensor& workspace,
@@ -1054,24 +1054,25 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1054
TORCH_CHECK(size_k == a.size(1),
1055
"Shape mismatch: a.size(1) = " + str(a.size(1)) +
1056
", size_k = " + str(size_k));
1057
- TORCH_CHECK(size_k % marlin::tile_size == 0,
1058
- "size_k = " + str(size_k) +
1059
- " is not divisible by tile_size = " + str(marlin::tile_size));
1060
- TORCH_CHECK((size_k / marlin::tile_size) == b_q_weight.size(0),
1057
+ TORCH_CHECK(size_k % marlin_dense::tile_size == 0,
1058
+ "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1059
+ str(marlin_dense::tile_size));
1060
+ TORCH_CHECK((size_k / marlin_dense::tile_size) == b_q_weight.size(0),
1061
"Shape mismatch: b_q_weight.size(0) = " +
1062
str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1063
- ", tile_size = " + str(marlin::tile_size));
1063
+ ", tile_size = " + str(marlin_dense::tile_size));
1064
1065
// Verify N
1066
TORCH_CHECK(b_scales.size(1) == size_n,
1067
"b_scales.size(1) = " + str(b_scales.size(1)) +
1068
", size_n = " + str(size_n));
1069
- TORCH_CHECK(b_q_weight.size(1) % marlin::tile_size == 0,
1070
- "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1071
- " is not divisible by tile_size = " + str(marlin::tile_size));
1069
+ TORCH_CHECK(
1070
+ b_q_weight.size(1) % marlin_dense::tile_size == 0,
1071
+ "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1072
+ " is not divisible by tile_size = " + str(marlin_dense::tile_size));
1073
1073
- int actual_size_n =
1074
- (b_q_weight.size(1) / marlin::tile_size) * marlin::pack_factor_4bit;
1074
+ int actual_size_n = (b_q_weight.size(1) / marlin_dense::tile_size) *
1075
+ marlin_dense::pack_factor_4bit;
1076
TORCH_CHECK(
1077
size_n == actual_size_n,
1078
"size_n = " + str(size_n) + ", actual_size_n = " + str(actual_size_n));
@@ -1116,21 +1117,22 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1117
"Unexpected groupsize = " + str(groupsize));
1118
1119
// Verify workspace size
1119
- TORCH_CHECK(
1120
- size_n % marlin::min_thread_n == 0,
1121
- "size_n = " + str(size_n) +
1122
- ", is not divisible by min_thread_n = " + str(marlin::min_thread_n));
1123
- int min_workspace_size = (size_n / marlin::min_thread_n) * marlin::max_par;
1120
+ TORCH_CHECK(size_n % marlin_dense::min_thread_n == 0,
1121
+ "size_n = " + str(size_n) +
1122
+ ", is not divisible by min_thread_n = " +
1123
+ str(marlin_dense::min_thread_n));
1124
+ int min_workspace_size =
1125
+ (size_n / marlin_dense::min_thread_n) * marlin_dense::max_par;
1126
TORCH_CHECK(workspace.numel() >= min_workspace_size,
1127
"workspace.numel = " + str(workspace.numel()) +
1128
" is below min_workspace_size = " + str(min_workspace_size));
1129
1130
int dev = a.get_device();
1129
- marlin::marlin_cuda(a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(),
1130
- b_scales.data_ptr(), size_m, size_n, size_k,
1131
- workspace.data_ptr(), groupsize, dev,
1132
- at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n,
1133
- sms, marlin::max_par);
1131
+ marlin_dense::marlin_cuda(a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(),
1132
+ b_scales.data_ptr(), size_m, size_n, size_k,
1133
+ workspace.data_ptr(), groupsize, dev,
1134
+ at::cuda::getCurrentCUDAStream(dev), thread_k,
1135
+ thread_n, sms, marlin_dense::max_par);
1136
1137
return c;
1138
}
csrc/torch_bindings.cpp
+4
@@ -141,6 +141,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
141
ops.def("gptq_marlin_repack", &gptq_marlin_repack);
142
ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
143
144
+ // awq_marlin repack from AWQ.
145
+ ops.def("awq_marlin_repack", &awq_marlin_repack);
146
+ ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
147
+
148
// fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
149
ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
150
ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
tests/kernels/test_marlin_gemm.py
+129
-12
@@ -12,16 +12,18 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
12
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
13
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
14
GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
15
- GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
16
- marlin_permute_scales)
15
+ MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS,
16
+ marlin_make_empty_g_idx, marlin_permute_scales)
17
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
18
pack_fp8_to_int32)
19
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20
- MarlinWorkspace, get_weight_perm, marlin_quantize, marlin_weights)
20
+ MarlinWorkspace, awq_marlin_quantize, get_weight_perm, marlin_quantize,
21
+ marlin_weights)
22
from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
23
marlin_24_quantize)
24
from vllm.model_executor.layers.quantization.utils.quant_utils import (
24
- gptq_pack, quantize_weights, sort_weights)
25
+ awq_pack, gptq_pack, quantize_weights, quantize_weights_with_zp,
26
+ sort_weights)
27
28
ACT_ORDER_OPTS = [False, True]
29
K_FULL_OPTS = [False, True]
@@ -57,12 +59,12 @@ def rand_data(shape, dtype=torch.float16):
59
reason="Marlin is not supported on this GPU type.")
60
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
61
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
60
-@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
61
-@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
62
+@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
63
+@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
64
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
65
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
64
-def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
65
- mnk_factors):
66
+def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
67
+ mnk_factors):
68
m_factor, n_factor, k_factor = mnk_factors
69
70
size_m = m_factor
@@ -120,12 +122,60 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
122
reason="Marlin is not supported on this GPU type.")
123
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
124
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
123
-@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
124
-@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
125
+@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
126
+@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
127
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
128
+def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
129
+ mnk_factors):
130
+ m_factor, n_factor, k_factor = mnk_factors
131
+
132
+ size_m = m_factor
133
+ size_k = k_chunk * k_factor
134
+ size_n = n_chunk * n_factor
135
+
136
+ print(f"MNK = {size_m} {size_n} {size_k}")
137
+
138
+ # Normalize group_size
139
+ if group_size == -1:
140
+ group_size = size_k
141
+ assert group_size <= size_k
142
+
143
+ # Create input
144
+ b_weight = rand_data((size_k, size_n))
145
+
146
+ # Quantize
147
+ w_ref, q_w, s, zp = quantize_weights_with_zp(b_weight, num_bits,
148
+ group_size)
149
+
150
+ # Pack to AWQ format
151
+ q_w_awq = awq_pack(q_w, num_bits, size_k, size_n)
152
+
153
+ # Pack to Marlin format
154
+ weight_perm = get_weight_perm(num_bits)
155
+ marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
156
+
157
+ # Run Marlin repack GPU kernel
158
+ marlin_q_w_2 = ops.awq_marlin_repack(
159
+ q_w_awq,
160
+ size_k,
161
+ size_n,
162
+ num_bits,
163
+ )
164
+ torch.cuda.synchronize()
165
+
166
+ assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
167
+
168
+
169
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
170
+ reason="Marlin is not supported on this GPU type.")
171
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
172
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
173
+@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
174
+@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
175
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
176
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
177
@pytest.mark.parametrize("is_k_full", K_FULL_OPTS)
128
-def test_marlin_gemm(
178
+def test_gptq_marlin_gemm(
179
k_chunk,
180
n_chunk,
181
num_bits,
@@ -155,6 +205,8 @@ def test_marlin_gemm(
205
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
206
b_weight, num_bits, group_size, act_order)
207
208
+ marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
209
+
210
workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
211
GPTQ_MARLIN_MAX_PARALLEL)
212
@@ -162,6 +214,7 @@ def test_marlin_gemm(
214
a_input,
215
marlin_q_w,
216
marlin_s,
217
+ marlin_zp,
218
g_idx,
219
sort_indices,
220
workspace.scratch,
@@ -170,6 +223,7 @@ def test_marlin_gemm(
223
b_weight.shape[1],
224
a_input.shape[1],
225
is_k_full,
226
+ has_zp=False,
227
)
228
output_ref = torch.matmul(a_input, w_ref)
229
@@ -188,7 +242,8 @@ def test_marlin_gemm(
242
@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
243
@pytest.mark.parametrize("group_size", GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES)
244
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
191
-def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
245
+def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
246
+ mnk_factors):
247
m_factor, n_factor, k_factor = mnk_factors
248
249
size_m = m_factor
@@ -301,3 +356,65 @@ def test_fp8_marlin_gemm(
356
print("max_diff = {}".format(max_diff))
357
358
assert max_diff < 0.04
359
+
360
+
361
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
362
+ reason="Marlin is not supported on this GPU type.")
363
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
364
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
365
+@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
366
+@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
367
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
368
+def test_awq_marlin_gemm(
369
+ k_chunk,
370
+ n_chunk,
371
+ num_bits,
372
+ group_size,
373
+ mnk_factors,
374
+):
375
+ m_factor, n_factor, k_factor = mnk_factors
376
+
377
+ size_m = m_factor
378
+ size_k = k_chunk * k_factor
379
+ size_n = n_chunk * n_factor
380
+
381
+ print(f"MNK = {size_m} {size_n} {size_k}")
382
+ print(f"groupsize = {group_size}")
383
+
384
+ a_input = rand_data((size_m, size_k))
385
+ b_weight = rand_data((size_k, size_n))
386
+
387
+ w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
388
+ b_weight, num_bits, group_size)
389
+
390
+ g_idx = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
391
+ sort_indices = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
392
+ is_k_full = True
393
+ has_zp = True
394
+
395
+ workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
396
+ GPTQ_MARLIN_MAX_PARALLEL)
397
+
398
+ output = ops.gptq_marlin_gemm(
399
+ a_input,
400
+ marlin_q_w,
401
+ marlin_s,
402
+ marlin_zp,
403
+ g_idx,
404
+ sort_indices,
405
+ workspace.scratch,
406
+ num_bits,
407
+ a_input.shape[0],
408
+ b_weight.shape[1],
409
+ a_input.shape[1],
410
+ is_k_full,
411
+ has_zp,
412
+ )
413
+ output_ref = torch.matmul(a_input, w_ref)
414
+
415
+ torch.cuda.synchronize()
416
+
417
+ max_diff = compute_max_diff(output, output_ref)
418
+ print("max_diff = {}".format(max_diff))
419
+
420
+ assert max_diff < 0.04
tests/quantization/test_configs.py
+2
-2
@@ -44,9 +44,9 @@ MODEL_ARG_EXPTYPES = [
44
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
45
46
# AUTOAWQ
47
- ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, "awq"),
47
+ ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, "awq_marlin"),
48
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
49
- ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", "ERROR"),
49
+ ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", "awq_marlin"),
50
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
51
]
52
vllm/_custom_ops.py
+15
-7
@@ -276,14 +276,22 @@ def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
276
num_bits)
277
278
279
+# gptq_marlin
280
+def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
281
+ num_bits: int) -> torch.Tensor:
282
+ return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
283
+
284
+
285
def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
280
- b_scales: torch.Tensor, g_idx: torch.Tensor,
281
- perm: torch.Tensor, workspace: torch.Tensor,
282
- num_bits: int, size_m: int, size_n: int, size_k: int,
283
- is_k_full: bool) -> torch.Tensor:
284
- return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
285
- workspace, num_bits, size_m, size_n,
286
- size_k, is_k_full)
286
+ b_scales: torch.Tensor, b_zeros: torch.Tensor,
287
+ g_idx: torch.Tensor, perm: torch.Tensor,
288
+ workspace: torch.Tensor, num_bits: int, size_m: int,
289
+ size_n: int, size_k: int, is_k_full: bool,
290
+ has_zp: bool) -> torch.Tensor:
291
+ return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
292
+ g_idx, perm, workspace, num_bits,
293
+ size_m, size_n, size_k, is_k_full,
294
+ has_zp)
295
296
297
# fp8 marlin
vllm/config.py
+1
-1
@@ -251,7 +251,7 @@ class ModelConfig:
251
f"supported in ROCm.")
252
if (self.quantization
253
not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
254
- "fbgemm_fp8", "compressed_tensors")):
254
+ "awq_marlin", "fbgemm_fp8", "compressed_tensors")):
255
logger.warning(
256
"%s quantization is not fully "
257
"optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -2,6 +2,7 @@ from typing import Dict, Type
2
3
from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
+from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
6
from vllm.model_executor.layers.quantization.base_config import (
7
QuantizationConfig)
8
from vllm.model_executor.layers.quantization.bitsandbytes import (
@@ -31,6 +32,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
32
"marlin": MarlinConfig,
33
"gptq_marlin_24": GPTQMarlin24Config,
34
"gptq_marlin": GPTQMarlinConfig,
35
+ "awq_marlin": AWQMarlinConfig,
36
"gptq": GPTQConfig,
37
"squeezellm": SqueezeLLMConfig,
38
"compressed-tensors": CompressedTensorsConfig,
vllm/model_executor/layers/quantization/awq_marlin.py
new
+268
@@ -0,0 +1,268 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
+ set_weight_attrs)
10
+from vllm.model_executor.layers.quantization.base_config import (
11
+ QuantizationConfig)
12
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13
+ apply_awq_marlin_linear, awq_to_marlin_zero_points,
14
+ check_awq_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
15
+ marlin_permute_scales, replace_tensor, verify_awq_marlin_supported,
16
+ verify_marlin_supports_shape)
17
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
+
19
+logger = init_logger(__name__)
20
+
21
+
22
+class AWQMarlinConfig(QuantizationConfig):
23
+ """Config class for AWQ Marlin"""
24
+
25
+ def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
26
+ lm_head_quantized: bool) -> None:
27
+ self.weight_bits = weight_bits
28
+ self.pack_factor = 32 // self.weight_bits # packed into int32
29
+ self.group_size = group_size
30
+ self.has_zp = has_zp
31
+ self.lm_head_quantized = lm_head_quantized
32
+
33
+ verify_awq_marlin_supported(num_bits=self.weight_bits,
34
+ group_size=self.group_size,
35
+ has_zp=self.has_zp)
36
+
37
+ def __repr__(self) -> str:
38
+ return (f"AWQMarlinConfig(weight_bits={self.weight_bits}, "
39
+ f"group_size={self.group_size}, "
40
+ f"has_zp={self.has_zp}, "
41
+ f"lm_head_quantized={self.lm_head_quantized})")
42
+
43
+ @classmethod
44
+ def get_name(cls) -> str:
45
+ return "awq_marlin"
46
+
47
+ @classmethod
48
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
49
+ return [torch.half, torch.bfloat16]
50
+
51
+ @classmethod
52
+ def get_min_capability(cls) -> int:
53
+ return 80
54
+
55
+ @classmethod
56
+ def get_config_filenames(cls) -> List[str]:
57
+ return ["quantize_config.json"]
58
+
59
+ @classmethod
60
+ def from_config(cls, config: Dict[str, Any]) -> "AWQMarlinConfig":
61
+ weight_bits = cls.get_from_keys(config, ["bits"])
62
+ group_size = cls.get_from_keys(config, ["group_size"])
63
+ has_zp = cls.get_from_keys(config, ["zero_point"])
64
+ lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
65
+ default=False)
66
+ return cls(weight_bits, group_size, has_zp, lm_head_quantized)
67
+
68
+ @classmethod
69
+ def override_quantization_method(cls, hf_quant_cfg,
70
+ user_quant) -> Optional[str]:
71
+ can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
72
+ is_valid_user_quant = (user_quant is None or user_quant == "marlin")
73
+
74
+ if can_convert and is_valid_user_quant:
75
+ msg = ("The model is convertible to {} during runtime."
76
+ " Using {} kernel.".format(cls.get_name(), cls.get_name()))
77
+ logger.info(msg)
78
+ return cls.get_name()
79
+
80
+ if can_convert and user_quant == "awq":
81
+ logger.info("Detected that the model can run with awq_marlin"
82
+ ", however you specified quantization=awq explicitly,"
83
+ " so forcing awq. Use quantization=awq_marlin for"
84
+ " faster inference")
85
+ return None
86
+
87
+ def get_quant_method(self, layer: torch.nn.Module,
88
+ prefix: str) -> Optional["AWQMarlinLinearMethod"]:
89
+ if (isinstance(layer, LinearBase) or
90
+ (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
91
+ return AWQMarlinLinearMethod(self)
92
+ return None
93
+
94
+ def get_scaled_act_names(self) -> List[str]:
95
+ return []
96
+
97
+ @classmethod
98
+ def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]):
99
+ # Extract data from quant config.
100
+ quant_method = quant_config.get("quant_method", "").lower()
101
+ num_bits = quant_config.get("bits", None)
102
+ group_size = quant_config.get("group_size", None)
103
+ has_zp = quant_config.get("zero_point", None)
104
+
105
+ if quant_method != "awq":
106
+ return False
107
+
108
+ # If we cannot find the info needed in the config, cannot convert.
109
+ if (num_bits is None or group_size is None or has_zp is None):
110
+ return False
111
+
112
+ return check_awq_marlin_supported(
113
+ num_bits=num_bits,
114
+ group_size=group_size,
115
+ has_zp=has_zp,
116
+ min_capability=cls.get_min_capability())
117
+
118
+
119
+class AWQMarlinLinearMethod(LinearMethodBase):
120
+ """Linear method for AWQ Marlin.
121
+
122
+ Args:
123
+ quant_config: The AWQ Marlin quantization config.
124
+ """
125
+
126
+ def __init__(self, quant_config: AWQMarlinConfig) -> None:
127
+ self.quant_config = quant_config
128
+
129
+ def create_weights(
130
+ self,
131
+ layer: torch.nn.Module,
132
+ input_size_per_partition: int,
133
+ output_partition_sizes: List[int],
134
+ input_size: int,
135
+ output_size: int,
136
+ params_dtype: torch.dtype,
137
+ **extra_weight_attrs,
138
+ ) -> None:
139
+ del output_size
140
+ output_size_per_partition = sum(output_partition_sizes)
141
+
142
+ # Normalize group_size
143
+ if self.quant_config.group_size != -1:
144
+ group_size = self.quant_config.group_size
145
+ else:
146
+ group_size = input_size
147
+
148
+ verify_marlin_supports_shape(
149
+ output_size_per_partition=output_size_per_partition,
150
+ input_size_per_partition=input_size_per_partition,
151
+ input_size=input_size,
152
+ group_size=group_size)
153
+
154
+ qweight = Parameter(
155
+ torch.empty(
156
+ input_size_per_partition,
157
+ output_size_per_partition // self.quant_config.pack_factor,
158
+ dtype=torch.int32,
159
+ ),
160
+ requires_grad=False,
161
+ )
162
+ set_weight_attrs(
163
+ qweight, {
164
+ "input_dim": 0,
165
+ "output_dim": 1,
166
+ "packed_dim": 1,
167
+ "pack_factor": self.quant_config.pack_factor,
168
+ })
169
+
170
+ num_groups = input_size_per_partition // group_size
171
+
172
+ qzeros = Parameter(
173
+ torch.empty(
174
+ num_groups,
175
+ output_size_per_partition // self.quant_config.pack_factor,
176
+ dtype=torch.int32,
177
+ ),
178
+ requires_grad=False,
179
+ )
180
+ set_weight_attrs(
181
+ qzeros, {
182
+ "input_dim": 0,
183
+ "output_dim": 1,
184
+ "packed_dim": 1,
185
+ "pack_factor": self.quant_config.pack_factor,
186
+ })
187
+
188
+ scales = Parameter(
189
+ torch.empty(
190
+ num_groups,
191
+ output_size_per_partition,
192
+ dtype=params_dtype,
193
+ ),
194
+ requires_grad=False,
195
+ )
196
+ set_weight_attrs(scales, {
197
+ "input_dim": 0,
198
+ "output_dim": 1,
199
+ })
200
+
201
+ layer.register_parameter("qweight", qweight)
202
+ set_weight_attrs(qweight, extra_weight_attrs)
203
+ layer.register_parameter("qzeros", qzeros)
204
+ set_weight_attrs(qzeros, extra_weight_attrs)
205
+ layer.register_parameter("scales", scales)
206
+ set_weight_attrs(scales, extra_weight_attrs)
207
+
208
+ layer.input_size_per_partition = input_size_per_partition
209
+ layer.output_size_per_partition = output_size_per_partition
210
+ layer.num_groups = num_groups
211
+
212
+ # TODO: Update this docs
213
+ # Checkpoints are serialized in AutoAWQ format, which is different from the
214
+ # marlin format. This function is called after the weights are loaded.
215
+ # Here, we handle the repacking
216
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
217
+ device = layer.qweight.device
218
+
219
+ # Allocate marlin workspace
220
+ layer.workspace = marlin_make_workspace(
221
+ layer.output_size_per_partition, device)
222
+
223
+ # Repack weights from AWQ format to marlin format.
224
+ marlin_qweight = ops.awq_marlin_repack(
225
+ layer.qweight,
226
+ size_k=layer.input_size_per_partition,
227
+ size_n=layer.output_size_per_partition,
228
+ num_bits=self.quant_config.weight_bits)
229
+ replace_tensor(layer, "qweight", marlin_qweight)
230
+
231
+ # Permute scales from AWQ format to marlin format.
232
+ marlin_scales = marlin_permute_scales(
233
+ layer.scales,
234
+ size_k=layer.input_size_per_partition,
235
+ size_n=layer.output_size_per_partition,
236
+ group_size=self.quant_config.group_size)
237
+ replace_tensor(layer, "scales", marlin_scales)
238
+
239
+ # Permute zero-points from AWQ format to marlin format.
240
+ marlin_zp = awq_to_marlin_zero_points(
241
+ layer.qzeros,
242
+ size_k=layer.num_groups,
243
+ size_n=layer.output_size_per_partition,
244
+ num_bits=self.quant_config.weight_bits)
245
+ replace_tensor(layer, "qzeros", marlin_zp)
246
+
247
+ # Not-used
248
+ layer.g_idx = marlin_make_empty_g_idx(device)
249
+ layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
250
+
251
+ def apply(
252
+ self,
253
+ layer: torch.nn.Module,
254
+ x: torch.Tensor,
255
+ bias: Optional[torch.Tensor] = None,
256
+ ) -> torch.Tensor:
257
+ return apply_awq_marlin_linear(
258
+ input=x,
259
+ weight=layer.qweight,
260
+ weight_scale=layer.scales,
261
+ weight_zp=layer.qzeros,
262
+ g_idx=layer.g_idx,
263
+ g_idx_sort_indices=layer.g_idx_sort_indices,
264
+ workspace=layer.workspace,
265
+ num_bits=self.quant_config.weight_bits,
266
+ output_size_per_partition=layer.output_size_per_partition,
267
+ input_size_per_partition=layer.input_size_per_partition,
268
+ bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+10
-6
@@ -7,8 +7,8 @@ from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
10
- apply_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11
- marlin_permute_scales, replace_tensor, verify_marlin_supported,
10
+ apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11
+ marlin_permute_scales, replace_tensor, verify_gptq_marlin_supported,
12
verify_marlin_supports_shape)
13
from vllm.model_executor.utils import set_weight_attrs
14
@@ -38,9 +38,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
38
self.group_size = group_size
39
40
# Verify supported on platform.
41
- verify_marlin_supported(num_bits=self.num_bits,
42
- group_size=self.group_size,
43
- is_sym=True)
41
+ verify_gptq_marlin_supported(num_bits=self.num_bits,
42
+ group_size=self.group_size,
43
+ is_sym=True)
44
45
def get_min_capability(self) -> int:
46
# ampere and up
@@ -135,6 +135,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
135
layer.g_idx = marlin_make_empty_g_idx(device)
136
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
137
138
+ # No zero-point
139
+ layer.weight_zp = marlin_make_empty_g_idx(device)
140
+
141
# Repack weights from compressed-tensors format to marlin format.
142
marlin_qweight = ops.gptq_marlin_repack(
143
layer.weight_packed.t().contiguous(),
@@ -155,10 +158,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
158
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
159
bias: Optional[torch.Tensor]) -> torch.Tensor:
160
158
- return apply_marlin_linear(
161
+ return apply_gptq_marlin_linear(
162
input=x,
163
weight=layer.weight_packed,
164
weight_scale=layer.weight_scale,
165
+ weight_zp=layer.weight_zp,
166
g_idx=layer.g_idx,
167
g_idx_sort_indices=layer.g_idx_sort_indices,
168
workspace=layer.workspace,
vllm/model_executor/layers/quantization/gptq_marlin.py
+21
-12
@@ -10,10 +10,10 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig)
12
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13
- apply_marlin_linear, check_marlin_supported, marlin_is_k_full,
13
+ apply_gptq_marlin_linear, check_gptq_marlin_supported, marlin_is_k_full,
14
marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15
marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
16
- verify_marlin_supported, verify_marlin_supports_shape)
16
+ verify_gptq_marlin_supported, verify_marlin_supports_shape)
17
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
19
logger = init_logger(__name__)
@@ -37,9 +37,9 @@ class GPTQMarlinConfig(QuantizationConfig):
37
self.lm_head_quantized = lm_head_quantized
38
39
# Verify supported on platform.
40
- verify_marlin_supported(num_bits=self.weight_bits,
41
- group_size=self.group_size,
42
- is_sym=self.is_sym)
40
+ verify_gptq_marlin_supported(num_bits=self.weight_bits,
41
+ group_size=self.group_size,
42
+ is_sym=self.is_sym)
43
44
def __repr__(self) -> str:
45
return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
@@ -77,7 +77,7 @@ class GPTQMarlinConfig(QuantizationConfig):
77
@classmethod
78
def override_quantization_method(cls, hf_quant_cfg,
79
user_quant) -> Optional[str]:
80
- can_convert = cls.is_marlin_compatible(hf_quant_cfg)
80
+ can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
81
82
is_valid_user_quant = (user_quant is None or user_quant == "marlin")
83
@@ -105,22 +105,27 @@ class GPTQMarlinConfig(QuantizationConfig):
105
return []
106
107
@classmethod
108
- def is_marlin_compatible(cls, quant_config: Dict[str, Any]):
108
+ def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
109
# Extract data from quant config.
110
+ quant_method = quant_config.get("quant_method", "").lower()
111
num_bits = quant_config.get("bits", None)
112
group_size = quant_config.get("group_size", None)
113
sym = quant_config.get("sym", None)
114
desc_act = quant_config.get("desc_act", None)
115
116
+ if quant_method != "gptq":
117
+ return False
118
+
119
# If we cannot find the info needed in the config, cannot convert.
120
if (num_bits is None or group_size is None or sym is None
121
or desc_act is None):
122
return False
123
120
- return check_marlin_supported(num_bits=num_bits,
121
- group_size=group_size,
122
- is_sym=sym,
123
- min_capability=cls.get_min_capability())
124
+ return check_gptq_marlin_supported(
125
+ num_bits=num_bits,
126
+ group_size=group_size,
127
+ is_sym=sym,
128
+ min_capability=cls.get_min_capability())
129
130
131
class GPTQMarlinLinearMethod(LinearMethodBase):
@@ -278,6 +283,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
283
layer.g_idx = marlin_make_empty_g_idx(device)
284
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
285
286
+ # No zero-point
287
+ layer.zp = marlin_make_empty_g_idx(device)
288
+
289
# Repack weights from autogptq format to marlin format.
290
marlin_qweight = ops.gptq_marlin_repack(
291
layer.qweight,
@@ -302,10 +310,11 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
310
x: torch.Tensor,
311
bias: Optional[torch.Tensor] = None,
312
) -> torch.Tensor:
305
- return apply_marlin_linear(
313
+ return apply_gptq_marlin_linear(
314
input=x,
315
weight=layer.qweight,
316
weight_scale=layer.scales,
317
+ weight_zp=layer.zp,
318
g_idx=layer.g_idx,
319
g_idx_sort_indices=layer.g_idx_sort_indices,
320
workspace=layer.workspace,
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+173
-51
@@ -1,54 +1,92 @@
1
from typing import List, Optional, Tuple
2
3
+import numpy
4
import torch
5
6
from vllm import _custom_ops as ops
7
from vllm.platforms import current_platform
8
9
+from .quant_utils import pack_cols, unpack_cols
10
+
11
GPTQ_MARLIN_TILE = 16
12
GPTQ_MARLIN_MIN_THREAD_N = 64
13
GPTQ_MARLIN_MIN_THREAD_K = 128
14
GPTQ_MARLIN_MAX_PARALLEL = 16
15
13
-GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
14
-GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
15
-GPTQ_MARLIN_SUPPORTED_SYM = [True]
16
-GTPQ_MARLIN_UNSUPPORTED_GROUP_SIZE_ACT_ORDER = [-1]
17
-
18
-
19
-def check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
20
- min_capability: int) -> bool:
21
-
22
- # If the capability of the device is too low, cannot convert.
23
- major, minor = current_platform.get_device_capability()
24
- device_capability = major * 10 + minor
25
- if device_capability < min_capability:
26
- return False
27
-
28
- return (device_capability >= min_capability
29
- and num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
30
- and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
31
- and is_sym in GPTQ_MARLIN_SUPPORTED_SYM)
32
-
33
-
34
-def verify_marlin_supported(num_bits: int, group_size: Optional[int],
35
- is_sym: bool) -> None:
36
-
37
- if num_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
38
- raise ValueError(
39
- f"Marlin does not support weight_bits = {num_bits}. "
40
- f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
41
- "are supported.")
42
- if (group_size is None
43
- or group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES):
44
- raise ValueError(
45
- f"Marlin does not support group_size = {group_size}. "
46
- f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
47
- "are supported.")
48
- if is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
49
- raise ValueError(
50
- f"Marlin does not support is_sym = is_sym. "
51
- f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
16
+MARLIN_SUPPORTED_NUM_BITS = [4, 8]
17
+MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
18
+
19
+
20
+def _check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
21
+ min_capability: Optional[int],
22
+ has_zp: bool) -> Tuple[bool, Optional[str]]:
23
+ if min_capability is not None:
24
+ major, minor = current_platform.get_device_capability()
25
+ device_capability = major * 10 + minor
26
+ if device_capability < min_capability:
27
+ return (False, "Marlin does not support device_capability = {}"
28
+ ", the min_capability required is {}".format(
29
+ device_capability, min_capability))
30
+
31
+ if num_bits not in MARLIN_SUPPORTED_NUM_BITS:
32
+ return (False, "Marlin does not support weight_bits = {}. "
33
+ "Only weight_bits = {} are supported.".format(
34
+ num_bits, MARLIN_SUPPORTED_NUM_BITS))
35
+
36
+ if group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
37
+ return (False, "Marlin does not support group_size = {}. Only "
38
+ "group_sizes = {} are supported.".format(
39
+ group_size, MARLIN_SUPPORTED_GROUP_SIZES))
40
+
41
+ if not has_zp and not is_sym:
42
+ return (False,
43
+ "Marlin without zero_points must have symmetric quantization")
44
+
45
+ return True, None
46
+
47
+
48
+def check_gptq_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
49
+ min_capability: int) -> bool:
50
+ cond, _ = _check_marlin_supported(num_bits,
51
+ group_size,
52
+ is_sym,
53
+ min_capability,
54
+ has_zp=False)
55
+ return cond
56
+
57
+
58
+def check_awq_marlin_supported(num_bits: int, group_size: int, has_zp: bool,
59
+ min_capability: int) -> bool:
60
+ cond, _ = _check_marlin_supported(num_bits,
61
+ group_size,
62
+ False,
63
+ min_capability,
64
+ has_zp=has_zp)
65
+ return cond
66
+
67
+
68
+def verify_gptq_marlin_supported(num_bits: int, group_size: int,
69
+ is_sym: bool) -> None:
70
+ cond, err_msg = _check_marlin_supported(num_bits,
71
+ group_size,
72
+ is_sym,
73
+ min_capability=None,
74
+ has_zp=False)
75
+ if not cond:
76
+ assert err_msg is not None
77
+ raise ValueError("GPTQ" + err_msg)
78
+
79
+
80
+def verify_awq_marlin_supported(num_bits: int, group_size: int,
81
+ has_zp: bool) -> None:
82
+ cond, err_msg = _check_marlin_supported(num_bits,
83
+ group_size,
84
+ False,
85
+ min_capability=None,
86
+ has_zp=has_zp)
87
+ if not cond:
88
+ assert err_msg is not None
89
+ raise ValueError("AWQ" + err_msg)
90
91
92
def verify_marlin_supports_shape(output_size_per_partition: int,
@@ -138,6 +176,51 @@ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
176
return s
177
178
179
+def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int,
180
+ num_bits: int) -> torch.Tensor:
181
+ # Permute zero-points in a similar way to scales, but do not use the
182
+ # "single" permutation, since zero-points are applied on every MMA
183
+ scale_perm, _ = get_scale_perms()
184
+ zp = zp.reshape((-1, len(scale_perm)))[:, scale_perm]
185
+
186
+ # Interleave column dim (for the dequantize code) and pack it to int32
187
+ if num_bits == 4:
188
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
189
+ elif num_bits == 8:
190
+ interleave = numpy.array([0, 2, 1, 3])
191
+ else:
192
+ raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
193
+
194
+ zp = zp.reshape((-1, len(interleave)))[:, interleave].ravel()
195
+ zp = zp.reshape((-1, size_n)).contiguous()
196
+ zp = pack_cols(zp, num_bits, size_k, size_n)
197
+
198
+ return zp
199
+
200
+
201
+def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
202
+ size_n: int, num_bits: int) -> torch.Tensor:
203
+ # AWQ zero-points are quantized and packed on the column dim.
204
+ # In addition, the values are permuted based on dequantizer.
205
+ # Here we undo both of these, and then apply marlin permutation
206
+ # and pack it back.
207
+ q_zp = unpack_cols(q_zp_packed, num_bits, size_k, size_n)
208
+
209
+ # Undo interleaving (use argsort(..) to get inverse perm)
210
+ if num_bits == 4:
211
+ undo_interleave = numpy.argsort(numpy.array([0, 2, 4, 6, 1, 3, 5, 7]))
212
+ elif num_bits == 8:
213
+ undo_interleave = numpy.argsort(numpy.array([0, 2, 1, 3]))
214
+ else:
215
+ raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
216
+
217
+ q_zp = q_zp.reshape((-1, len(undo_interleave)))[:, undo_interleave].ravel()
218
+ q_zp = q_zp.reshape((-1, size_n)).contiguous()
219
+
220
+ marlin_zp = marlin_zero_points(q_zp, size_k, size_n, num_bits)
221
+ return marlin_zp
222
+
223
+
224
# Newly generated tensors need to replace existing tensors that are
225
# already registered as parameters by vLLM (and won't be freed)
226
def replace_tensor(layer: torch.nn.Module, name: str,
@@ -149,23 +232,61 @@ def replace_tensor(layer: torch.nn.Module, name: str,
232
del new_t
233
234
152
-def apply_marlin_linear(input: torch.Tensor,
153
- weight: torch.Tensor,
154
- weight_scale: torch.Tensor,
155
- g_idx: torch.Tensor,
156
- g_idx_sort_indices: torch.Tensor,
157
- workspace: torch.Tensor,
158
- num_bits: int,
159
- output_size_per_partition: int,
160
- input_size_per_partition: int,
161
- is_k_full: bool,
162
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
235
+def apply_gptq_marlin_linear(
236
+ input: torch.Tensor,
237
+ weight: torch.Tensor,
238
+ weight_scale: torch.Tensor,
239
+ weight_zp: torch.Tensor,
240
+ g_idx: torch.Tensor,
241
+ g_idx_sort_indices: torch.Tensor,
242
+ workspace: torch.Tensor,
243
+ num_bits: int,
244
+ output_size_per_partition: int,
245
+ input_size_per_partition: int,
246
+ is_k_full: bool,
247
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
248
+ reshaped_x = input.reshape(-1, input.shape[-1])
249
+ out_shape = input.shape[:-1] + (output_size_per_partition, )
250
+
251
+ output = ops.gptq_marlin_gemm(reshaped_x,
252
+ weight,
253
+ weight_scale,
254
+ weight_zp,
255
+ g_idx,
256
+ g_idx_sort_indices,
257
+ workspace,
258
+ num_bits,
259
+ size_m=reshaped_x.shape[0],
260
+ size_n=output_size_per_partition,
261
+ size_k=input_size_per_partition,
262
+ is_k_full=is_k_full,
263
+ has_zp=False)
264
+
265
+ if bias is not None:
266
+ output.add_(bias) # In-place add
267
+
268
+ return output.reshape(out_shape)
269
+
270
+
271
+def apply_awq_marlin_linear(
272
+ input: torch.Tensor,
273
+ weight: torch.Tensor,
274
+ weight_scale: torch.Tensor,
275
+ weight_zp: torch.Tensor,
276
+ g_idx: torch.Tensor,
277
+ g_idx_sort_indices: torch.Tensor,
278
+ workspace: torch.Tensor,
279
+ num_bits: int,
280
+ output_size_per_partition: int,
281
+ input_size_per_partition: int,
282
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
283
reshaped_x = input.reshape(-1, input.shape[-1])
284
out_shape = input.shape[:-1] + (output_size_per_partition, )
285
286
output = ops.gptq_marlin_gemm(reshaped_x,
287
weight,
288
weight_scale,
289
+ weight_zp,
290
g_idx,
291
g_idx_sort_indices,
292
workspace,
@@ -173,7 +294,8 @@ def apply_marlin_linear(input: torch.Tensor,
294
size_m=reshaped_x.shape[0],
295
size_n=output_size_per_partition,
296
size_k=input_size_per_partition,
176
- is_k_full=is_k_full)
297
+ is_k_full=True,
298
+ has_zp=True)
299
300
if bias is not None:
301
output.add_(bias) # In-place add
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
+41
-10
@@ -2,11 +2,13 @@
2
3
from typing import List
4
5
-import numpy
5
+import numpy as np
6
import torch
7
8
-from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales
9
-from .quant_utils import get_pack_factor, quantize_weights, sort_weights
8
+from .marlin_utils import (GPTQ_MARLIN_TILE, marlin_permute_scales,
9
+ marlin_zero_points)
10
+from .quant_utils import (get_pack_factor, quantize_weights,
11
+ quantize_weights_with_zp, sort_weights)
12
13
14
class MarlinWorkspace:
@@ -46,14 +48,14 @@ def marlin_weights(q_w, size_k, size_n, num_bits, perm):
48
pack_factor = get_pack_factor(num_bits)
49
orig_device = q_w.device
50
49
- q_w = q_w.cpu().numpy().astype(numpy.uint32)
51
+ q_w = q_w.cpu().numpy().astype(np.uint32)
52
51
- q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
52
- dtype=numpy.uint32)
53
+ q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
54
+ dtype=np.uint32)
55
for i in range(pack_factor):
56
q_packed |= q_w[:, i::pack_factor] << num_bits * i
57
56
- q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
58
+ q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device)
59
60
return q_packed
61
@@ -74,12 +76,12 @@ def get_weight_perm(num_bits: int):
76
for j in range(4):
77
perm_list.extend([p + 256 * j for p in perm1])
78
77
- perm = numpy.array(perm_list)
79
+ perm = np.array(perm_list)
80
81
if num_bits == 4:
80
- interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
82
+ interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
83
elif num_bits == 8:
82
- interleave = numpy.array([0, 2, 1, 3])
84
+ interleave = np.array([0, 2, 1, 3])
85
else:
86
raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
87
@@ -118,3 +120,32 @@ def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
120
res_list[i] = res_list[i].to(w.device)
121
122
return res_list
123
+
124
+
125
+def awq_marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int):
126
+ size_k, size_n = w.shape
127
+
128
+ # Normalize group_size
129
+ if group_size == -1:
130
+ group_size = size_k
131
+ assert group_size <= size_k
132
+
133
+ # Detect num groups
134
+ assert size_k % group_size == 0
135
+ num_groups = size_k // group_size
136
+
137
+ # Quantize with zp
138
+ w_ref, q_w, s, zp = quantize_weights_with_zp(w, num_bits, group_size)
139
+
140
+ # Reformat to marlin
141
+ weight_perm = get_weight_perm(num_bits)
142
+ marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
143
+ marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
144
+ marlin_zp = marlin_zero_points(zp, num_groups, size_n, num_bits)
145
+
146
+ # Create result
147
+ res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
148
+ for i in range(len(res_list)):
149
+ res_list[i] = res_list[i].to(w.device)
150
+
151
+ return res_list
vllm/model_executor/layers/quantization/utils/quant_utils.py
+149
-1
@@ -106,6 +106,67 @@ def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
106
)
107
108
109
+def quantize_weights_with_zp(w: torch.Tensor, num_bits: int, group_size: int):
110
+ orig_device = w.device
111
+ size_k, size_n = w.shape
112
+
113
+ assert w.is_floating_point(), "w must be float"
114
+ assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
115
+ assert group_size in SUPPORTED_GROUP_SIZES + [
116
+ size_k
117
+ ], f"Unsupported groupsize = {group_size}"
118
+
119
+ if group_size == -1:
120
+ group_size = size_k
121
+ assert group_size <= size_k
122
+
123
+ max_q_val = 2**num_bits - 1
124
+ min_q_val = 0
125
+
126
+ # Reshape to [groupsize, -1]
127
+ if group_size < size_k:
128
+ w = w.reshape((-1, group_size, size_n))
129
+ w = w.permute(1, 0, 2)
130
+ w = w.reshape((group_size, -1))
131
+
132
+ # Compute scale for each group
133
+ max = torch.max(w, 0, keepdim=True)[0]
134
+ min = torch.min(w, 0, keepdim=True)[0]
135
+ s = (max - min).clamp(min=1e-5) / max_q_val
136
+
137
+ # Compute zero-point for each group
138
+ zp = (-torch.round(min / s)).clamp(min_q_val, max_q_val).int()
139
+
140
+ # Quantize
141
+ q_w = torch.round(w / s).int() + zp
142
+ q_w = torch.clamp(q_w, min_q_val, max_q_val)
143
+
144
+ # Compute ref (dequantized)
145
+ w_ref = (q_w - zp).half() * s
146
+
147
+ # Restore original shapes
148
+ if group_size < size_k:
149
+
150
+ def reshape_w(w):
151
+ w = w.reshape((group_size, -1, size_n))
152
+ w = w.permute(1, 0, 2)
153
+ w = w.reshape((size_k, size_n)).contiguous()
154
+ return w
155
+
156
+ q_w = reshape_w(q_w)
157
+ w_ref = reshape_w(w_ref)
158
+
159
+ s = s.reshape((-1, size_n)).contiguous()
160
+ zp = zp.reshape((-1, size_n)).contiguous()
161
+
162
+ return (
163
+ w_ref.to(device=orig_device),
164
+ q_w.to(device=orig_device),
165
+ s.to(device=orig_device),
166
+ zp.to(device=orig_device),
167
+ )
168
+
169
+
170
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
171
orig_device = q_w.device
172
@@ -122,7 +183,7 @@ def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
183
)
184
185
125
-def gptq_pack(
186
+def pack_rows(
187
q_w: torch.Tensor,
188
num_bits: int,
189
size_k: int,
@@ -144,3 +205,90 @@ def gptq_pack(
205
206
q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
207
return q_res
208
+
209
+
210
+def pack_cols(
211
+ q_w: torch.Tensor,
212
+ num_bits: int,
213
+ size_k: int,
214
+ size_n: int,
215
+):
216
+ assert q_w.shape == (size_k, size_n)
217
+
218
+ pack_factor = get_pack_factor(num_bits)
219
+ assert size_n % pack_factor == 0
220
+
221
+ orig_device = q_w.device
222
+
223
+ q_w = q_w.cpu().numpy().astype(numpy.uint32)
224
+
225
+ q_res = numpy.zeros((size_k, size_n // pack_factor), dtype=numpy.uint32)
226
+
227
+ for i in range(pack_factor):
228
+ q_res |= q_w[:, i::pack_factor] << num_bits * i
229
+
230
+ q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
231
+ q_res = q_res.contiguous()
232
+
233
+ return q_res
234
+
235
+
236
+def unpack_cols(
237
+ packed_q_w: torch.Tensor,
238
+ num_bits: int,
239
+ size_k: int,
240
+ size_n: int,
241
+):
242
+ pack_factor = get_pack_factor(num_bits)
243
+ assert size_n % pack_factor == 0
244
+ assert packed_q_w.shape == (
245
+ size_k, size_n // pack_factor
246
+ ), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format(
247
+ packed_q_w.shape, size_k, size_n, pack_factor)
248
+
249
+ orig_device = packed_q_w.device
250
+
251
+ packed_q_w_cpu = packed_q_w.cpu().numpy().astype(numpy.uint32)
252
+ q_res = numpy.zeros((size_k, size_n), dtype=numpy.uint32)
253
+
254
+ mask = (1 << num_bits) - 1
255
+ for i in range(pack_factor):
256
+ vals = packed_q_w_cpu & mask
257
+ packed_q_w_cpu >>= num_bits
258
+ q_res[:, i::pack_factor] = vals
259
+
260
+ q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
261
+ q_res = q_res.contiguous()
262
+
263
+ return q_res
264
+
265
+
266
+def gptq_pack(
267
+ q_w: torch.Tensor,
268
+ num_bits: int,
269
+ size_k: int,
270
+ size_n: int,
271
+):
272
+ return pack_rows(q_w, num_bits, size_k, size_n)
273
+
274
+
275
+def awq_pack(
276
+ q_w: torch.Tensor,
277
+ num_bits: int,
278
+ size_k: int,
279
+ size_n: int,
280
+):
281
+ assert q_w.shape == (size_k, size_n)
282
+
283
+ # Interleave column dim (for the dequantize code) and pack it to int32
284
+ if num_bits == 4:
285
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
286
+ elif num_bits == 8:
287
+ interleave = numpy.array([0, 2, 1, 3])
288
+ else:
289
+ raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
290
+
291
+ q_w = q_w.reshape((-1, len(interleave)))[:, interleave].ravel()
292
+ q_w = q_w.reshape((-1, size_n)).contiguous()
293
+
294
+ return pack_cols(q_w, num_bits, size_k, size_n)