@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 396d92d5e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Kernel][Core] Add AWQ support to the Marlin kernel (#6612)

Alexander Matveev committed Jul 21, 2024 at 19:41 UTC 396d92d5e064d131422886482cb1b75e5b6538d9
21 files changed +1602 -284
CMakeLists.txt
+1
@@ -172,6 +172,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
172 "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
173 "csrc/quantization/gptq_marlin/gptq_marlin.cu"
174 "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
175 + "csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
176 "csrc/quantization/fp8/fp8_marlin.cu"
177 "csrc/custom_all_reduce.cu"
178 "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
csrc/ops.h
+8 -4
@@ -89,15 +89,19 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
89 int64_t size_k);
90
91 torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
92 - torch::Tensor& b_scales, torch::Tensor& g_idx,
93 - torch::Tensor& perm, torch::Tensor& workspace,
94 - int64_t num_bits, int64_t size_m, int64_t size_n,
95 - int64_t size_k, bool is_k_full);
92 + torch::Tensor& b_scales, torch::Tensor& b_zeros,
93 + torch::Tensor& g_idx, torch::Tensor& perm,
94 + torch::Tensor& workspace, int64_t num_bits,
95 + int64_t size_m, int64_t size_n, int64_t size_k,
96 + bool is_k_full, bool has_zp);
97
98 torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
99 int64_t size_k, int64_t size_n,
100 int64_t num_bits);
101
102 +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
103 + int64_t size_n, int64_t num_bits);
104 +
105 torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
106 torch::Tensor& b_scales, torch::Tensor& workspace,
107 int64_t num_bits, int64_t size_m, int64_t size_n,
csrc/quantization/fp8/fp8_marlin.cu
+15 -18
@@ -19,10 +19,10 @@
19 * Adapted from https://github.com/IST-DASLab/marlin
20 */
21
22 -#include "../gptq_marlin/gptq_marlin.cuh"
23 -#include "../gptq_marlin/gptq_marlin_dtypes.cuh"
22 +#include "../gptq_marlin/marlin.cuh"
23 +#include "../gptq_marlin/marlin_dtypes.cuh"
24
25 -using namespace gptq_marlin;
25 +using namespace marlin;
26
27 #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
28 static_assert(std::is_same<scalar_t, half>::value || \
@@ -1224,16 +1224,15 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1224 ", size_k = ", size_k);
1225
1226 // Verify B
1227 - TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1228 - " is not divisible by tile_size = ", gptq_marlin::tile_size);
1229 - TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
1227 + TORCH_CHECK(size_k % marlin::tile_size == 0, "size_k = ", size_k,
1228 + " is not divisible by tile_size = ", marlin::tile_size);
1229 + TORCH_CHECK((size_k / marlin::tile_size) == b_q_weight.size(0),
1230 "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1231 - ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1232 - TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1231 + ", size_k = ", size_k, ", tile_size = ", marlin::tile_size);
1232 + TORCH_CHECK(b_q_weight.size(1) % marlin::tile_size == 0,
1233 "b_q_weight.size(1) = ", b_q_weight.size(1),
1234 - " is not divisible by tile_size = ", gptq_marlin::tile_size);
1235 - int actual_size_n =
1236 - (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
1234 + " is not divisible by tile_size = ", marlin::tile_size);
1235 + int actual_size_n = (b_q_weight.size(1) / marlin::tile_size) * pack_factor;
1236 TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
1237 ", actual_size_n = ", actual_size_n);
1238
@@ -1274,11 +1273,9 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1273 num_groups = b_scales.size(0);
1274
1275 // Verify workspace size
1277 - TORCH_CHECK(
1278 - size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1279 - ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1280 - int min_workspace_size =
1281 - (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
1276 + TORCH_CHECK(size_n % marlin::min_thread_n == 0, "size_n = ", size_n,
1277 + ", is not divisible by min_thread_n = ", marlin::min_thread_n);
1278 + int min_workspace_size = (size_n / marlin::min_thread_n) * marlin::max_par;
1279 TORCH_CHECK(workspace.numel() >= min_workspace_size,
1280 "workspace.numel = ", workspace.numel(),
1281 " is below min_workspace_size = ", min_workspace_size);
@@ -1290,14 +1287,14 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1287 b_scales.data_ptr<at::Half>(), size_m, size_n, size_k,
1288 workspace.data_ptr(), num_bits, num_groups, group_size, dev,
1289 at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1293 - gptq_marlin::max_par);
1290 + marlin::max_par);
1291 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
1292 fp8_marlin::marlin_mm_f16i4<nv_bfloat16>(
1293 a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
1294 c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(), size_m,
1295 size_n, size_k, workspace.data_ptr(), num_bits, num_groups, group_size,
1296 dev, at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1300 - gptq_marlin::max_par);
1297 + marlin::max_par);
1298 } else {
1299 TORCH_CHECK(false, "fp8_marlin_gemm only supports bfloat16 and float16");
1300 }
csrc/quantization/gptq_marlin/awq_marlin_repack.cu new
+269
@@ -0,0 +1,269 @@
1 +#include "marlin.cuh"
2 +
3 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4 +
5 +namespace marlin {
6 +
7 +template <int const num_threads, int const num_bits, bool const has_perm>
8 +__global__ void awq_marlin_repack_kernel(
9 + uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
10 + int size_k, int size_n) {}
11 +
12 +} // namespace marlin
13 +
14 +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
15 + int64_t size_k, int64_t size_n,
16 + int64_t num_bits) {
17 + TORCH_CHECK_NOT_IMPLEMENTED(
18 + false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
19 + return torch::empty({1, 1});
20 +}
21 +
22 +#else
23 +
24 +namespace marlin {
25 +
26 +template <int const num_threads, int const num_bits>
27 +__global__ void awq_marlin_repack_kernel(
28 + uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
29 + int size_k, int size_n) {
30 + constexpr int pack_factor = 32 / num_bits;
31 +
32 + int k_tiles = size_k / tile_k_size;
33 + int n_tiles = size_n / tile_n_size;
34 + int block_k_tiles = div_ceil(k_tiles, gridDim.x);
35 +
36 + int start_k_tile = blockIdx.x * block_k_tiles;
37 + if (start_k_tile >= k_tiles) {
38 + return;
39 + }
40 +
41 + int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles);
42 +
43 + // Wait until the next thread tile has been loaded to shared memory.
44 + auto wait_for_stage = [&]() {
45 + // We only have `stages - 2` active fetches since we are double buffering
46 + // and can only issue the next fetch when it is guaranteed that the previous
47 + // shared memory load is fully complete (as it may otherwise be
48 + // overwritten).
49 + cp_async_wait<repack_stages - 2>();
50 + __syncthreads();
51 + };
52 +
53 + extern __shared__ int4 sh[];
54 +
55 + constexpr int tile_n_ints = tile_n_size / pack_factor;
56 +
57 + constexpr int stage_n_threads = tile_n_ints / 4;
58 + constexpr int stage_k_threads = tile_k_size;
59 + constexpr int stage_size = stage_k_threads * stage_n_threads;
60 +
61 + auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) {
62 + if (n_tile_id >= n_tiles) {
63 + cp_async_fence();
64 + return;
65 + }
66 +
67 + int first_n = n_tile_id * tile_n_size;
68 + int first_n_packed = first_n / pack_factor;
69 +
70 + int4* sh_ptr = sh + stage_size * pipe;
71 +
72 + if (threadIdx.x < stage_size) {
73 + int k_id = threadIdx.x / stage_n_threads;
74 + int n_id = threadIdx.x % stage_n_threads;
75 +
76 + int first_k = k_tile_id * tile_k_size;
77 +
78 + cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
79 + reinterpret_cast<int4 const*>(
80 + &(b_q_weight_ptr[(first_k + k_id) * (size_n / pack_factor) +
81 + first_n_packed + (n_id * 4)])));
82 + }
83 +
84 + cp_async_fence();
85 + };
86 +
87 + auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) {
88 + if (n_tile_id >= n_tiles) {
89 + return;
90 + }
91 +
92 + int warp_id = threadIdx.x / 32;
93 + int th_id = threadIdx.x % 32;
94 +
95 + if (warp_id >= 4) {
96 + return;
97 + }
98 +
99 + int tc_col = th_id / 4;
100 + int tc_row = (th_id % 4) * 2;
101 +
102 + constexpr int tc_offsets[4] = {0, 1, 8, 9};
103 +
104 + int cur_n = warp_id * 16 + tc_col;
105 + int cur_n_packed = cur_n / pack_factor;
106 + int cur_n_pos = cur_n % pack_factor;
107 +
108 + constexpr int sh_stride = tile_n_ints;
109 + constexpr uint32_t mask = (1 << num_bits) - 1;
110 +
111 + int4* sh_stage_ptr = sh + stage_size * pipe;
112 + uint32_t* sh_stage_int_ptr = reinterpret_cast<uint32_t*>(sh_stage_ptr);
113 +
114 + // Undo interleaving
115 + int cur_n_pos_unpacked;
116 + if constexpr (num_bits == 4) {
117 + constexpr int undo_pack[8] = {0, 4, 1, 5, 2, 6, 3, 7};
118 + cur_n_pos_unpacked = undo_pack[cur_n_pos];
119 + } else {
120 + constexpr int undo_pack[4] = {0, 2, 1, 3};
121 + cur_n_pos_unpacked = undo_pack[cur_n_pos];
122 + }
123 +
124 + uint32_t vals[8];
125 + #pragma unroll
126 + for (int i = 0; i < 4; i++) {
127 + int cur_elem = tc_row + tc_offsets[i];
128 +
129 + int packed_src_0 = sh_stage_int_ptr[cur_n_packed + sh_stride * cur_elem];
130 + int packed_src_1 = sh_stage_int_ptr[cur_n_packed + (8 / pack_factor) +
131 + sh_stride * cur_elem];
132 +
133 + vals[i] = (packed_src_0 >> (cur_n_pos_unpacked * num_bits)) & mask;
134 + vals[4 + i] = (packed_src_1 >> (cur_n_pos_unpacked * num_bits)) & mask;
135 + }
136 +
137 + constexpr int tile_size = tile_k_size * tile_n_size / pack_factor;
138 + int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size;
139 +
140 + // Result of:
141 + // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
142 + if constexpr (num_bits == 4) {
143 + constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
144 +
145 + uint32_t res = 0;
146 + #pragma unroll
147 + for (int i = 0; i < 8; i++) {
148 + res |= vals[pack_idx[i]] << (i * 4);
149 + }
150 +
151 + out_ptr[out_offset + th_id * 4 + warp_id] = res;
152 +
153 + } else {
154 + constexpr int pack_idx[4] = {0, 2, 1, 3};
155 +
156 + uint32_t res1 = 0;
157 + uint32_t res2 = 0;
158 + #pragma unroll
159 + for (int i = 0; i < 4; i++) {
160 + res1 |= vals[pack_idx[i]] << (i * 8);
161 + res2 |= vals[4 + pack_idx[i]] << (i * 8);
162 + }
163 +
164 + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1;
165 + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2;
166 + }
167 + };
168 +
169 + auto start_pipes = [&](int k_tile_id, int n_tile_id) {
170 + #pragma unroll
171 + for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
172 + fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
173 + }
174 +
175 + wait_for_stage();
176 + };
177 + #pragma unroll
178 + for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
179 + int n_tile_id = 0;
180 +
181 + start_pipes(k_tile_id, n_tile_id);
182 +
183 + while (n_tile_id < n_tiles) {
184 + #pragma unroll
185 + for (int pipe = 0; pipe < repack_stages; pipe++) {
186 + fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
187 + n_tile_id + pipe + repack_stages - 1);
188 + repack_tile(pipe, k_tile_id, n_tile_id + pipe);
189 + wait_for_stage();
190 + }
191 + n_tile_id += repack_stages;
192 + }
193 + }
194 +}
195 +
196 +} // namespace marlin
197 +
198 + #define CALL_IF(NUM_BITS) \
199 + else if (num_bits == NUM_BITS) { \
200 + cudaFuncSetAttribute( \
201 + marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS>, \
202 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
203 + marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS> \
204 + <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
205 + b_q_weight_ptr, out_ptr, size_k, size_n); \
206 + }
207 +
208 +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
209 + int64_t size_n, int64_t num_bits) {
210 + // Verify compatibility with marlin tile of 16x64
211 + TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k,
212 + " is not divisible by tile_k_size = ", marlin::tile_k_size);
213 + TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n,
214 + " is not divisible by tile_n_size = ", marlin::tile_n_size);
215 +
216 + TORCH_CHECK(num_bits == 4 || num_bits == 8,
217 + "num_bits must be 4 or 8. Got = ", num_bits);
218 + int const pack_factor = 32 / num_bits;
219 +
220 + // Verify B
221 + TORCH_CHECK(b_q_weight.size(0) == size_k,
222 + "b_q_weight.size(0) = ", b_q_weight.size(0),
223 + " is not size_k = ", size_k);
224 + TORCH_CHECK((size_n / pack_factor) == b_q_weight.size(1),
225 + "Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1),
226 + ", size_n = ", size_n, ", pack_factor = ", pack_factor);
227 +
228 + // Verify device and strides
229 + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
230 + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
231 + TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt");
232 +
233 + // Alloc buffers
234 + const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight));
235 + auto options = torch::TensorOptions()
236 + .dtype(b_q_weight.dtype())
237 + .device(b_q_weight.device());
238 + torch::Tensor out = torch::empty(
239 + {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
240 + options);
241 +
242 + // Get ptrs
243 + uint32_t const* b_q_weight_ptr =
244 + reinterpret_cast<uint32_t const*>(b_q_weight.data_ptr());
245 + uint32_t* out_ptr = reinterpret_cast<uint32_t*>(out.data_ptr());
246 +
247 + // Get dev info
248 + int dev = b_q_weight.get_device();
249 + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
250 + int blocks;
251 + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
252 +
253 + int max_shared_mem = 0;
254 + cudaDeviceGetAttribute(&max_shared_mem,
255 + cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
256 + TORCH_CHECK(max_shared_mem > 0);
257 +
258 + if (false) {
259 + }
260 + CALL_IF(4)
261 + CALL_IF(8)
262 + else {
263 + TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits);
264 + }
265 +
266 + return out;
267 +}
268 +
269 +#endif
csrc/quantization/gptq_marlin/gptq_marlin.cu
+425 -100
@@ -19,8 +19,8 @@
19 * Adapted from https://github.com/IST-DASLab/marlin
20 */
21
22 -#include "gptq_marlin.cuh"
23 -#include "gptq_marlin_dtypes.cuh"
22 +#include "marlin.cuh"
23 +#include "marlin_dtypes.cuh"
24
25 #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
26 static_assert(std::is_same<scalar_t, half>::value || \
@@ -32,7 +32,7 @@ inline std::string str(T x) {
32 return std::to_string(x);
33 }
34
35 -namespace gptq_marlin {
35 +namespace marlin {
36
37 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
38
@@ -72,10 +72,11 @@ __global__ void Marlin(
72 } // namespace gptq_marlin
73
74 torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
75 - torch::Tensor& b_scales, torch::Tensor& g_idx,
76 - torch::Tensor& perm, torch::Tensor& workspace,
77 - int64_t num_bits, int64_t size_m, int64_t size_n,
78 - int64_t size_k, bool is_k_full) {
75 + torch::Tensor& b_scales, torch::Tensor& b_zeros,
76 + torch::Tensor& g_idx, torch::Tensor& perm,
77 + torch::Tensor& workspace, int64_t num_bits,
78 + int64_t size_m, int64_t size_n, int64_t size_k,
79 + bool is_k_full) {
80 TORCH_CHECK_NOT_IMPLEMENTED(false,
81 "marlin_gemm(..) requires CUDA_ARCH >= 8.0");
82 return torch::empty({1, 1});
@@ -264,6 +265,114 @@ dequant_8bit<nv_bfloat16>(int q) {
265 return frag_b;
266 }
267
268 +// Zero-point dequantizers
269 +
270 +template <typename scalar_t>
271 +__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit_zp(int q) {
272 + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
273 +}
274 +
275 +template <>
276 +__device__ inline typename ScalarType<half>::FragB dequant_4bit_zp<half>(
277 + int q) {
278 + const int LO = 0x000f000f;
279 + const int HI = 0x00f000f0;
280 + const int EX = 0x64006400;
281 + // Guarantee that the `(a & b) | c` operations are LOP3s.
282 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
283 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
284 +
285 + const int SUB = 0x64006400;
286 + const int MUL = 0x2c002c00;
287 + const int ADD = 0xd400d400;
288 + typename ScalarType<half>::FragB frag_b;
289 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
290 + *reinterpret_cast<const half2*>(&SUB));
291 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
292 + *reinterpret_cast<const half2*>(&MUL),
293 + *reinterpret_cast<const half2*>(&ADD));
294 + return frag_b;
295 +}
296 +
297 +template <>
298 +__device__ inline typename ScalarType<nv_bfloat16>::FragB
299 +dequant_4bit_zp<nv_bfloat16>(int q) {
300 + static constexpr uint32_t MASK = 0x000f000f;
301 + static constexpr uint32_t EX = 0x43004300;
302 +
303 + // Guarantee that the `(a & b) | c` operations are LOP3s.
304 +
305 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
306 + q >>= 4;
307 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
308 +
309 + typename ScalarType<nv_bfloat16>::FragB frag_b;
310 + static constexpr uint32_t MUL = 0x3F803F80;
311 + static constexpr uint32_t ADD = 0xC300C300;
312 +
313 + frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
314 + *reinterpret_cast<const nv_bfloat162*>(&MUL),
315 + *reinterpret_cast<const nv_bfloat162*>(&ADD));
316 + frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
317 + *reinterpret_cast<const nv_bfloat162*>(&MUL),
318 + *reinterpret_cast<const nv_bfloat162*>(&ADD));
319 + return frag_b;
320 +}
321 +
322 +template <typename scalar_t>
323 +__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit_zp(int q) {
324 + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
325 +}
326 +
327 +template <>
328 +__device__ inline typename ScalarType<half>::FragB dequant_8bit_zp<half>(
329 + int q) {
330 + static constexpr uint32_t mask_for_elt_01 = 0x5250;
331 + static constexpr uint32_t mask_for_elt_23 = 0x5351;
332 + static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
333 +
334 + uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
335 + uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
336 +
337 + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64006400;
338 +
339 + typename ScalarType<half>::FragB frag_b;
340 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
341 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
342 + frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
343 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
344 + return frag_b;
345 +}
346 +
347 +template <>
348 +__device__ inline typename ScalarType<nv_bfloat16>::FragB
349 +dequant_8bit_zp<nv_bfloat16>(int q) {
350 + typename ScalarType<nv_bfloat16>::FragB frag_b;
351 +
352 + float fp32_intermediates[4];
353 + uint32_t* fp32_intermediates_casted =
354 + reinterpret_cast<uint32_t*>(fp32_intermediates);
355 +
356 + static constexpr uint32_t fp32_base = 0x4B000000;
357 + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
358 + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652);
359 + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
360 + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653);
361 +
362 + fp32_intermediates[0] -= 8388608.f;
363 + fp32_intermediates[1] -= 8388608.f;
364 + fp32_intermediates[2] -= 8388608.f;
365 + fp32_intermediates[3] -= 8388608.f;
366 +
367 + uint32_t* bf16_result_ptr = reinterpret_cast<uint32_t*>(&frag_b);
368 + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0],
369 + fp32_intermediates_casted[1], 0x7632);
370 + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2],
371 + fp32_intermediates_casted[3], 0x7632);
372 +
373 + return frag_b;
374 +}
375 +
376 // Multiply dequantized values by the corresponding quantization scale; used
377 // only for grouped quantization.
378 template <typename scalar_t>
@@ -277,6 +386,17 @@ __device__ inline void scale(typename ScalarType<scalar_t>::FragB& frag_b,
386 frag_b[1] = __hmul2(frag_b[1], s);
387 }
388
389 +template <typename scalar_t>
390 +__device__ inline void sub_zp(typename ScalarType<scalar_t>::FragB& frag_b,
391 + typename ScalarType<scalar_t>::scalar_t2& frag_zp,
392 + int i) {
393 + using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
394 + scalar_t2 zp =
395 + ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t*>(&frag_zp)[i]);
396 + frag_b[0] = __hsub2(frag_b[0], zp);
397 + frag_b[1] = __hsub2(frag_b[1], zp);
398 +}
399 +
400 // Same as above, but for act_order (each K is multiplied individually)
401 template <typename scalar_t>
402 __device__ inline void scale4(typename ScalarType<scalar_t>::FragB& frag_b,
@@ -404,6 +524,7 @@ template <typename scalar_t, // compute dtype, half or nv_float16
524 const int stages, // number of stages for the async global->shared
525 // fetch pipeline
526 const bool has_act_order, // whether act_order is enabled
527 + const bool has_zp, // whether zero-points are enabled
528 const int group_blocks = -1 // number of consecutive 16x16 blocks
529 // with a separate quantization scale
530 >
@@ -413,6 +534,8 @@ __global__ void Marlin(
534 int4* __restrict__ C, // fp16 output buffer of shape mxn
535 const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
536 // (k/groupsize)xn
537 + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
538 + // (k/groupsize)x(n/pack_factor)
539 const int* __restrict__ g_idx, // int32 group indices of shape k
540 int num_groups, // number of scale groups per output channel
541 int prob_m, // batch dimension m
@@ -437,6 +560,7 @@ __global__ void Marlin(
560 using FragB = typename ScalarType<scalar_t>::FragB;
561 using FragC = typename ScalarType<scalar_t>::FragC;
562 using FragS = typename ScalarType<scalar_t>::FragS;
563 + using FragZP = typename ScalarType<scalar_t>::FragZP;
564
565 constexpr int pack_factor = 32 / num_bits;
566
@@ -566,6 +690,13 @@ __global__ void Marlin(
690 int tb_n_warps = thread_n_blocks / 4;
691 int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
692
693 + // Zero-points sizes/strides
694 + int zp_gl_stride = (prob_n / pack_factor) / 4;
695 + constexpr int zp_sh_stride = ((16 * thread_n_blocks) / pack_factor) / 4;
696 + constexpr int zp_tb_groups = s_tb_groups;
697 + constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0;
698 + int zp_gl_rd_delta = zp_gl_stride;
699 +
700 // Global A read index of current thread.
701 int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
702 (threadIdx.x % a_gl_rd_delta_o);
@@ -605,6 +736,19 @@ __global__ void Marlin(
736 int s_sh_wr = threadIdx.x;
737 bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
738
739 + // Zero-points
740 + int zp_gl_rd;
741 + if constexpr (has_zp) {
742 + if constexpr (group_blocks == -1) {
743 + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
744 + } else {
745 + zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
746 + zp_sh_stride * slice_col + threadIdx.x;
747 + }
748 + }
749 + int zp_sh_wr = threadIdx.x;
750 + bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride;
751 +
752 // We use a different scale layout for grouped and column-wise quantization as
753 // we scale a `half2` tile in column-major layout in the former and in
754 // row-major in the latter case.
@@ -616,6 +760,18 @@ __global__ void Marlin(
760 s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
761 (threadIdx.x % 32) % 4;
762
763 + // Zero-points have the same read layout as the scales
764 + // (without column-wise case)
765 + constexpr int num_col_threads = 8;
766 + constexpr int num_row_threads = 4;
767 + constexpr int num_ints_per_thread = 8 / pack_factor;
768 + int zp_sh_rd;
769 + if constexpr (has_zp) {
770 + zp_sh_rd = num_ints_per_thread * num_col_threads *
771 + ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
772 + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
773 + }
774 +
775 // Precompute which thread should not read memory in which iterations; this is
776 // needed if there are more threads than required for a certain tilesize or
777 // when the batchsize is not a multiple of 16.
@@ -664,14 +820,17 @@ __global__ void Marlin(
820 int4* sh_a = sh;
821 int4* sh_b = sh_a + (stages * a_sh_stage);
822 int4* sh_g_idx = sh_b + (stages * b_sh_stage);
667 - int4* sh_s = sh_g_idx + (stages * g_idx_stage);
823 + int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
824 + int4* sh_s = sh_zp + (stages * zp_sh_stage);
825
826 // Register storage for double buffer of shared memory reads.
827 FragA frag_a[2][thread_m_blocks];
828 I4 frag_b_quant[2][b_thread_vecs];
829 FragC frag_c[thread_m_blocks][4][2];
673 - FragS frag_s[2][4]; // No act-order
674 - FragS act_frag_s[2][4][4]; // For act-order
830 + FragS frag_s[2][4]; // No act-order
831 + FragS act_frag_s[2][4][4]; // For act-order
832 + int frag_qzp[2][num_ints_per_thread]; // Zero-points
833 + FragZP frag_zp; // Zero-points in fp16
834
835 // Zero accumulators.
836 auto zero_accums = [&]() {
@@ -777,6 +936,28 @@ __global__ void Marlin(
936 }
937 }
938 }
939 +
940 + if constexpr (has_zp && group_blocks != -1) {
941 + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
942 +
943 + if constexpr (group_blocks >= thread_k_blocks) {
944 + // Only fetch zero-points if this tile starts a new group
945 + if (pipe % (group_blocks / thread_k_blocks) == 0) {
946 + if (zp_sh_wr_pred) {
947 + cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
948 + }
949 + zp_gl_rd += zp_gl_rd_delta;
950 + }
951 + } else {
952 + for (int i = 0; i < zp_tb_groups; i++) {
953 + if (zp_sh_wr_pred) {
954 + cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr],
955 + &zp_ptr[zp_gl_rd]);
956 + }
957 + zp_gl_rd += zp_gl_rd_delta;
958 + }
959 + }
960 + }
961 }
962 }
963 // Insert a fence even when we are winding down the pipeline to ensure that
@@ -784,6 +965,12 @@ __global__ void Marlin(
965 cp_async_fence();
966 };
967
968 + auto fetch_zp_to_shared = [&]() {
969 + if (zp_sh_wr_pred) {
970 + cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]);
971 + }
972 + };
973 +
974 // Wait until the next thread tile has been loaded to shared memory.
975 auto wait_for_stage = [&]() {
976 // We only have `stages - 2` active fetches since we are double buffering
@@ -932,8 +1119,73 @@ __global__ void Marlin(
1119 }
1120 };
1121
1122 + auto fetch_zp_to_registers = [&](int k, int full_pipe) {
1123 + if constexpr (!has_zp) {
1124 + return;
1125 + }
1126 +
1127 + int pipe = full_pipe % stages;
1128 +
1129 + if constexpr (group_blocks == -1) {
1130 + for (int i = 0; i < num_ints_per_thread; i++) {
1131 + frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
1132 + }
1133 +
1134 + } else if constexpr (group_blocks >= thread_k_blocks) {
1135 + int4* sh_zp_stage =
1136 + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1137 + (pipe / (group_blocks / thread_k_blocks)));
1138 + for (int i = 0; i < num_ints_per_thread; i++) {
1139 + frag_qzp[k % 2][i] =
1140 + (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1141 + }
1142 + } else {
1143 + int warp_id = threadIdx.x / 32;
1144 + int n_warps = thread_n_blocks / 4;
1145 +
1146 + int warp_row = warp_id / n_warps;
1147 +
1148 + int cur_k = warp_row * 16;
1149 + cur_k += k_iter_size * (k % b_sh_wr_iters);
1150 +
1151 + int k_blocks = cur_k / 16;
1152 + int cur_group_id = k_blocks / group_blocks;
1153 +
1154 + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1155 +
1156 + sh_zp_stage += cur_group_id * zp_sh_stride;
1157 +
1158 + for (int i = 0; i < num_ints_per_thread; i++) {
1159 + frag_qzp[k % 2][i] =
1160 + (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1161 + }
1162 + }
1163 + };
1164 +
1165 // Execute the actual tensor core matmul of a sub-tile.
1166 auto matmul = [&](int k) {
1167 + if constexpr (has_zp) {
1168 + FragB frag_zp_0;
1169 + FragB frag_zp_1;
1170 + if constexpr (num_bits == 4) {
1171 + int zp_quant = frag_qzp[k % 2][0];
1172 + int zp_quant_shift = zp_quant >> 8;
1173 + frag_zp_0 = dequant_4bit_zp<scalar_t>(zp_quant);
1174 + frag_zp_1 = dequant_4bit_zp<scalar_t>(zp_quant_shift);
1175 +
1176 + } else {
1177 + int zp_quant_0 = frag_qzp[k % 2][0];
1178 + int zp_quant_1 = frag_qzp[k % 2][1];
1179 + frag_zp_0 = dequant_8bit_zp<scalar_t>(zp_quant_0);
1180 + frag_zp_1 = dequant_8bit_zp<scalar_t>(zp_quant_1);
1181 + }
1182 +
1183 + frag_zp[0] = frag_zp_0[0];
1184 + frag_zp[1] = frag_zp_0[1];
1185 + frag_zp[2] = frag_zp_1[0];
1186 + frag_zp[3] = frag_zp_1[1];
1187 + }
1188 +
1189 // We have the m dimension as the inner loop in order to encourage overlapping
1190 // dequantization and matmul operations.
1191 #pragma unroll
@@ -944,16 +1196,32 @@ __global__ void Marlin(
1196 int b_quant = frag_b_quant[k % 2][0][j];
1197 int b_quant_shift = b_quant >> 8;
1198
947 - frag_b0 = dequant_4bit<scalar_t>(b_quant);
948 - frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
1199 + if constexpr (has_zp) {
1200 + frag_b0 = dequant_4bit_zp<scalar_t>(b_quant);
1201 + frag_b1 = dequant_4bit_zp<scalar_t>(b_quant_shift);
1202 +
1203 + } else {
1204 + frag_b0 = dequant_4bit<scalar_t>(b_quant);
1205 + frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
1206 + }
1207
1208 } else {
1209 int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
1210 int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
1211 int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
1212
955 - frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
956 - frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
1213 + if constexpr (has_zp) {
1214 + frag_b0 = dequant_8bit_zp<scalar_t>(b_quant_0);
1215 + frag_b1 = dequant_8bit_zp<scalar_t>(b_quant_1);
1216 + } else {
1217 + frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
1218 + frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
1219 + }
1220 + }
1221 +
1222 + // Apply zero-point to frag_b0
1223 + if constexpr (has_zp) {
1224 + sub_zp<scalar_t>(frag_b0, frag_zp[j], 0);
1225 }
1226
1227 // Apply scale to frag_b0
@@ -967,6 +1235,11 @@ __global__ void Marlin(
1235 }
1236 }
1237
1238 + // Apply zero-point to frag_b1
1239 + if constexpr (has_zp) {
1240 + sub_zp<scalar_t>(frag_b1, frag_zp[j], 1);
1241 + }
1242 +
1243 // Apply scale to frag_b1
1244 if constexpr (has_act_order) {
1245 scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j],
@@ -1189,6 +1462,12 @@ __global__ void Marlin(
1462 }
1463 fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1464 }
1465 +
1466 + if constexpr (has_zp && group_blocks == -1) {
1467 + if (i == 0) {
1468 + fetch_zp_to_shared();
1469 + }
1470 + }
1471 fetch_to_shared(i, i, i < slice_iters);
1472 }
1473
@@ -1197,6 +1476,7 @@ __global__ void Marlin(
1476 init_same_group(0);
1477 fetch_to_registers(0, 0);
1478 fetch_scales_to_registers(0, 0);
1479 + fetch_zp_to_registers(0, 0);
1480 a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1481 slice_k_start_shared_fetch += tb_k * (stages - 1);
1482 };
@@ -1217,6 +1497,7 @@ __global__ void Marlin(
1497 for (int k = 0; k < b_sh_wr_iters; k++) {
1498 fetch_to_registers(k + 1, pipe % stages);
1499 fetch_scales_to_registers(k + 1, pipe);
1500 + fetch_zp_to_registers(k + 1, pipe);
1501 if (k == b_sh_wr_iters - 2) {
1502 fetch_to_shared((pipe + stages - 1) % stages, pipe,
1503 slice_iters >= stages);
@@ -1354,6 +1635,7 @@ __global__ void Marlin(
1635
1636 } else {
1637 s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1638 + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
1639 }
1640
1641 start_pipes();
@@ -1363,22 +1645,24 @@ __global__ void Marlin(
1645 }
1646
1647 #define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1366 - THREAD_K_BLOCKS, HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1648 + THREAD_K_BLOCKS, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, \
1649 + NUM_THREADS) \
1650 else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1651 thread_n_blocks == THREAD_N_BLOCKS && \
1652 thread_k_blocks == THREAD_K_BLOCKS && \
1370 - has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1371 - num_threads == NUM_THREADS) { \
1653 + has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1654 + group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1655 cudaFuncSetAttribute( \
1656 Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1657 THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1375 - GROUP_BLOCKS>, \
1658 + HAS_ZP, GROUP_BLOCKS>, \
1659 cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1660 Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1661 THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1379 - GROUP_BLOCKS><<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1380 - A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1381 - prob_k, locks); \
1662 + HAS_ZP, GROUP_BLOCKS> \
1663 + <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1664 + A_ptr, B_ptr, C_ptr, s_ptr, zp_ptr, g_idx_ptr, num_groups, \
1665 + prob_m, prob_n, prob_k, locks); \
1666 }
1667
1668 typedef struct {
@@ -1548,39 +1832,61 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1832 return exec_config_t{0, {-1, -1, -1}};
1833 }
1834
1551 - #define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1552 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1553 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1554 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1555 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1556 - \
1557 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1558 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1559 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1560 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1561 - \
1562 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1563 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1564 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1565 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1566 - \
1567 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1568 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1569 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1570 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1571 - \
1572 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1573 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1574 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1575 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1835 + #define GPTQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1836 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1837 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1838 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1839 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1840 + \
1841 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1842 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1843 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1844 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1845 + \
1846 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1847 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1848 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1849 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1850 + \
1851 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1852 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1853 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1854 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1855 + \
1856 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1857 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1858 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1859 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1860 +
1861 + #define AWQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1862 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1863 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1864 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1865 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1866 + \
1867 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1868 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1869 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1870 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1871 + \
1872 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1873 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1874 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1875 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1876 + \
1877 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1878 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1879 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1880 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1881
1882 template <typename scalar_t>
1578 -void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
1883 +void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, void* zp,
1884 void* g_idx, void* perm, void* a_tmp, int prob_m,
1885 int prob_n, int prob_k, void* workspace, int num_bits,
1581 - bool has_act_order, bool is_k_full, int num_groups,
1582 - int group_size, int dev, cudaStream_t stream, int thread_k,
1583 - int thread_n, int sms, int max_par) {
1886 + bool has_act_order, bool is_k_full, bool has_zp,
1887 + int num_groups, int group_size, int dev,
1888 + cudaStream_t stream, int thread_k, int thread_n, int sms,
1889 + int max_par) {
1890 TORCH_CHECK(num_bits == 4 || num_bits == 8,
1891 "num_bits must be 4 or 8. Got = ", num_bits);
1892 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
@@ -1665,6 +1971,7 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
1971 const int4* B_ptr = (const int4*)B;
1972 int4* C_ptr = (int4*)C;
1973 const int4* s_ptr = (const int4*)s;
1974 + const int4* zp_ptr = (const int4*)zp;
1975 const int* g_idx_ptr = (const int*)g_idx;
1976 const int* perm_ptr = (const int*)perm;
1977 int4* a_tmp_ptr = (int4*)a_tmp;
@@ -1701,28 +2008,33 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
2008 thread_m_blocks = exec_cfg.max_m_blocks;
2009 }
2010
1704 - // Define kernel configurations
2011 if (false) {
2012 }
1707 - CALL_IF(4, 32, 2, 256)
1708 - CALL_IF(4, 16, 4, 256)
1709 - CALL_IF(4, 8, 8, 256)
1710 - CALL_IF(4, 8, 4, 128)
1711 - CALL_IF(4, 4, 8, 128)
1712 - CALL_IF(8, 32, 2, 256)
1713 - CALL_IF(8, 16, 4, 256)
1714 - CALL_IF(8, 8, 8, 256)
1715 - CALL_IF(8, 8, 4, 128)
1716 - CALL_IF(8, 4, 8, 128)
2013 + GPTQ_CALL_IF(4, 16, 4, 256)
2014 + GPTQ_CALL_IF(4, 8, 8, 256)
2015 + GPTQ_CALL_IF(4, 8, 4, 128)
2016 + GPTQ_CALL_IF(4, 4, 8, 128)
2017 + GPTQ_CALL_IF(8, 16, 4, 256)
2018 + GPTQ_CALL_IF(8, 8, 8, 256)
2019 + GPTQ_CALL_IF(8, 8, 4, 128)
2020 + GPTQ_CALL_IF(8, 4, 8, 128)
2021 +
2022 + AWQ_CALL_IF(4, 16, 4, 256)
2023 + AWQ_CALL_IF(4, 8, 8, 256)
2024 + AWQ_CALL_IF(4, 8, 4, 128)
2025 + AWQ_CALL_IF(4, 4, 8, 128)
2026 + AWQ_CALL_IF(8, 16, 4, 256)
2027 + AWQ_CALL_IF(8, 8, 8, 256)
2028 + AWQ_CALL_IF(8, 8, 4, 128)
2029 + AWQ_CALL_IF(8, 4, 8, 128)
2030 else {
1718 - TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1719 - str(prob_n) + ", " + str(prob_k) + "]" +
1720 - ", has_act_order = " + str(has_act_order) +
1721 - ", num_groups = " + str(num_groups) +
1722 - ", group_size = " + str(group_size) +
1723 - ", thread_m_blocks = " + str(thread_m_blocks) +
1724 - ", thread_n_blocks = " + str(thread_n_blocks) +
1725 - ", thread_k_blocks = " + str(thread_k_blocks));
2031 + TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
2032 + ", ", prob_k, "]", ", has_act_order = ", has_act_order,
2033 + ", num_groups = ", num_groups, ", group_size = ", group_size,
2034 + ", thread_m_blocks = ", thread_m_blocks,
2035 + ", thread_n_blocks = ", thread_n_blocks,
2036 + ", thread_k_blocks = ", thread_k_blocks,
2037 + ", num_bits = ", num_bits);
2038 }
2039
2040 A_ptr += 16 * thread_m_blocks * (prob_k / 8) * par;
@@ -1733,10 +2045,11 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
2045 } // namespace gptq_marlin
2046
2047 torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1736 - torch::Tensor& b_scales, torch::Tensor& g_idx,
1737 - torch::Tensor& perm, torch::Tensor& workspace,
1738 - int64_t num_bits, int64_t size_m, int64_t size_n,
1739 - int64_t size_k, bool is_k_full) {
2048 + torch::Tensor& b_scales, torch::Tensor& b_zeros,
2049 + torch::Tensor& g_idx, torch::Tensor& perm,
2050 + torch::Tensor& workspace, int64_t num_bits,
2051 + int64_t size_m, int64_t size_n, int64_t size_k,
2052 + bool is_k_full, bool has_zp) {
2053 // Verify num_bits
2054 TORCH_CHECK(num_bits == 4 || num_bits == 8,
2055 "num_bits must be 4 or 8. Got = ", num_bits);
@@ -1749,16 +2062,15 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2062 ", size_k = ", size_k);
2063
2064 // Verify B
1752 - TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1753 - " is not divisible by tile_size = ", gptq_marlin::tile_size);
1754 - TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
2065 + TORCH_CHECK(size_k % marlin::tile_size == 0, "size_k = ", size_k,
2066 + " is not divisible by tile_size = ", marlin::tile_size);
2067 + TORCH_CHECK((size_k / marlin::tile_size) == b_q_weight.size(0),
2068 "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1756 - ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1757 - TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
2069 + ", size_k = ", size_k, ", tile_size = ", marlin::tile_size);
2070 + TORCH_CHECK(b_q_weight.size(1) % marlin::tile_size == 0,
2071 "b_q_weight.size(1) = ", b_q_weight.size(1),
1759 - " is not divisible by tile_size = ", gptq_marlin::tile_size);
1760 - int actual_size_n =
1761 - (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
2072 + " is not divisible by tile_size = ", marlin::tile_size);
2073 + int actual_size_n = (b_q_weight.size(1) / marlin::tile_size) * pack_factor;
2074 TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
2075 ", actual_size_n = ", actual_size_n);
2076
@@ -1772,6 +2084,9 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2084 TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
2085 TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
2086
2087 + TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU");
2088 + TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous");
2089 +
2090 TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
2091 TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
2092
@@ -1805,8 +2120,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2120 int group_size = -1;
2121 bool has_act_order = g_idx.size(0) != 0;
2122
1808 - int b_rank = b_scales.sizes().size();
1809 - TORCH_CHECK(b_rank == 2, "b_scales rank = ", b_rank, " is not 2");
2123 + int rank = b_scales.sizes().size();
2124 + TORCH_CHECK(rank == 2, "b_scales rank = ", rank, " is not 2");
2125 TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1),
2126 " is not size_n = ", size_n);
2127 num_groups = b_scales.size(0);
@@ -1832,34 +2147,44 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2147 }
2148 }
2149
2150 + // Verify b_zeros
2151 + if (has_zp) {
2152 + int rank = b_zeros.sizes().size();
2153 + TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2");
2154 + TORCH_CHECK(b_zeros.size(0) == num_groups,
2155 + "b_zeros dim 0 = ", b_zeros.size(0),
2156 + " is not num_groups = ", num_groups);
2157 + TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2158 + "b_zeros dim 1 = ", b_scales.size(1),
2159 + " is not size_n / pack_factor = ", size_n / pack_factor);
2160 + }
2161 +
2162 // Verify workspace size
1836 - TORCH_CHECK(
1837 - size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1838 - ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1839 - int min_workspace_size =
1840 - (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
2163 + TORCH_CHECK(size_n % marlin::min_thread_n == 0, "size_n = ", size_n,
2164 + ", is not divisible by min_thread_n = ", marlin::min_thread_n);
2165 + int min_workspace_size = (size_n / marlin::min_thread_n) * marlin::max_par;
2166 TORCH_CHECK(workspace.numel() >= min_workspace_size,
2167 "workspace.numel = ", workspace.numel(),
2168 " is below min_workspace_size = ", min_workspace_size);
2169
2170 int dev = a.get_device();
2171 if (a.scalar_type() == at::ScalarType::Half) {
1847 - gptq_marlin::marlin_mm_f16i4<half>(
2172 + marlin::marlin_mm_f16i4<half>(
2173 a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
1849 - b_scales.data_ptr<at::Half>(), g_idx.data_ptr(), perm.data_ptr(),
1850 - a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
1851 - workspace.data_ptr(), num_bits, has_act_order, is_k_full, num_groups,
1852 - group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1853 - thread_n, sms, gptq_marlin::max_par);
2174 + b_scales.data_ptr<at::Half>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2175 + perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2176 + workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2177 + num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2178 + thread_k, thread_n, sms, marlin::max_par);
2179 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
1855 - gptq_marlin::marlin_mm_f16i4<nv_bfloat16>(
2180 + marlin::marlin_mm_f16i4<nv_bfloat16>(
2181 a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
2182 c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1858 - g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(),
1859 - size_m, size_n, size_k, workspace.data_ptr(), num_bits, has_act_order,
1860 - is_k_full, num_groups, group_size, dev,
1861 - at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1862 - gptq_marlin::max_par);
2183 + b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2184 + a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2185 + workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2186 + num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2187 + thread_k, thread_n, sms, marlin::max_par);
2188 } else {
2189 TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
2190 }
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+27 -33
@@ -1,23 +1,16 @@
1 -#include "gptq_marlin.cuh"
2 -
3 -namespace gptq_marlin {
4 -
5 -static constexpr int repack_stages = 8;
6 -
7 -static constexpr int repack_threads = 256;
8 -
9 -static constexpr int tile_k_size = tile_size;
10 -static constexpr int tile_n_size = tile_k_size * 4;
1 +#include "marlin.cuh"
2
3 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4
5 +namespace marlin {
6 +
7 template <int const num_threads, int const num_bits, bool const has_perm>
15 -__global__ void marlin_repack_kernel(
8 +__global__ void gptq_marlin_repack_kernel(
9 uint32_t const* __restrict__ b_q_weight_ptr,
10 uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
11 int size_k, int size_n) {}
12
20 -} // namespace gptq_marlin
13 +} // namespace marlin
14
15 torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
16 int64_t size_k, int64_t size_n,
@@ -29,8 +22,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
22
23 #else
24
25 +namespace marlin {
26 +
27 template <int const num_threads, int const num_bits, bool const has_perm>
33 -__global__ void marlin_repack_kernel(
28 +__global__ void gptq_marlin_repack_kernel(
29 uint32_t const* __restrict__ b_q_weight_ptr,
30 uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
31 int size_k, int size_n) {
@@ -259,28 +254,28 @@ __global__ void marlin_repack_kernel(
254 }
255 }
256
262 -} // namespace gptq_marlin
263 -
264 - #define CALL_IF(NUM_BITS, HAS_PERM) \
265 - else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
266 - cudaFuncSetAttribute( \
267 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
268 - NUM_BITS, HAS_PERM>, \
269 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
270 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
271 - HAS_PERM> \
272 - <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
273 - b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
257 +} // namespace marlin
258 +
259 + #define CALL_IF(NUM_BITS, HAS_PERM) \
260 + else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
261 + cudaFuncSetAttribute( \
262 + marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
263 + HAS_PERM>, \
264 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
265 + marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
266 + HAS_PERM> \
267 + <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
268 + b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
269 }
270
271 torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
272 int64_t size_k, int64_t size_n,
273 int64_t num_bits) {
274 // Verify compatibility with marlin tile of 16x64
280 - TORCH_CHECK(size_k % gptq_marlin::tile_k_size == 0, "size_k = ", size_k,
281 - " is not divisible by tile_k_size = ", gptq_marlin::tile_k_size);
282 - TORCH_CHECK(size_n % gptq_marlin::tile_n_size == 0, "size_n = ", size_n,
283 - " is not divisible by tile_n_size = ", gptq_marlin::tile_n_size);
275 + TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k,
276 + " is not divisible by tile_k_size = ", marlin::tile_k_size);
277 + TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n,
278 + " is not divisible by tile_n_size = ", marlin::tile_n_size);
279
280 TORCH_CHECK(num_bits == 4 || num_bits == 8,
281 "num_bits must be 4 or 8. Got = ", num_bits);
@@ -308,10 +303,9 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
303 auto options = torch::TensorOptions()
304 .dtype(b_q_weight.dtype())
305 .device(b_q_weight.device());
311 - torch::Tensor out =
312 - torch::empty({size_k / gptq_marlin::tile_size,
313 - size_n * gptq_marlin::tile_size / pack_factor},
314 - options);
306 + torch::Tensor out = torch::empty(
307 + {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
308 + options);
309
310 // Detect if there is act_order
311 bool has_perm = perm.size(0) != 0;
csrc/quantization/gptq_marlin/marlin.cuh renamed
+13 -2
@@ -9,7 +9,9 @@
9 #include <cuda_runtime.h>
10 #include <iostream>
11
12 -namespace gptq_marlin {
12 +namespace marlin {
13 +
14 +// Marlin params
15
16 // 8 warps are a good choice since every SM has 4 schedulers and having more
17 // than 1 warp per schedule allows some more latency hiding. At the same time,
@@ -25,6 +27,15 @@ static constexpr int min_thread_k = 64;
27 static constexpr int tile_size = 16;
28 static constexpr int max_par = 16;
29
30 +// Repack params
31 +static constexpr int repack_stages = 8;
32 +
33 +static constexpr int repack_threads = 256;
34 +
35 +static constexpr int tile_k_size = tile_size;
36 +static constexpr int tile_n_size = tile_k_size * 4;
37 +
38 +// Helpers
39 template <typename T, int n>
40 struct Vec {
41 T elems[n];
@@ -73,4 +84,4 @@ __device__ inline void cp_async_wait() {
84
85 #endif
86
76 -} // namespace gptq_marlin
87 +} // namespace marlin
csrc/quantization/gptq_marlin/marlin_dtypes.cuh renamed
+5 -3
@@ -1,11 +1,11 @@
1
2 #ifndef _data_types_cuh
3 #define _data_types_cuh
4 -#include "gptq_marlin.cuh"
4 +#include "marlin.cuh"
5 #include <cuda_fp16.h>
6 #include <cuda_bf16.h>
7
8 -namespace gptq_marlin {
8 +namespace marlin {
9
10 template <typename scalar_t>
11 class ScalarType {};
@@ -23,6 +23,7 @@ class ScalarType<half> {
23 using FragB = Vec<half2, 2>;
24 using FragC = Vec<float, 4>;
25 using FragS = Vec<half2, 1>;
26 + using FragZP = Vec<half2, 4>;
27
28 static __device__ float inline num2float(const half x) {
29 return __half2float(x);
@@ -51,6 +52,7 @@ class ScalarType<nv_bfloat16> {
52 using FragB = Vec<nv_bfloat162, 2>;
53 using FragC = Vec<float, 4>;
54 using FragS = Vec<nv_bfloat162, 1>;
55 + using FragZP = Vec<nv_bfloat162, 4>;
56
57 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
58 static __device__ float inline num2float(const nv_bfloat16 x) {
@@ -72,6 +74,6 @@ class ScalarType<nv_bfloat16> {
74 #endif
75 };
76
75 -} // namespace gptq_marlin
77 +} // namespace marlin
78
79 #endif
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+24 -22
@@ -30,7 +30,7 @@ inline std::string str(T x) {
30 return std::to_string(x);
31 }
32
33 -namespace marlin {
33 +namespace marlin_dense {
34
35 constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
36
@@ -1040,7 +1040,7 @@ void marlin_cuda(const void* A, const void* B, void* C, void* s, int prob_m,
1040 }
1041 }
1042
1043 -} // namespace marlin
1043 +} // namespace marlin_dense
1044
1045 torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1046 torch::Tensor& b_scales, torch::Tensor& workspace,
@@ -1054,24 +1054,25 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1054 TORCH_CHECK(size_k == a.size(1),
1055 "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1056 ", size_k = " + str(size_k));
1057 - TORCH_CHECK(size_k % marlin::tile_size == 0,
1058 - "size_k = " + str(size_k) +
1059 - " is not divisible by tile_size = " + str(marlin::tile_size));
1060 - TORCH_CHECK((size_k / marlin::tile_size) == b_q_weight.size(0),
1057 + TORCH_CHECK(size_k % marlin_dense::tile_size == 0,
1058 + "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1059 + str(marlin_dense::tile_size));
1060 + TORCH_CHECK((size_k / marlin_dense::tile_size) == b_q_weight.size(0),
1061 "Shape mismatch: b_q_weight.size(0) = " +
1062 str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1063 - ", tile_size = " + str(marlin::tile_size));
1063 + ", tile_size = " + str(marlin_dense::tile_size));
1064
1065 // Verify N
1066 TORCH_CHECK(b_scales.size(1) == size_n,
1067 "b_scales.size(1) = " + str(b_scales.size(1)) +
1068 ", size_n = " + str(size_n));
1069 - TORCH_CHECK(b_q_weight.size(1) % marlin::tile_size == 0,
1070 - "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1071 - " is not divisible by tile_size = " + str(marlin::tile_size));
1069 + TORCH_CHECK(
1070 + b_q_weight.size(1) % marlin_dense::tile_size == 0,
1071 + "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1072 + " is not divisible by tile_size = " + str(marlin_dense::tile_size));
1073
1073 - int actual_size_n =
1074 - (b_q_weight.size(1) / marlin::tile_size) * marlin::pack_factor_4bit;
1074 + int actual_size_n = (b_q_weight.size(1) / marlin_dense::tile_size) *
1075 + marlin_dense::pack_factor_4bit;
1076 TORCH_CHECK(
1077 size_n == actual_size_n,
1078 "size_n = " + str(size_n) + ", actual_size_n = " + str(actual_size_n));
@@ -1116,21 +1117,22 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1117 "Unexpected groupsize = " + str(groupsize));
1118
1119 // Verify workspace size
1119 - TORCH_CHECK(
1120 - size_n % marlin::min_thread_n == 0,
1121 - "size_n = " + str(size_n) +
1122 - ", is not divisible by min_thread_n = " + str(marlin::min_thread_n));
1123 - int min_workspace_size = (size_n / marlin::min_thread_n) * marlin::max_par;
1120 + TORCH_CHECK(size_n % marlin_dense::min_thread_n == 0,
1121 + "size_n = " + str(size_n) +
1122 + ", is not divisible by min_thread_n = " +
1123 + str(marlin_dense::min_thread_n));
1124 + int min_workspace_size =
1125 + (size_n / marlin_dense::min_thread_n) * marlin_dense::max_par;
1126 TORCH_CHECK(workspace.numel() >= min_workspace_size,
1127 "workspace.numel = " + str(workspace.numel()) +
1128 " is below min_workspace_size = " + str(min_workspace_size));
1129
1130 int dev = a.get_device();
1129 - marlin::marlin_cuda(a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(),
1130 - b_scales.data_ptr(), size_m, size_n, size_k,
1131 - workspace.data_ptr(), groupsize, dev,
1132 - at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n,
1133 - sms, marlin::max_par);
1131 + marlin_dense::marlin_cuda(a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(),
1132 + b_scales.data_ptr(), size_m, size_n, size_k,
1133 + workspace.data_ptr(), groupsize, dev,
1134 + at::cuda::getCurrentCUDAStream(dev), thread_k,
1135 + thread_n, sms, marlin_dense::max_par);
1136
1137 return c;
1138 }
csrc/torch_bindings.cpp
+4
@@ -141,6 +141,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
141 ops.def("gptq_marlin_repack", &gptq_marlin_repack);
142 ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
143
144 + // awq_marlin repack from AWQ.
145 + ops.def("awq_marlin_repack", &awq_marlin_repack);
146 + ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
147 +
148 // fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
149 ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
150 ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
tests/kernels/test_marlin_gemm.py
+129 -12
@@ -12,16 +12,18 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
12 GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
13 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
14 GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
15 - GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
16 - marlin_permute_scales)
15 + MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS,
16 + marlin_make_empty_g_idx, marlin_permute_scales)
17 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
18 pack_fp8_to_int32)
19 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20 - MarlinWorkspace, get_weight_perm, marlin_quantize, marlin_weights)
20 + MarlinWorkspace, awq_marlin_quantize, get_weight_perm, marlin_quantize,
21 + marlin_weights)
22 from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
23 marlin_24_quantize)
24 from vllm.model_executor.layers.quantization.utils.quant_utils import (
24 - gptq_pack, quantize_weights, sort_weights)
25 + awq_pack, gptq_pack, quantize_weights, quantize_weights_with_zp,
26 + sort_weights)
27
28 ACT_ORDER_OPTS = [False, True]
29 K_FULL_OPTS = [False, True]
@@ -57,12 +59,12 @@ def rand_data(shape, dtype=torch.float16):
59 reason="Marlin is not supported on this GPU type.")
60 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
61 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
60 -@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
61 -@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
62 +@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
63 +@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
64 @pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
65 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
64 -def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
65 - mnk_factors):
66 +def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
67 + mnk_factors):
68 m_factor, n_factor, k_factor = mnk_factors
69
70 size_m = m_factor
@@ -120,12 +122,60 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
122 reason="Marlin is not supported on this GPU type.")
123 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
124 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
123 -@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
124 -@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
125 +@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
126 +@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
127 +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
128 +def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
129 + mnk_factors):
130 + m_factor, n_factor, k_factor = mnk_factors
131 +
132 + size_m = m_factor
133 + size_k = k_chunk * k_factor
134 + size_n = n_chunk * n_factor
135 +
136 + print(f"MNK = {size_m} {size_n} {size_k}")
137 +
138 + # Normalize group_size
139 + if group_size == -1:
140 + group_size = size_k
141 + assert group_size <= size_k
142 +
143 + # Create input
144 + b_weight = rand_data((size_k, size_n))
145 +
146 + # Quantize
147 + w_ref, q_w, s, zp = quantize_weights_with_zp(b_weight, num_bits,
148 + group_size)
149 +
150 + # Pack to AWQ format
151 + q_w_awq = awq_pack(q_w, num_bits, size_k, size_n)
152 +
153 + # Pack to Marlin format
154 + weight_perm = get_weight_perm(num_bits)
155 + marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
156 +
157 + # Run Marlin repack GPU kernel
158 + marlin_q_w_2 = ops.awq_marlin_repack(
159 + q_w_awq,
160 + size_k,
161 + size_n,
162 + num_bits,
163 + )
164 + torch.cuda.synchronize()
165 +
166 + assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
167 +
168 +
169 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
170 + reason="Marlin is not supported on this GPU type.")
171 +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
172 +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
173 +@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
174 +@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
175 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
176 @pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
177 @pytest.mark.parametrize("is_k_full", K_FULL_OPTS)
128 -def test_marlin_gemm(
178 +def test_gptq_marlin_gemm(
179 k_chunk,
180 n_chunk,
181 num_bits,
@@ -155,6 +205,8 @@ def test_marlin_gemm(
205 w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
206 b_weight, num_bits, group_size, act_order)
207
208 + marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
209 +
210 workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
211 GPTQ_MARLIN_MAX_PARALLEL)
212
@@ -162,6 +214,7 @@ def test_marlin_gemm(
214 a_input,
215 marlin_q_w,
216 marlin_s,
217 + marlin_zp,
218 g_idx,
219 sort_indices,
220 workspace.scratch,
@@ -170,6 +223,7 @@ def test_marlin_gemm(
223 b_weight.shape[1],
224 a_input.shape[1],
225 is_k_full,
226 + has_zp=False,
227 )
228 output_ref = torch.matmul(a_input, w_ref)
229
@@ -188,7 +242,8 @@ def test_marlin_gemm(
242 @pytest.mark.parametrize("num_bits", GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
243 @pytest.mark.parametrize("group_size", GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES)
244 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
191 -def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
245 +def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
246 + mnk_factors):
247 m_factor, n_factor, k_factor = mnk_factors
248
249 size_m = m_factor
@@ -301,3 +356,65 @@ def test_fp8_marlin_gemm(
356 print("max_diff = {}".format(max_diff))
357
358 assert max_diff < 0.04
359 +
360 +
361 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
362 + reason="Marlin is not supported on this GPU type.")
363 +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
364 +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
365 +@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
366 +@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
367 +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
368 +def test_awq_marlin_gemm(
369 + k_chunk,
370 + n_chunk,
371 + num_bits,
372 + group_size,
373 + mnk_factors,
374 +):
375 + m_factor, n_factor, k_factor = mnk_factors
376 +
377 + size_m = m_factor
378 + size_k = k_chunk * k_factor
379 + size_n = n_chunk * n_factor
380 +
381 + print(f"MNK = {size_m} {size_n} {size_k}")
382 + print(f"groupsize = {group_size}")
383 +
384 + a_input = rand_data((size_m, size_k))
385 + b_weight = rand_data((size_k, size_n))
386 +
387 + w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
388 + b_weight, num_bits, group_size)
389 +
390 + g_idx = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
391 + sort_indices = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
392 + is_k_full = True
393 + has_zp = True
394 +
395 + workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
396 + GPTQ_MARLIN_MAX_PARALLEL)
397 +
398 + output = ops.gptq_marlin_gemm(
399 + a_input,
400 + marlin_q_w,
401 + marlin_s,
402 + marlin_zp,
403 + g_idx,
404 + sort_indices,
405 + workspace.scratch,
406 + num_bits,
407 + a_input.shape[0],
408 + b_weight.shape[1],
409 + a_input.shape[1],
410 + is_k_full,
411 + has_zp,
412 + )
413 + output_ref = torch.matmul(a_input, w_ref)
414 +
415 + torch.cuda.synchronize()
416 +
417 + max_diff = compute_max_diff(output, output_ref)
418 + print("max_diff = {}".format(max_diff))
419 +
420 + assert max_diff < 0.04
tests/quantization/test_configs.py
+2 -2
@@ -44,9 +44,9 @@ MODEL_ARG_EXPTYPES = [
44 ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
45
46 # AUTOAWQ
47 - ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, "awq"),
47 + ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, "awq_marlin"),
48 ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
49 - ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", "ERROR"),
49 + ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", "awq_marlin"),
50 ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
51 ]
52
vllm/_custom_ops.py
+15 -7
@@ -276,14 +276,22 @@ def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
276 num_bits)
277
278
279 +# gptq_marlin
280 +def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
281 + num_bits: int) -> torch.Tensor:
282 + return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
283 +
284 +
285 def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
280 - b_scales: torch.Tensor, g_idx: torch.Tensor,
281 - perm: torch.Tensor, workspace: torch.Tensor,
282 - num_bits: int, size_m: int, size_n: int, size_k: int,
283 - is_k_full: bool) -> torch.Tensor:
284 - return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
285 - workspace, num_bits, size_m, size_n,
286 - size_k, is_k_full)
286 + b_scales: torch.Tensor, b_zeros: torch.Tensor,
287 + g_idx: torch.Tensor, perm: torch.Tensor,
288 + workspace: torch.Tensor, num_bits: int, size_m: int,
289 + size_n: int, size_k: int, is_k_full: bool,
290 + has_zp: bool) -> torch.Tensor:
291 + return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
292 + g_idx, perm, workspace, num_bits,
293 + size_m, size_n, size_k, is_k_full,
294 + has_zp)
295
296
297 # fp8 marlin
vllm/config.py
+1 -1
@@ -251,7 +251,7 @@ class ModelConfig:
251 f"supported in ROCm.")
252 if (self.quantization
253 not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
254 - "fbgemm_fp8", "compressed_tensors")):
254 + "awq_marlin", "fbgemm_fp8", "compressed_tensors")):
255 logger.warning(
256 "%s quantization is not fully "
257 "optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -2,6 +2,7 @@ from typing import Dict, Type
2
3 from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 +from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
6 from vllm.model_executor.layers.quantization.base_config import (
7 QuantizationConfig)
8 from vllm.model_executor.layers.quantization.bitsandbytes import (
@@ -31,6 +32,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
32 "marlin": MarlinConfig,
33 "gptq_marlin_24": GPTQMarlin24Config,
34 "gptq_marlin": GPTQMarlinConfig,
35 + "awq_marlin": AWQMarlinConfig,
36 "gptq": GPTQConfig,
37 "squeezellm": SqueezeLLMConfig,
38 "compressed-tensors": CompressedTensorsConfig,
vllm/model_executor/layers/quantization/awq_marlin.py new
+268
@@ -0,0 +1,268 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm import _custom_ops as ops
7 +from vllm.logger import init_logger
8 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 + set_weight_attrs)
10 +from vllm.model_executor.layers.quantization.base_config import (
11 + QuantizationConfig)
12 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13 + apply_awq_marlin_linear, awq_to_marlin_zero_points,
14 + check_awq_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
15 + marlin_permute_scales, replace_tensor, verify_awq_marlin_supported,
16 + verify_marlin_supports_shape)
17 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18 +
19 +logger = init_logger(__name__)
20 +
21 +
22 +class AWQMarlinConfig(QuantizationConfig):
23 + """Config class for AWQ Marlin"""
24 +
25 + def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
26 + lm_head_quantized: bool) -> None:
27 + self.weight_bits = weight_bits
28 + self.pack_factor = 32 // self.weight_bits # packed into int32
29 + self.group_size = group_size
30 + self.has_zp = has_zp
31 + self.lm_head_quantized = lm_head_quantized
32 +
33 + verify_awq_marlin_supported(num_bits=self.weight_bits,
34 + group_size=self.group_size,
35 + has_zp=self.has_zp)
36 +
37 + def __repr__(self) -> str:
38 + return (f"AWQMarlinConfig(weight_bits={self.weight_bits}, "
39 + f"group_size={self.group_size}, "
40 + f"has_zp={self.has_zp}, "
41 + f"lm_head_quantized={self.lm_head_quantized})")
42 +
43 + @classmethod
44 + def get_name(cls) -> str:
45 + return "awq_marlin"
46 +
47 + @classmethod
48 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
49 + return [torch.half, torch.bfloat16]
50 +
51 + @classmethod
52 + def get_min_capability(cls) -> int:
53 + return 80
54 +
55 + @classmethod
56 + def get_config_filenames(cls) -> List[str]:
57 + return ["quantize_config.json"]
58 +
59 + @classmethod
60 + def from_config(cls, config: Dict[str, Any]) -> "AWQMarlinConfig":
61 + weight_bits = cls.get_from_keys(config, ["bits"])
62 + group_size = cls.get_from_keys(config, ["group_size"])
63 + has_zp = cls.get_from_keys(config, ["zero_point"])
64 + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
65 + default=False)
66 + return cls(weight_bits, group_size, has_zp, lm_head_quantized)
67 +
68 + @classmethod
69 + def override_quantization_method(cls, hf_quant_cfg,
70 + user_quant) -> Optional[str]:
71 + can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
72 + is_valid_user_quant = (user_quant is None or user_quant == "marlin")
73 +
74 + if can_convert and is_valid_user_quant:
75 + msg = ("The model is convertible to {} during runtime."
76 + " Using {} kernel.".format(cls.get_name(), cls.get_name()))
77 + logger.info(msg)
78 + return cls.get_name()
79 +
80 + if can_convert and user_quant == "awq":
81 + logger.info("Detected that the model can run with awq_marlin"
82 + ", however you specified quantization=awq explicitly,"
83 + " so forcing awq. Use quantization=awq_marlin for"
84 + " faster inference")
85 + return None
86 +
87 + def get_quant_method(self, layer: torch.nn.Module,
88 + prefix: str) -> Optional["AWQMarlinLinearMethod"]:
89 + if (isinstance(layer, LinearBase) or
90 + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
91 + return AWQMarlinLinearMethod(self)
92 + return None
93 +
94 + def get_scaled_act_names(self) -> List[str]:
95 + return []
96 +
97 + @classmethod
98 + def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]):
99 + # Extract data from quant config.
100 + quant_method = quant_config.get("quant_method", "").lower()
101 + num_bits = quant_config.get("bits", None)
102 + group_size = quant_config.get("group_size", None)
103 + has_zp = quant_config.get("zero_point", None)
104 +
105 + if quant_method != "awq":
106 + return False
107 +
108 + # If we cannot find the info needed in the config, cannot convert.
109 + if (num_bits is None or group_size is None or has_zp is None):
110 + return False
111 +
112 + return check_awq_marlin_supported(
113 + num_bits=num_bits,
114 + group_size=group_size,
115 + has_zp=has_zp,
116 + min_capability=cls.get_min_capability())
117 +
118 +
119 +class AWQMarlinLinearMethod(LinearMethodBase):
120 + """Linear method for AWQ Marlin.
121 +
122 + Args:
123 + quant_config: The AWQ Marlin quantization config.
124 + """
125 +
126 + def __init__(self, quant_config: AWQMarlinConfig) -> None:
127 + self.quant_config = quant_config
128 +
129 + def create_weights(
130 + self,
131 + layer: torch.nn.Module,
132 + input_size_per_partition: int,
133 + output_partition_sizes: List[int],
134 + input_size: int,
135 + output_size: int,
136 + params_dtype: torch.dtype,
137 + **extra_weight_attrs,
138 + ) -> None:
139 + del output_size
140 + output_size_per_partition = sum(output_partition_sizes)
141 +
142 + # Normalize group_size
143 + if self.quant_config.group_size != -1:
144 + group_size = self.quant_config.group_size
145 + else:
146 + group_size = input_size
147 +
148 + verify_marlin_supports_shape(
149 + output_size_per_partition=output_size_per_partition,
150 + input_size_per_partition=input_size_per_partition,
151 + input_size=input_size,
152 + group_size=group_size)
153 +
154 + qweight = Parameter(
155 + torch.empty(
156 + input_size_per_partition,
157 + output_size_per_partition // self.quant_config.pack_factor,
158 + dtype=torch.int32,
159 + ),
160 + requires_grad=False,
161 + )
162 + set_weight_attrs(
163 + qweight, {
164 + "input_dim": 0,
165 + "output_dim": 1,
166 + "packed_dim": 1,
167 + "pack_factor": self.quant_config.pack_factor,
168 + })
169 +
170 + num_groups = input_size_per_partition // group_size
171 +
172 + qzeros = Parameter(
173 + torch.empty(
174 + num_groups,
175 + output_size_per_partition // self.quant_config.pack_factor,
176 + dtype=torch.int32,
177 + ),
178 + requires_grad=False,
179 + )
180 + set_weight_attrs(
181 + qzeros, {
182 + "input_dim": 0,
183 + "output_dim": 1,
184 + "packed_dim": 1,
185 + "pack_factor": self.quant_config.pack_factor,
186 + })
187 +
188 + scales = Parameter(
189 + torch.empty(
190 + num_groups,
191 + output_size_per_partition,
192 + dtype=params_dtype,
193 + ),
194 + requires_grad=False,
195 + )
196 + set_weight_attrs(scales, {
197 + "input_dim": 0,
198 + "output_dim": 1,
199 + })
200 +
201 + layer.register_parameter("qweight", qweight)
202 + set_weight_attrs(qweight, extra_weight_attrs)
203 + layer.register_parameter("qzeros", qzeros)
204 + set_weight_attrs(qzeros, extra_weight_attrs)
205 + layer.register_parameter("scales", scales)
206 + set_weight_attrs(scales, extra_weight_attrs)
207 +
208 + layer.input_size_per_partition = input_size_per_partition
209 + layer.output_size_per_partition = output_size_per_partition
210 + layer.num_groups = num_groups
211 +
212 + # TODO: Update this docs
213 + # Checkpoints are serialized in AutoAWQ format, which is different from the
214 + # marlin format. This function is called after the weights are loaded.
215 + # Here, we handle the repacking
216 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
217 + device = layer.qweight.device
218 +
219 + # Allocate marlin workspace
220 + layer.workspace = marlin_make_workspace(
221 + layer.output_size_per_partition, device)
222 +
223 + # Repack weights from AWQ format to marlin format.
224 + marlin_qweight = ops.awq_marlin_repack(
225 + layer.qweight,
226 + size_k=layer.input_size_per_partition,
227 + size_n=layer.output_size_per_partition,
228 + num_bits=self.quant_config.weight_bits)
229 + replace_tensor(layer, "qweight", marlin_qweight)
230 +
231 + # Permute scales from AWQ format to marlin format.
232 + marlin_scales = marlin_permute_scales(
233 + layer.scales,
234 + size_k=layer.input_size_per_partition,
235 + size_n=layer.output_size_per_partition,
236 + group_size=self.quant_config.group_size)
237 + replace_tensor(layer, "scales", marlin_scales)
238 +
239 + # Permute zero-points from AWQ format to marlin format.
240 + marlin_zp = awq_to_marlin_zero_points(
241 + layer.qzeros,
242 + size_k=layer.num_groups,
243 + size_n=layer.output_size_per_partition,
244 + num_bits=self.quant_config.weight_bits)
245 + replace_tensor(layer, "qzeros", marlin_zp)
246 +
247 + # Not-used
248 + layer.g_idx = marlin_make_empty_g_idx(device)
249 + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
250 +
251 + def apply(
252 + self,
253 + layer: torch.nn.Module,
254 + x: torch.Tensor,
255 + bias: Optional[torch.Tensor] = None,
256 + ) -> torch.Tensor:
257 + return apply_awq_marlin_linear(
258 + input=x,
259 + weight=layer.qweight,
260 + weight_scale=layer.scales,
261 + weight_zp=layer.qzeros,
262 + g_idx=layer.g_idx,
263 + g_idx_sort_indices=layer.g_idx_sort_indices,
264 + workspace=layer.workspace,
265 + num_bits=self.quant_config.weight_bits,
266 + output_size_per_partition=layer.output_size_per_partition,
267 + input_size_per_partition=layer.input_size_per_partition,
268 + bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+10 -6
@@ -7,8 +7,8 @@ from vllm import _custom_ops as ops
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
10 - apply_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11 - marlin_permute_scales, replace_tensor, verify_marlin_supported,
10 + apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11 + marlin_permute_scales, replace_tensor, verify_gptq_marlin_supported,
12 verify_marlin_supports_shape)
13 from vllm.model_executor.utils import set_weight_attrs
14
@@ -38,9 +38,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
38 self.group_size = group_size
39
40 # Verify supported on platform.
41 - verify_marlin_supported(num_bits=self.num_bits,
42 - group_size=self.group_size,
43 - is_sym=True)
41 + verify_gptq_marlin_supported(num_bits=self.num_bits,
42 + group_size=self.group_size,
43 + is_sym=True)
44
45 def get_min_capability(self) -> int:
46 # ampere and up
@@ -135,6 +135,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
135 layer.g_idx = marlin_make_empty_g_idx(device)
136 layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
137
138 + # No zero-point
139 + layer.weight_zp = marlin_make_empty_g_idx(device)
140 +
141 # Repack weights from compressed-tensors format to marlin format.
142 marlin_qweight = ops.gptq_marlin_repack(
143 layer.weight_packed.t().contiguous(),
@@ -155,10 +158,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
158 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
159 bias: Optional[torch.Tensor]) -> torch.Tensor:
160
158 - return apply_marlin_linear(
161 + return apply_gptq_marlin_linear(
162 input=x,
163 weight=layer.weight_packed,
164 weight_scale=layer.weight_scale,
165 + weight_zp=layer.weight_zp,
166 g_idx=layer.g_idx,
167 g_idx_sort_indices=layer.g_idx_sort_indices,
168 workspace=layer.workspace,
vllm/model_executor/layers/quantization/gptq_marlin.py
+21 -12
@@ -10,10 +10,10 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13 - apply_marlin_linear, check_marlin_supported, marlin_is_k_full,
13 + apply_gptq_marlin_linear, check_gptq_marlin_supported, marlin_is_k_full,
14 marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15 marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
16 - verify_marlin_supported, verify_marlin_supports_shape)
16 + verify_gptq_marlin_supported, verify_marlin_supports_shape)
17 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
19 logger = init_logger(__name__)
@@ -37,9 +37,9 @@ class GPTQMarlinConfig(QuantizationConfig):
37 self.lm_head_quantized = lm_head_quantized
38
39 # Verify supported on platform.
40 - verify_marlin_supported(num_bits=self.weight_bits,
41 - group_size=self.group_size,
42 - is_sym=self.is_sym)
40 + verify_gptq_marlin_supported(num_bits=self.weight_bits,
41 + group_size=self.group_size,
42 + is_sym=self.is_sym)
43
44 def __repr__(self) -> str:
45 return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
@@ -77,7 +77,7 @@ class GPTQMarlinConfig(QuantizationConfig):
77 @classmethod
78 def override_quantization_method(cls, hf_quant_cfg,
79 user_quant) -> Optional[str]:
80 - can_convert = cls.is_marlin_compatible(hf_quant_cfg)
80 + can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
81
82 is_valid_user_quant = (user_quant is None or user_quant == "marlin")
83
@@ -105,22 +105,27 @@ class GPTQMarlinConfig(QuantizationConfig):
105 return []
106
107 @classmethod
108 - def is_marlin_compatible(cls, quant_config: Dict[str, Any]):
108 + def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
109 # Extract data from quant config.
110 + quant_method = quant_config.get("quant_method", "").lower()
111 num_bits = quant_config.get("bits", None)
112 group_size = quant_config.get("group_size", None)
113 sym = quant_config.get("sym", None)
114 desc_act = quant_config.get("desc_act", None)
115
116 + if quant_method != "gptq":
117 + return False
118 +
119 # If we cannot find the info needed in the config, cannot convert.
120 if (num_bits is None or group_size is None or sym is None
121 or desc_act is None):
122 return False
123
120 - return check_marlin_supported(num_bits=num_bits,
121 - group_size=group_size,
122 - is_sym=sym,
123 - min_capability=cls.get_min_capability())
124 + return check_gptq_marlin_supported(
125 + num_bits=num_bits,
126 + group_size=group_size,
127 + is_sym=sym,
128 + min_capability=cls.get_min_capability())
129
130
131 class GPTQMarlinLinearMethod(LinearMethodBase):
@@ -278,6 +283,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
283 layer.g_idx = marlin_make_empty_g_idx(device)
284 layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
285
286 + # No zero-point
287 + layer.zp = marlin_make_empty_g_idx(device)
288 +
289 # Repack weights from autogptq format to marlin format.
290 marlin_qweight = ops.gptq_marlin_repack(
291 layer.qweight,
@@ -302,10 +310,11 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
310 x: torch.Tensor,
311 bias: Optional[torch.Tensor] = None,
312 ) -> torch.Tensor:
305 - return apply_marlin_linear(
313 + return apply_gptq_marlin_linear(
314 input=x,
315 weight=layer.qweight,
316 weight_scale=layer.scales,
317 + weight_zp=layer.zp,
318 g_idx=layer.g_idx,
319 g_idx_sort_indices=layer.g_idx_sort_indices,
320 workspace=layer.workspace,
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+173 -51
@@ -1,54 +1,92 @@
1 from typing import List, Optional, Tuple
2
3 +import numpy
4 import torch
5
6 from vllm import _custom_ops as ops
7 from vllm.platforms import current_platform
8
9 +from .quant_utils import pack_cols, unpack_cols
10 +
11 GPTQ_MARLIN_TILE = 16
12 GPTQ_MARLIN_MIN_THREAD_N = 64
13 GPTQ_MARLIN_MIN_THREAD_K = 128
14 GPTQ_MARLIN_MAX_PARALLEL = 16
15
13 -GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
14 -GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
15 -GPTQ_MARLIN_SUPPORTED_SYM = [True]
16 -GTPQ_MARLIN_UNSUPPORTED_GROUP_SIZE_ACT_ORDER = [-1]
17 -
18 -
19 -def check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
20 - min_capability: int) -> bool:
21 -
22 - # If the capability of the device is too low, cannot convert.
23 - major, minor = current_platform.get_device_capability()
24 - device_capability = major * 10 + minor
25 - if device_capability < min_capability:
26 - return False
27 -
28 - return (device_capability >= min_capability
29 - and num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
30 - and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
31 - and is_sym in GPTQ_MARLIN_SUPPORTED_SYM)
32 -
33 -
34 -def verify_marlin_supported(num_bits: int, group_size: Optional[int],
35 - is_sym: bool) -> None:
36 -
37 - if num_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
38 - raise ValueError(
39 - f"Marlin does not support weight_bits = {num_bits}. "
40 - f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
41 - "are supported.")
42 - if (group_size is None
43 - or group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES):
44 - raise ValueError(
45 - f"Marlin does not support group_size = {group_size}. "
46 - f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
47 - "are supported.")
48 - if is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
49 - raise ValueError(
50 - f"Marlin does not support is_sym = is_sym. "
51 - f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
16 +MARLIN_SUPPORTED_NUM_BITS = [4, 8]
17 +MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
18 +
19 +
20 +def _check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
21 + min_capability: Optional[int],
22 + has_zp: bool) -> Tuple[bool, Optional[str]]:
23 + if min_capability is not None:
24 + major, minor = current_platform.get_device_capability()
25 + device_capability = major * 10 + minor
26 + if device_capability < min_capability:
27 + return (False, "Marlin does not support device_capability = {}"
28 + ", the min_capability required is {}".format(
29 + device_capability, min_capability))
30 +
31 + if num_bits not in MARLIN_SUPPORTED_NUM_BITS:
32 + return (False, "Marlin does not support weight_bits = {}. "
33 + "Only weight_bits = {} are supported.".format(
34 + num_bits, MARLIN_SUPPORTED_NUM_BITS))
35 +
36 + if group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
37 + return (False, "Marlin does not support group_size = {}. Only "
38 + "group_sizes = {} are supported.".format(
39 + group_size, MARLIN_SUPPORTED_GROUP_SIZES))
40 +
41 + if not has_zp and not is_sym:
42 + return (False,
43 + "Marlin without zero_points must have symmetric quantization")
44 +
45 + return True, None
46 +
47 +
48 +def check_gptq_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
49 + min_capability: int) -> bool:
50 + cond, _ = _check_marlin_supported(num_bits,
51 + group_size,
52 + is_sym,
53 + min_capability,
54 + has_zp=False)
55 + return cond
56 +
57 +
58 +def check_awq_marlin_supported(num_bits: int, group_size: int, has_zp: bool,
59 + min_capability: int) -> bool:
60 + cond, _ = _check_marlin_supported(num_bits,
61 + group_size,
62 + False,
63 + min_capability,
64 + has_zp=has_zp)
65 + return cond
66 +
67 +
68 +def verify_gptq_marlin_supported(num_bits: int, group_size: int,
69 + is_sym: bool) -> None:
70 + cond, err_msg = _check_marlin_supported(num_bits,
71 + group_size,
72 + is_sym,
73 + min_capability=None,
74 + has_zp=False)
75 + if not cond:
76 + assert err_msg is not None
77 + raise ValueError("GPTQ" + err_msg)
78 +
79 +
80 +def verify_awq_marlin_supported(num_bits: int, group_size: int,
81 + has_zp: bool) -> None:
82 + cond, err_msg = _check_marlin_supported(num_bits,
83 + group_size,
84 + False,
85 + min_capability=None,
86 + has_zp=has_zp)
87 + if not cond:
88 + assert err_msg is not None
89 + raise ValueError("AWQ" + err_msg)
90
91
92 def verify_marlin_supports_shape(output_size_per_partition: int,
@@ -138,6 +176,51 @@ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
176 return s
177
178
179 +def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int,
180 + num_bits: int) -> torch.Tensor:
181 + # Permute zero-points in a similar way to scales, but do not use the
182 + # "single" permutation, since zero-points are applied on every MMA
183 + scale_perm, _ = get_scale_perms()
184 + zp = zp.reshape((-1, len(scale_perm)))[:, scale_perm]
185 +
186 + # Interleave column dim (for the dequantize code) and pack it to int32
187 + if num_bits == 4:
188 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
189 + elif num_bits == 8:
190 + interleave = numpy.array([0, 2, 1, 3])
191 + else:
192 + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
193 +
194 + zp = zp.reshape((-1, len(interleave)))[:, interleave].ravel()
195 + zp = zp.reshape((-1, size_n)).contiguous()
196 + zp = pack_cols(zp, num_bits, size_k, size_n)
197 +
198 + return zp
199 +
200 +
201 +def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
202 + size_n: int, num_bits: int) -> torch.Tensor:
203 + # AWQ zero-points are quantized and packed on the column dim.
204 + # In addition, the values are permuted based on dequantizer.
205 + # Here we undo both of these, and then apply marlin permutation
206 + # and pack it back.
207 + q_zp = unpack_cols(q_zp_packed, num_bits, size_k, size_n)
208 +
209 + # Undo interleaving (use argsort(..) to get inverse perm)
210 + if num_bits == 4:
211 + undo_interleave = numpy.argsort(numpy.array([0, 2, 4, 6, 1, 3, 5, 7]))
212 + elif num_bits == 8:
213 + undo_interleave = numpy.argsort(numpy.array([0, 2, 1, 3]))
214 + else:
215 + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
216 +
217 + q_zp = q_zp.reshape((-1, len(undo_interleave)))[:, undo_interleave].ravel()
218 + q_zp = q_zp.reshape((-1, size_n)).contiguous()
219 +
220 + marlin_zp = marlin_zero_points(q_zp, size_k, size_n, num_bits)
221 + return marlin_zp
222 +
223 +
224 # Newly generated tensors need to replace existing tensors that are
225 # already registered as parameters by vLLM (and won't be freed)
226 def replace_tensor(layer: torch.nn.Module, name: str,
@@ -149,23 +232,61 @@ def replace_tensor(layer: torch.nn.Module, name: str,
232 del new_t
233
234
152 -def apply_marlin_linear(input: torch.Tensor,
153 - weight: torch.Tensor,
154 - weight_scale: torch.Tensor,
155 - g_idx: torch.Tensor,
156 - g_idx_sort_indices: torch.Tensor,
157 - workspace: torch.Tensor,
158 - num_bits: int,
159 - output_size_per_partition: int,
160 - input_size_per_partition: int,
161 - is_k_full: bool,
162 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
235 +def apply_gptq_marlin_linear(
236 + input: torch.Tensor,
237 + weight: torch.Tensor,
238 + weight_scale: torch.Tensor,
239 + weight_zp: torch.Tensor,
240 + g_idx: torch.Tensor,
241 + g_idx_sort_indices: torch.Tensor,
242 + workspace: torch.Tensor,
243 + num_bits: int,
244 + output_size_per_partition: int,
245 + input_size_per_partition: int,
246 + is_k_full: bool,
247 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
248 + reshaped_x = input.reshape(-1, input.shape[-1])
249 + out_shape = input.shape[:-1] + (output_size_per_partition, )
250 +
251 + output = ops.gptq_marlin_gemm(reshaped_x,
252 + weight,
253 + weight_scale,
254 + weight_zp,
255 + g_idx,
256 + g_idx_sort_indices,
257 + workspace,
258 + num_bits,
259 + size_m=reshaped_x.shape[0],
260 + size_n=output_size_per_partition,
261 + size_k=input_size_per_partition,
262 + is_k_full=is_k_full,
263 + has_zp=False)
264 +
265 + if bias is not None:
266 + output.add_(bias) # In-place add
267 +
268 + return output.reshape(out_shape)
269 +
270 +
271 +def apply_awq_marlin_linear(
272 + input: torch.Tensor,
273 + weight: torch.Tensor,
274 + weight_scale: torch.Tensor,
275 + weight_zp: torch.Tensor,
276 + g_idx: torch.Tensor,
277 + g_idx_sort_indices: torch.Tensor,
278 + workspace: torch.Tensor,
279 + num_bits: int,
280 + output_size_per_partition: int,
281 + input_size_per_partition: int,
282 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
283 reshaped_x = input.reshape(-1, input.shape[-1])
284 out_shape = input.shape[:-1] + (output_size_per_partition, )
285
286 output = ops.gptq_marlin_gemm(reshaped_x,
287 weight,
288 weight_scale,
289 + weight_zp,
290 g_idx,
291 g_idx_sort_indices,
292 workspace,
@@ -173,7 +294,8 @@ def apply_marlin_linear(input: torch.Tensor,
294 size_m=reshaped_x.shape[0],
295 size_n=output_size_per_partition,
296 size_k=input_size_per_partition,
176 - is_k_full=is_k_full)
297 + is_k_full=True,
298 + has_zp=True)
299
300 if bias is not None:
301 output.add_(bias) # In-place add
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
+41 -10
@@ -2,11 +2,13 @@
2
3 from typing import List
4
5 -import numpy
5 +import numpy as np
6 import torch
7
8 -from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales
9 -from .quant_utils import get_pack_factor, quantize_weights, sort_weights
8 +from .marlin_utils import (GPTQ_MARLIN_TILE, marlin_permute_scales,
9 + marlin_zero_points)
10 +from .quant_utils import (get_pack_factor, quantize_weights,
11 + quantize_weights_with_zp, sort_weights)
12
13
14 class MarlinWorkspace:
@@ -46,14 +48,14 @@ def marlin_weights(q_w, size_k, size_n, num_bits, perm):
48 pack_factor = get_pack_factor(num_bits)
49 orig_device = q_w.device
50
49 - q_w = q_w.cpu().numpy().astype(numpy.uint32)
51 + q_w = q_w.cpu().numpy().astype(np.uint32)
52
51 - q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
52 - dtype=numpy.uint32)
53 + q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
54 + dtype=np.uint32)
55 for i in range(pack_factor):
56 q_packed |= q_w[:, i::pack_factor] << num_bits * i
57
56 - q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
58 + q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device)
59
60 return q_packed
61
@@ -74,12 +76,12 @@ def get_weight_perm(num_bits: int):
76 for j in range(4):
77 perm_list.extend([p + 256 * j for p in perm1])
78
77 - perm = numpy.array(perm_list)
79 + perm = np.array(perm_list)
80
81 if num_bits == 4:
80 - interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
82 + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
83 elif num_bits == 8:
82 - interleave = numpy.array([0, 2, 1, 3])
84 + interleave = np.array([0, 2, 1, 3])
85 else:
86 raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
87
@@ -118,3 +120,32 @@ def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
120 res_list[i] = res_list[i].to(w.device)
121
122 return res_list
123 +
124 +
125 +def awq_marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int):
126 + size_k, size_n = w.shape
127 +
128 + # Normalize group_size
129 + if group_size == -1:
130 + group_size = size_k
131 + assert group_size <= size_k
132 +
133 + # Detect num groups
134 + assert size_k % group_size == 0
135 + num_groups = size_k // group_size
136 +
137 + # Quantize with zp
138 + w_ref, q_w, s, zp = quantize_weights_with_zp(w, num_bits, group_size)
139 +
140 + # Reformat to marlin
141 + weight_perm = get_weight_perm(num_bits)
142 + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
143 + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
144 + marlin_zp = marlin_zero_points(zp, num_groups, size_n, num_bits)
145 +
146 + # Create result
147 + res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
148 + for i in range(len(res_list)):
149 + res_list[i] = res_list[i].to(w.device)
150 +
151 + return res_list
vllm/model_executor/layers/quantization/utils/quant_utils.py
+149 -1
@@ -106,6 +106,67 @@ def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
106 )
107
108
109 +def quantize_weights_with_zp(w: torch.Tensor, num_bits: int, group_size: int):
110 + orig_device = w.device
111 + size_k, size_n = w.shape
112 +
113 + assert w.is_floating_point(), "w must be float"
114 + assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
115 + assert group_size in SUPPORTED_GROUP_SIZES + [
116 + size_k
117 + ], f"Unsupported groupsize = {group_size}"
118 +
119 + if group_size == -1:
120 + group_size = size_k
121 + assert group_size <= size_k
122 +
123 + max_q_val = 2**num_bits - 1
124 + min_q_val = 0
125 +
126 + # Reshape to [groupsize, -1]
127 + if group_size < size_k:
128 + w = w.reshape((-1, group_size, size_n))
129 + w = w.permute(1, 0, 2)
130 + w = w.reshape((group_size, -1))
131 +
132 + # Compute scale for each group
133 + max = torch.max(w, 0, keepdim=True)[0]
134 + min = torch.min(w, 0, keepdim=True)[0]
135 + s = (max - min).clamp(min=1e-5) / max_q_val
136 +
137 + # Compute zero-point for each group
138 + zp = (-torch.round(min / s)).clamp(min_q_val, max_q_val).int()
139 +
140 + # Quantize
141 + q_w = torch.round(w / s).int() + zp
142 + q_w = torch.clamp(q_w, min_q_val, max_q_val)
143 +
144 + # Compute ref (dequantized)
145 + w_ref = (q_w - zp).half() * s
146 +
147 + # Restore original shapes
148 + if group_size < size_k:
149 +
150 + def reshape_w(w):
151 + w = w.reshape((group_size, -1, size_n))
152 + w = w.permute(1, 0, 2)
153 + w = w.reshape((size_k, size_n)).contiguous()
154 + return w
155 +
156 + q_w = reshape_w(q_w)
157 + w_ref = reshape_w(w_ref)
158 +
159 + s = s.reshape((-1, size_n)).contiguous()
160 + zp = zp.reshape((-1, size_n)).contiguous()
161 +
162 + return (
163 + w_ref.to(device=orig_device),
164 + q_w.to(device=orig_device),
165 + s.to(device=orig_device),
166 + zp.to(device=orig_device),
167 + )
168 +
169 +
170 def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
171 orig_device = q_w.device
172
@@ -122,7 +183,7 @@ def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
183 )
184
185
125 -def gptq_pack(
186 +def pack_rows(
187 q_w: torch.Tensor,
188 num_bits: int,
189 size_k: int,
@@ -144,3 +205,90 @@ def gptq_pack(
205
206 q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
207 return q_res
208 +
209 +
210 +def pack_cols(
211 + q_w: torch.Tensor,
212 + num_bits: int,
213 + size_k: int,
214 + size_n: int,
215 +):
216 + assert q_w.shape == (size_k, size_n)
217 +
218 + pack_factor = get_pack_factor(num_bits)
219 + assert size_n % pack_factor == 0
220 +
221 + orig_device = q_w.device
222 +
223 + q_w = q_w.cpu().numpy().astype(numpy.uint32)
224 +
225 + q_res = numpy.zeros((size_k, size_n // pack_factor), dtype=numpy.uint32)
226 +
227 + for i in range(pack_factor):
228 + q_res |= q_w[:, i::pack_factor] << num_bits * i
229 +
230 + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
231 + q_res = q_res.contiguous()
232 +
233 + return q_res
234 +
235 +
236 +def unpack_cols(
237 + packed_q_w: torch.Tensor,
238 + num_bits: int,
239 + size_k: int,
240 + size_n: int,
241 +):
242 + pack_factor = get_pack_factor(num_bits)
243 + assert size_n % pack_factor == 0
244 + assert packed_q_w.shape == (
245 + size_k, size_n // pack_factor
246 + ), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format(
247 + packed_q_w.shape, size_k, size_n, pack_factor)
248 +
249 + orig_device = packed_q_w.device
250 +
251 + packed_q_w_cpu = packed_q_w.cpu().numpy().astype(numpy.uint32)
252 + q_res = numpy.zeros((size_k, size_n), dtype=numpy.uint32)
253 +
254 + mask = (1 << num_bits) - 1
255 + for i in range(pack_factor):
256 + vals = packed_q_w_cpu & mask
257 + packed_q_w_cpu >>= num_bits
258 + q_res[:, i::pack_factor] = vals
259 +
260 + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
261 + q_res = q_res.contiguous()
262 +
263 + return q_res
264 +
265 +
266 +def gptq_pack(
267 + q_w: torch.Tensor,
268 + num_bits: int,
269 + size_k: int,
270 + size_n: int,
271 +):
272 + return pack_rows(q_w, num_bits, size_k, size_n)
273 +
274 +
275 +def awq_pack(
276 + q_w: torch.Tensor,
277 + num_bits: int,
278 + size_k: int,
279 + size_n: int,
280 +):
281 + assert q_w.shape == (size_k, size_n)
282 +
283 + # Interleave column dim (for the dequantize code) and pack it to int32
284 + if num_bits == 4:
285 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
286 + elif num_bits == 8:
287 + interleave = numpy.array([0, 2, 1, 3])
288 + else:
289 + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
290 +
291 + q_w = q_w.reshape((-1, len(interleave)))[:, interleave].ravel()
292 + q_w = q_w.reshape((-1, size_n)).contiguous()
293 +
294 + return pack_cols(q_w, num_bits, size_k, size_n)