@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 360bd67cf0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Support loading GGUF model (#5191)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Isotr0py committed Aug 6, 2024 at 07:54 UTC 360bd67cf0ea4a79a59c1aae736cc495a5a63ec5
29 files changed +4970 -21
.github/workflows/clang-format.yml
+5
@@ -30,6 +30,11 @@ jobs:
30 run: |
31 EXCLUDES=(
32 'csrc/moe/topk_softmax_kernels.cu'
33 + 'csrc/quantization/gguf/ggml-common.h'
34 + 'csrc/quantization/gguf/dequantize.cuh'
35 + 'csrc/quantization/gguf/vecdotq.cuh'
36 + 'csrc/quantization/gguf/mmq.cuh'
37 + 'csrc/quantization/gguf/mmvq.cuh'
38 )
39 find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
40 | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
CMakeLists.txt
+1
@@ -208,6 +208,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
208 "csrc/quantization/gptq_marlin/gptq_marlin.cu"
209 "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
210 "csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
211 + "csrc/quantization/gguf/gguf_kernel.cu"
212 "csrc/quantization/fp8/fp8_marlin.cu"
213 "csrc/custom_all_reduce.cu"
214 "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
csrc/ops.h
+9
@@ -107,6 +107,15 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
107 torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
108 int64_t size_n, int64_t num_bits);
109
110 +torch::Tensor ggml_dequantize(torch::Tensor W, int8_t type, int64_t m,
111 + int64_t n);
112 +
113 +torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X, int8_t type,
114 + int64_t row);
115 +
116 +torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int8_t type,
117 + int64_t row);
118 +
119 torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
120 torch::Tensor& b_scales, torch::Tensor& workspace,
121 int64_t num_bits, int64_t size_m, int64_t size_n,
csrc/quantization/gguf/dequantize.cuh new
+531
@@ -0,0 +1,531 @@
1 +// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu
2 +// Dequant functions
3 +static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
4 + const block_q4_0 * x = (const block_q4_0 *) vx;
5 +
6 + const dfloat d = x[ib].d;
7 +
8 + const int vui = x[ib].qs[iqs];
9 +
10 + v.x = __int2half_rn(vui & 0xF);
11 + v.y = __int2half_rn(vui >> 4);
12 +
13 + v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f));
14 + v = __hmul2(v, {d, d});
15 +}
16 +
17 +static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
18 + const block_q4_1 * x = (const block_q4_1 *) vx;
19 +
20 + const dfloat d = __low2half(x[ib].dm);
21 + const dfloat m = __high2half(x[ib].dm);
22 +
23 + const int vui = x[ib].qs[iqs];
24 +
25 + v.x = __int2half_rn(vui & 0xF);
26 + v.y = __int2half_rn(vui >> 4);
27 +
28 + v = __hmul2(v, {d, d});
29 + v = __hadd2(v, {m, m});
30 +}
31 +
32 +static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
33 + const block_q5_0 * x = (const block_q5_0 *) vx;
34 +
35 + const dfloat d = x[ib].d;
36 +
37 + uint32_t qh;
38 + memcpy(&qh, x[ib].qh, sizeof(qh));
39 +
40 + const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
41 + const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
42 +
43 + v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
44 + v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
45 +
46 + v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f));
47 + v = __hmul2(v, {d, d});
48 +}
49 +
50 +static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
51 + const block_q5_1 * x = (const block_q5_1 *) vx;
52 +
53 + const dfloat d = __low2half(x[ib].dm);
54 + const dfloat m = __high2half(x[ib].dm);
55 +
56 + uint32_t qh;
57 + memcpy(&qh, x[ib].qh, sizeof(qh));
58 +
59 + const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
60 + const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
61 +
62 + v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
63 + v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
64 +
65 + v = __hmul2(v, {d, d});
66 + v = __hadd2(v, {m, m});
67 +}
68 +
69 +static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
70 + const block_q8_0 * x = (const block_q8_0 *) vx;
71 +
72 + const dfloat d = x[ib].d;
73 +
74 + v.x = __int2half_rn(x[ib].qs[iqs + 0]);
75 + v.y = __int2half_rn(x[ib].qs[iqs + 1]);
76 +
77 + v = __hmul2(v, {d, d});
78 +}
79 +
80 +template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
81 +static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int k) {
82 + const int i = 2*(blockDim.x*blockIdx.x + threadIdx.x);
83 +
84 + if (i >= k) {
85 + return;
86 + }
87 +
88 + const int ib = i/qk; // block index
89 + const int iqs = (i%qk)/qr; // quant index
90 + const int iybs = i - i%qk; // y block start index
91 + const int y_offset = qr == 1 ? 1 : qk/2;
92 +
93 + // dequantize
94 + dfloat2 v;
95 + dequantize_kernel(vx, ib, iqs, v);
96 +
97 + y[iybs + iqs + 0] = v.x;
98 + y[iybs + iqs + y_offset] = v.y;
99 +}
100 +
101 +template<typename dst_t>
102 +static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
103 +
104 + const int i = blockIdx.x;
105 + const block_q2_K * x = (const block_q2_K *) vx;
106 +
107 + const int tid = threadIdx.x;
108 + const int n = tid/32;
109 + const int l = tid - 32*n;
110 + const int is = 8*n + l/16;
111 +
112 + const uint8_t q = x[i].qs[32*n + l];
113 + dst_t * y = yy + i*QK_K + 128*n;
114 +
115 + half dall = __low2half(x[i].dm);
116 + half dmin = __high2half(x[i].dm);
117 + y[l+ 0] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+0] & 0xF) * ((q >> 0) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+0] >> 4)));
118 + y[l+32] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+2] & 0xF) * ((q >> 2) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+2] >> 4)));
119 + y[l+64] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+4] & 0xF) * ((q >> 4) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+4] >> 4)));
120 + y[l+96] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+6] & 0xF) * ((q >> 6) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+6] >> 4)));
121 +}
122 +
123 +template<typename dst_t>
124 +static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
125 +
126 + const int i = blockIdx.x;
127 + const block_q3_K * x = (const block_q3_K *) vx;
128 +
129 + const int r = threadIdx.x/4;
130 + const int tid = r/2;
131 + const int is0 = r%2;
132 + const int l0 = 16*is0 + 4*(threadIdx.x%4);
133 + const int n = tid / 4;
134 + const int j = tid - 4*n;
135 +
136 + uint8_t m = 1 << (4*n + j);
137 + int is = 8*n + 2*j + is0;
138 + int shift = 2*j;
139 +
140 + int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :
141 + is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :
142 + is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :
143 + (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);
144 + half d_all = x[i].d;
145 + half dl = __hmul(d_all, __int2half_rn(us - 32));
146 +
147 + dst_t * y = yy + i*QK_K + 128*n + 32*j;
148 + const uint8_t * q = x[i].qs + 32*n;
149 + const uint8_t * hm = x[i].hmask;
150 +
151 + for (int l = l0; l < l0+4; ++l) y[l] = __hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)));
152 +}
153 +
154 +static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {
155 + if (j < 4) {
156 + d = q[j] & 63; m = q[j + 4] & 63;
157 + } else {
158 + d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
159 + m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
160 + }
161 +}
162 +
163 +template<typename dst_t>
164 +static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
165 + const block_q4_K * x = (const block_q4_K *) vx;
166 +
167 + const int i = blockIdx.x;
168 +
169 + // assume 32 threads
170 + const int tid = threadIdx.x;
171 + const int il = tid/8;
172 + const int ir = tid%8;
173 + const int is = 2*il;
174 + const int n = 4;
175 +
176 + dst_t * y = yy + i*QK_K + 64*il + n*ir;
177 +
178 + const half dall = __low2half(x[i].dm);
179 + const half dmin = __high2half(x[i].dm);
180 +
181 + const uint8_t * q = x[i].qs + 32*il + n*ir;
182 +
183 + uint8_t sc, m;
184 + get_scale_min_k4(is + 0, x[i].scales, sc, m);
185 + const half d1 = __hmul(dall, __int2half_rn(sc));
186 + const half m1 = __hmul(dmin, __int2half_rn(m));
187 + get_scale_min_k4(is + 1, x[i].scales, sc, m);
188 + const half d2 = __hmul(dall, __int2half_rn(sc));
189 + const half m2 = __hmul(dmin, __int2half_rn(m));
190 + for (int l = 0; l < n; ++l) {
191 + y[l + 0] = __hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1);
192 + y[l +32] = __hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2);
193 + }
194 +}
195 +
196 +template<typename dst_t>
197 +static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
198 + const block_q5_K * x = (const block_q5_K *) vx;
199 +
200 + const int i = blockIdx.x;
201 +
202 + // assume 64 threads - this is very slightly better than the one below
203 + const int tid = threadIdx.x;
204 + const int il = tid/16; // il is in 0...3
205 + const int ir = tid%16; // ir is in 0...15
206 + const int is = 2*il; // is is in 0...6
207 +
208 + dst_t * y = yy + i*QK_K + 64*il + 2*ir;
209 +
210 + const half dall = __low2half(x[i].dm);
211 + const half dmin = __high2half(x[i].dm);
212 +
213 + const uint8_t * ql = x[i].qs + 32*il + 2*ir;
214 + const uint8_t * qh = x[i].qh + 2*ir;
215 +
216 + uint8_t sc, m;
217 + get_scale_min_k4(is + 0, x[i].scales, sc, m);
218 + const half d1 = __hmul(dall, __int2half_rn(sc)); const half m1 = __hmul(dmin, __int2half_rn(m));
219 + get_scale_min_k4(is + 1, x[i].scales, sc, m);
220 + const half d2 = __hmul(dall, __int2half_rn(sc)); const half m2 = __hmul(dmin, __int2half_rn(m));
221 +
222 + uint8_t hm = 1 << (2*il);
223 + y[ 0] = __hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1);
224 + y[ 1] = __hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1);
225 + hm <<= 1;
226 + y[32] = __hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2);
227 + y[33] = __hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2);
228 +}
229 +
230 +template<typename dst_t>
231 +static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
232 + const block_q6_K * x = (const block_q6_K *) vx;
233 +
234 + const int i = blockIdx.x;
235 +
236 + // assume 64 threads - this is very slightly better than the one below
237 + const int tid = threadIdx.x;
238 + const int ip = tid/32; // ip is 0 or 1
239 + const int il = tid - 32*ip; // 0...32
240 + const int is = 8*ip + il/16;
241 +
242 + dst_t * y = yy + i*QK_K + 128*ip + il;
243 +
244 + const half d = x[i].d;
245 +
246 + const uint8_t * ql = x[i].ql + 64*ip + il;
247 + const uint8_t qh = x[i].qh[32*ip + il];
248 + const int8_t * sc = x[i].scales + is;
249 +
250 + y[ 0] = __hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32)));
251 + y[32] = __hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32)));
252 + y[64] = __hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32)));
253 + y[96] = __hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32)));
254 +}
255 +
256 +template<typename dst_t>
257 +static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
258 +
259 + const int i = blockIdx.x;
260 + const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
261 +
262 + const int tid = threadIdx.x;
263 + const int il = tid/8; // 0...3
264 + const int ib = tid%8; // 0...7
265 + dst_t * y = yy + i*QK_K + 32*ib + 8*il;
266 + const uint16_t * q2 = x[i].qs + 4*ib;
267 + const uint8_t * aux8 = (const uint8_t *)q2;
268 + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);
269 + const uint32_t aux32 = q2[2] | (q2[3] << 16);
270 + const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f;
271 + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
272 + for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
273 +}
274 +
275 +template<typename dst_t>
276 +static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
277 +
278 + const int i = blockIdx.x;
279 + const block_iq2_xs * x = (const block_iq2_xs *) vx;
280 +
281 + const int tid = threadIdx.x;
282 + const int il = tid/8; // 0...3
283 + const int ib = tid%8; // 0...7
284 + dst_t * y = yy + i*QK_K + 32*ib + 8*il;
285 + const uint16_t * q2 = x[i].qs + 4*ib;
286 + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
287 + const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
288 + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
289 + for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
290 +
291 +}
292 +
293 +template<typename dst_t>
294 +static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
295 +
296 + const int i = blockIdx.x;
297 + const block_iq2_s * x = (const block_iq2_s *) vx;
298 +
299 + const int tid = threadIdx.x;
300 + const int il = tid/8; // 0...3
301 + const int ib = tid%8; // 0...7
302 + dst_t * y = yy + i*QK_K + 32*ib + 8*il;
303 + const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));
304 + const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
305 + const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];
306 + for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
307 +}
308 +
309 +template<typename dst_t>
310 +static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
311 +
312 + const int i = blockIdx.x;
313 + const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
314 +
315 + const int tid = threadIdx.x;
316 + const int il = tid/8; // 0...3
317 + const int ib = tid%8; // 0...7
318 + dst_t * y = yy + i*QK_K + 32*ib + 8*il;
319 + const uint8_t * q3 = x[i].qs + 8*ib;
320 + const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;
321 + const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);
322 + const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);
323 + const uint32_t aux32 = gas[0] | (gas[1] << 16);
324 + const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f;
325 + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
326 + for (int j = 0; j < 4; ++j) {
327 + y[j+0] = __float2half(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));
328 + y[j+4] = __float2half(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));
329 + }
330 +}
331 +
332 +template<typename dst_t>
333 +static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
334 +
335 + const int i = blockIdx.x;
336 + const block_iq3_s * x = (const block_iq3_s *) vx;
337 +
338 + const int tid = threadIdx.x;
339 + const int il = tid/8; // 0...3
340 + const int ib = tid%8; // 0...7
341 + dst_t * y = yy + i*QK_K + 32*ib + 8*il;
342 + const uint8_t * qs = x[i].qs + 8*ib;
343 + const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));
344 + const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));
345 + const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f;
346 + const uint8_t signs = x[i].signs[4*ib + il];
347 + for (int j = 0; j < 4; ++j) {
348 + y[j+0] = __float2half(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));
349 + y[j+4] = __float2half(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));
350 + }
351 +}
352 +
353 +template<typename dst_t>
354 +static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
355 +
356 + const int i = blockIdx.x;
357 + const block_iq1_s * x = (const block_iq1_s *) vx;
358 +
359 + const int tid = threadIdx.x;
360 + const int il = tid/8; // 0...3
361 + const int ib = tid%8; // 0...7
362 + dst_t * y = yy + i*QK_K + 32*ib + 8*il;
363 + const int i8 = 4*ib+il;
364 + uint8_t h = x[i].scales[i8/2] >> 4*(i8%2);
365 + const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5)));
366 + const float d = __half2float(x[i].d) * (2*(h & 7) + 1);
367 + for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j]);
368 +}
369 +
370 +template<typename dst_t>
371 +static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {
372 +
373 + const int i = blockIdx.x;
374 + const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);
375 +
376 + const int tid = threadIdx.x;
377 + const int il = tid/8; // 0...3
378 + const int ib = tid%8; // 0...7
379 + dst_t * y = yy + i*QK_K + 32*ib + 4*il;
380 + const uint8_t * q4 = x[ib].qs + 4*il;
381 + const float d = __half2float(x[ib].d);
382 + for (int j = 0; j < 4; ++j) {
383 + y[j+ 0] = __float2half(d * kvalues_iq4nl[q4[j] & 0xf]);
384 + y[j+16] = __float2half(d * kvalues_iq4nl[q4[j] >> 4]);
385 + }
386 +
387 +}
388 +
389 +template<typename dst_t>
390 +static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
391 + const int i = blockIdx.x;
392 + const block_iq4_xs * x = (const block_iq4_xs *)vx;
393 +
394 + const int tid = threadIdx.x;
395 + const int il = tid/8; // 0...3
396 + const int ib = tid%8; // 0...7
397 + dst_t * y = yy + i*QK_K + 32*ib + 4*il;
398 + const uint8_t * q4 = x[i].qs + 16*ib + 4*il;
399 + const float d = __half2float(x[i].d) * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);
400 + for (int j = 0; j < 4; ++j) {
401 + y[j+ 0] = __float2half(d * kvalues_iq4nl[q4[j] & 0xf]);
402 + y[j+16] = __float2half(d * kvalues_iq4nl[q4[j] >> 4]);
403 + }
404 +}
405 +
406 +template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
407 +static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {
408 + const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);
409 + dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
410 +}
411 +
412 +template<typename dst_t>
413 +static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
414 + const int nb = k / QK_K;
415 + dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
416 +}
417 +
418 +template<typename dst_t>
419 +static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
420 + const int nb = k / QK_K;
421 + dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
422 +}
423 +
424 +template<typename dst_t>
425 +static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
426 + const int nb = k / QK_K;
427 + dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
428 +}
429 +
430 +template<typename dst_t>
431 +static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
432 + const int nb = k / QK_K;
433 + dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
434 +}
435 +
436 +template<typename dst_t>
437 +static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
438 + const int nb = k / QK_K;
439 + dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
440 +}
441 +
442 +template<typename dst_t>
443 +static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
444 + const int nb = k / QK_K;
445 + dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
446 +}
447 +
448 +template<typename dst_t>
449 +static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
450 + const int nb = k / QK_K;
451 + dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
452 +}
453 +
454 +template<typename dst_t>
455 +static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
456 + const int nb = k / QK_K;
457 + dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
458 +}
459 +
460 +template<typename dst_t>
461 +static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
462 + const int nb = k / QK_K;
463 + dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
464 +}
465 +
466 +template<typename dst_t>
467 +static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
468 + const int nb = k / QK_K;
469 + dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
470 +}
471 +
472 +template<typename dst_t>
473 +static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
474 + const int nb = k / QK_K;
475 + dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
476 +}
477 +
478 +template<typename dst_t>
479 +static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
480 + const int nb = (k + QK_K - 1) / QK_K;
481 + dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);
482 +}
483 +
484 +template<typename dst_t>
485 +static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
486 + const int nb = (k + QK_K - 1) / QK_K;
487 + dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
488 +}
489 +
490 +static to_fp16_cuda_t ggml_get_to_fp16_cuda(int type) {
491 + switch (type) {
492 + case 2:
493 + return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
494 + case 3:
495 + return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
496 + case 6:
497 + return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
498 + case 7:
499 + return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
500 + case 8:
501 + return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
502 + case 10:
503 + return dequantize_row_q2_K_cuda;
504 + case 11:
505 + return dequantize_row_q3_K_cuda;
506 + case 12:
507 + return dequantize_row_q4_K_cuda;
508 + case 13:
509 + return dequantize_row_q5_K_cuda;
510 + case 14:
511 + return dequantize_row_q6_K_cuda;
512 + case 16:
513 + return dequantize_row_iq2_xxs_cuda;
514 + case 17:
515 + return dequantize_row_iq2_xs_cuda;
516 + case 18:
517 + return dequantize_row_iq3_xxs_cuda;
518 + case 19:
519 + return dequantize_row_iq1_s_cuda;
520 + case 20:
521 + return dequantize_row_iq4_nl_cuda;
522 + case 21:
523 + return dequantize_row_iq3_s_cuda;
524 + case 22:
525 + return dequantize_row_iq2_s_cuda;
526 + case 23:
527 + return dequantize_row_iq4_xs_cuda;
528 + default:
529 + return nullptr;
530 + }
531 +}
\ No newline at end of file
csrc/quantization/gguf/ggml-common.h new
+969
@@ -0,0 +1,969 @@
1 +// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
2 +#define QK_K 256
3 +#define K_QUANTS_PER_ITERATION 2
4 +#define WARP_SIZE 32
5 +#define K_SCALE_SIZE 12
6 +#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
7 +#define CUDA_QUANTIZE_BLOCK_SIZE 256
8 +#define GGML_CUDA_DMMV_X 32
9 +#define GGML_CUDA_MMV_Y 1
10 +
11 +
12 +// Data Structures
13 +// QK = number of values after dequantization
14 +// QR = QK / number of values before dequantization
15 +// QI = number of 32 bit integers before dequantization
16 +
17 +#define QK4_0 32
18 +#define QR4_0 2
19 +#define QI4_0 (QK4_0 / (4 * QR4_0))
20 +typedef struct {
21 + half d; // delta
22 + uint8_t qs[QK4_0 / 2]; // nibbles / quants
23 +} block_q4_0;
24 +
25 +#define QK4_1 32
26 +#define QR4_1 2
27 +#define QI4_1 (QK4_1 / (4 * QR4_1))
28 +typedef struct {
29 + half2 dm; // dm.x = delta, dm.y = min
30 + uint8_t qs[QK4_1 / 2]; // nibbles / quants
31 +} block_q4_1;
32 +
33 +#define QK5_0 32
34 +#define QR5_0 2
35 +#define QI5_0 (QK5_0 / (4 * QR5_0))
36 +typedef struct {
37 + half d; // delta
38 + uint8_t qh[4]; // 5-th bit of quants
39 + uint8_t qs[QK5_0 / 2]; // nibbles / quants
40 +} block_q5_0;
41 +
42 +#define QK5_1 32
43 +#define QR5_1 2
44 +#define QI5_1 (QK5_1 / (4 * QR5_1))
45 +typedef struct {
46 + half2 dm; // dm.x = delta, dm.y = min
47 + uint8_t qh[4]; // 5-th bit of quants
48 + uint8_t qs[QK5_1 / 2]; // nibbles / quants
49 +} block_q5_1;
50 +
51 +#define QK8_0 32
52 +#define QR8_0 1
53 +#define QI8_0 (QK8_0 / (4 * QR8_0))
54 +typedef struct {
55 + half d; // delta
56 + int8_t qs[QK8_0]; // quants
57 +} block_q8_0;
58 +
59 +#define QK8_1 32
60 +#define QR8_1 1
61 +#define QI8_1 (QK8_1 / (4 * QR8_1))
62 +typedef struct {
63 + half2 ds; // ds.x = delta, ds.y = sum
64 + int8_t qs[QK8_0]; // quants
65 +} block_q8_1;
66 +
67 +#define QR2_K 4
68 +#define QI2_K (QK_K / (4*QR2_K))
69 +typedef struct {
70 + uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
71 + uint8_t qs[QK_K/4]; // quants
72 + half2 dm; // super-block scale for quantized scales/mins
73 +} block_q2_K;
74 +
75 +#define QR3_K 4
76 +#define QI3_K (QK_K / (4*QR3_K))
77 +typedef struct {
78 + uint8_t hmask[QK_K/8]; // quants - high bit
79 + uint8_t qs[QK_K/4]; // quants - low 2 bits
80 + uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits
81 + half d; // super-block scale
82 +} block_q3_K;
83 +
84 +#define QR4_K 2
85 +#define QI4_K (QK_K / (4*QR4_K))
86 +typedef struct {
87 + half2 dm; // super-block scale for quantized scales/mins
88 + uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits
89 + uint8_t qs[QK_K/2]; // 4--bit quants
90 +} block_q4_K;
91 +
92 +#define QR5_K 2
93 +#define QI5_K (QK_K / (4*QR5_K))
94 +typedef struct {
95 + half2 dm; // super-block scale for quantized scales/mins
96 + uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
97 + uint8_t qh[QK_K/8]; // quants, high bit
98 + uint8_t qs[QK_K/2]; // quants, low 4 bits
99 +} block_q5_K;
100 +
101 +#define QR6_K 2
102 +#define QI6_K (QK_K / (4*QR6_K))
103 +typedef struct {
104 + uint8_t ql[QK_K/2]; // quants, lower 4 bits
105 + uint8_t qh[QK_K/4]; // quants, upper 2 bits
106 + int8_t scales[QK_K/16]; // scales
107 + half d; // delta
108 +} block_q6_K;
109 +
110 +#define QR2_XXS 8
111 +#define QI2_XXS (QK_K / (4*QR2_XXS))
112 +typedef struct {
113 + half d;
114 + uint16_t qs[QK_K/8];
115 +} block_iq2_xxs;
116 +
117 +#define QR2_XS 8
118 +#define QI2_XS (QK_K / (4*QR2_XS))
119 +typedef struct {
120 + half d;
121 + uint16_t qs[QK_K/8];
122 + uint8_t scales[QK_K/32];
123 +} block_iq2_xs;
124 +
125 +#define QR2_S 8
126 +#define QI2_S (QK_K / (4*QR2_S))
127 +typedef struct {
128 + half d;
129 + uint8_t qs[QK_K/4];
130 + uint8_t qh[QK_K/32];
131 + uint8_t scales[QK_K/32];
132 +} block_iq2_s;
133 +
134 +#define QR3_XXS 8
135 +#define QI3_XXS (QK_K / (4*QR3_XXS))
136 +typedef struct {
137 + half d;
138 + uint8_t qs[3*(QK_K/8)];
139 +} block_iq3_xxs;
140 +
141 +#define QR3_XS 8
142 +#define QI3_XS (QK_K / (4*QR3_XS))
143 +#define IQ3S_N_SCALE QK_K/64
144 +typedef struct {
145 + half d;
146 + uint8_t qs[QK_K/4];
147 + uint8_t qh[QK_K/32];
148 + uint8_t signs[QK_K/8];
149 + uint8_t scales[IQ3S_N_SCALE];
150 +} block_iq3_s;
151 +
152 +#define QR1_S 8
153 +#define QI1_S (QK_K / (4*QR1_S))
154 +typedef struct {
155 + half d;
156 + uint8_t qs[QK_K/8];
157 + uint8_t scales[QK_K/16];
158 +} block_iq1_s;
159 +
160 +#define QK4_NL 32
161 +#define QR4_NL 2
162 +#define QI4_NL (QK4_NL / (4*QR4_NL))
163 +typedef struct {
164 + half d;
165 + uint8_t qs[QK4_NL/2];
166 +} block_iq4_nl;
167 +
168 +#define QR4_XS 8
169 +#define QI4_XS (QK_K / (4*QR4_XS))
170 +typedef struct {
171 + half d;
172 + uint16_t scales_h;
173 + uint8_t scales_l[QK_K/64];
174 + uint8_t qs[QK_K/2];
175 +} block_iq4_xs;
176 +
177 +static const __device__ uint64_t iq2xxs_grid[256] = {
178 + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
179 + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,
180 + 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,
181 + 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,
182 + 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,
183 + 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,
184 + 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,
185 + 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,
186 + 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,
187 + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,
188 + 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,
189 + 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,
190 + 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,
191 + 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,
192 + 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,
193 + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,
194 + 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,
195 + 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,
196 + 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,
197 + 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,
198 + 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,
199 + 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,
200 + 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,
201 + 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,
202 + 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,
203 + 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,
204 + 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,
205 + 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,
206 + 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,
207 + 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,
208 + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,
209 + 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,
210 + 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,
211 + 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,
212 + 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,
213 + 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,
214 + 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,
215 + 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,
216 + 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,
217 + 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,
218 + 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,
219 + 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,
220 + 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,
221 + 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,
222 + 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,
223 + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,
224 + 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,
225 + 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,
226 + 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,
227 + 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,
228 + 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,
229 + 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,
230 + 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,
231 + 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,
232 + 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,
233 + 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,
234 + 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,
235 + 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,
236 + 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,
237 + 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,
238 + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,
239 + 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,
240 + 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,
241 + 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,
242 +};
243 +
244 +static const __device__ uint64_t iq2xs_grid[512] = {
245 + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
246 + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
247 + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
248 + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
249 + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
250 + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,
251 + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,
252 + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,
253 + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,
254 + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,
255 + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,
256 + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,
257 + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,
258 + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,
259 + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,
260 + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,
261 + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,
262 + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,
263 + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,
264 + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,
265 + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,
266 + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,
267 + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,
268 + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,
269 + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,
270 + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,
271 + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,
272 + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,
273 + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,
274 + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,
275 + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,
276 + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,
277 + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,
278 + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,
279 + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,
280 + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,
281 + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,
282 + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,
283 + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,
284 + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,
285 + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,
286 + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,
287 + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,
288 + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,
289 + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,
290 + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,
291 + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,
292 + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,
293 + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,
294 + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,
295 + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,
296 + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,
297 + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,
298 + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,
299 + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,
300 + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,
301 + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,
302 + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,
303 + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,
304 + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,
305 + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,
306 + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,
307 + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,
308 + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,
309 + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,
310 + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,
311 + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,
312 + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,
313 + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,
314 + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,
315 + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,
316 + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,
317 + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,
318 + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,
319 + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,
320 + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,
321 + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,
322 + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,
323 + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,
324 + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,
325 + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,
326 + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,
327 + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,
328 + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,
329 + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,
330 + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,
331 + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,
332 + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,
333 + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,
334 + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,
335 + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,
336 + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,
337 + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,
338 + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,
339 + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,
340 + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,
341 + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,
342 + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,
343 + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,
344 + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,
345 + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,
346 + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,
347 + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,
348 + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,
349 + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,
350 + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,
351 + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,
352 + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,
353 + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,
354 + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,
355 + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,
356 + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,
357 + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,
358 + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,
359 + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,
360 + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,
361 + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,
362 + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,
363 + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,
364 + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,
365 + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,
366 + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,
367 + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,
368 + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,
369 + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,
370 + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,
371 + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,
372 + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,
373 +};
374 +
375 +static const __device__ uint64_t iq2s_grid[1024] = {
376 + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
377 + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
378 + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
379 + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
380 + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
381 + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,
382 + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,
383 + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,
384 + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,
385 + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
386 + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,
387 + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,
388 + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,
389 + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,
390 + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
391 + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,
392 + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,
393 + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,
394 + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,
395 + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
396 + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,
397 + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,
398 + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,
399 + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,
400 + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
401 + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,
402 + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,
403 + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,
404 + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,
405 + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
406 + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,
407 + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,
408 + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,
409 + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,
410 + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
411 + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,
412 + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,
413 + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,
414 + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,
415 + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
416 + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,
417 + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,
418 + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,
419 + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,
420 + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
421 + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,
422 + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,
423 + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,
424 + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,
425 + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
426 + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,
427 + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,
428 + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,
429 + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,
430 + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
431 + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,
432 + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,
433 + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,
434 + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,
435 + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
436 + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,
437 + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,
438 + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,
439 + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,
440 + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
441 + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,
442 + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,
443 + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,
444 + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,
445 + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
446 + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,
447 + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,
448 + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,
449 + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,
450 + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
451 + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,
452 + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,
453 + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,
454 + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,
455 + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
456 + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,
457 + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,
458 + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,
459 + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,
460 + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
461 + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,
462 + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,
463 + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,
464 + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,
465 + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
466 + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,
467 + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,
468 + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,
469 + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,
470 + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
471 + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,
472 + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,
473 + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,
474 + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,
475 + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
476 + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,
477 + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,
478 + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,
479 + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,
480 + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
481 + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,
482 + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,
483 + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,
484 + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,
485 + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
486 + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,
487 + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,
488 + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,
489 + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,
490 + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
491 + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,
492 + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,
493 + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
494 + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,
495 + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
496 + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,
497 + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,
498 + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,
499 + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,
500 + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
501 + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,
502 + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,
503 + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,
504 + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,
505 + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
506 + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,
507 + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,
508 + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,
509 + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,
510 + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
511 + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,
512 + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,
513 + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,
514 + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,
515 + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
516 + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,
517 + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,
518 + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,
519 + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,
520 + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
521 + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,
522 + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,
523 + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,
524 + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,
525 + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
526 + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,
527 + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,
528 + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,
529 + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,
530 + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
531 + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,
532 + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,
533 + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,
534 + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,
535 + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
536 + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,
537 + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,
538 + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,
539 + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,
540 + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
541 + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,
542 + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,
543 + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,
544 + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,
545 + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
546 + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,
547 + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,
548 + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,
549 + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,
550 + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
551 + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,
552 + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,
553 + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,
554 + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,
555 + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
556 + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,
557 + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,
558 + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,
559 + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,
560 + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
561 + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,
562 + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,
563 + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,
564 + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,
565 + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
566 + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,
567 + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,
568 + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,
569 + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,
570 + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
571 + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,
572 + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,
573 + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,
574 + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,
575 + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
576 + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,
577 + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,
578 + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,
579 + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,
580 + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
581 + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
582 + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,
583 + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,
584 + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,
585 + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
586 + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,
587 + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,
588 + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,
589 + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,
590 + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
591 + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,
592 + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,
593 + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,
594 + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,
595 + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
596 + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,
597 + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,
598 + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,
599 + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,
600 + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
601 + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,
602 + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,
603 + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,
604 + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,
605 + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
606 + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,
607 + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,
608 + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,
609 + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,
610 + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
611 + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,
612 + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,
613 + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,
614 + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,
615 + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
616 + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,
617 + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,
618 + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,
619 + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,
620 + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
621 + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,
622 + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,
623 + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,
624 + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,
625 + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
626 + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,
627 + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,
628 + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,
629 + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,
630 + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
631 + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
632 +};
633 +
634 +static const __device__ uint32_t iq3xxs_grid[256] = {
635 + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
636 + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
637 + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
638 + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
639 + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
640 + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
641 + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
642 + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
643 + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
644 + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
645 + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
646 + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
647 + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
648 + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
649 + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
650 + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
651 + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
652 + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
653 + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
654 + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
655 + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
656 + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
657 + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
658 + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
659 + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
660 + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
661 + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
662 + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
663 + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
664 + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
665 + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
666 + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,
667 +};
668 +
669 +static const __device__ uint32_t iq3xs_grid[512] = {
670 + 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14,
671 + 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414,
672 + 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24,
673 + 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c,
674 + 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c,
675 + 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34,
676 + 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c,
677 + 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414,
678 + 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,
679 + 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404,
680 + 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434,
681 + 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c,
682 + 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404,
683 + 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414,
684 + 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414,
685 + 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404,
686 + 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c,
687 + 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,
688 + 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404,
689 + 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e,
690 + 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14,
691 + 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c,
692 + 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424,
693 + 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c,
694 + 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c,
695 + 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e,
696 + 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,
697 + 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e,
698 + 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424,
699 + 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e,
700 + 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424,
701 + 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404,
702 + 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c,
703 + 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e,
704 + 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c,
705 + 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,
706 + 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c,
707 + 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404,
708 + 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04,
709 + 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c,
710 + 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414,
711 + 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c,
712 + 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c,
713 + 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424,
714 + 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,
715 + 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c,
716 + 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414,
717 + 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c,
718 + 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e,
719 + 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04,
720 + 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424,
721 + 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14,
722 + 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34,
723 + 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,
724 + 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434,
725 + 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c,
726 + 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424,
727 + 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24,
728 + 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24,
729 + 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e,
730 + 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c,
731 + 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c,
732 + 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,
733 + 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,
734 +};
735 +
736 +static const __device__ uint64_t iq1s_grid[512] = {
737 + 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,
738 + 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,
739 + 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,
740 + 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,
741 + 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,
742 + 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,
743 + 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,
744 + 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,
745 + 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,
746 + 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,
747 + 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,
748 + 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,
749 + 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,
750 + 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,
751 + 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,
752 + 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,
753 + 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,
754 + 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,
755 + 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,
756 + 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,
757 + 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,
758 + 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,
759 + 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,
760 + 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,
761 + 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,
762 + 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,
763 + 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,
764 + 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,
765 + 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,
766 + 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,
767 + 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,
768 + 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,
769 + 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,
770 + 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,
771 + 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,
772 + 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,
773 + 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,
774 + 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,
775 + 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,
776 + 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,
777 + 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,
778 + 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,
779 + 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,
780 + 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,
781 + 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,
782 + 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,
783 + 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,
784 + 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,
785 + 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,
786 + 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,
787 + 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,
788 + 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,
789 + 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,
790 + 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,
791 + 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,
792 + 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,
793 + 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,
794 + 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,
795 + 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,
796 + 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,
797 + 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,
798 + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,
799 + 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,
800 + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,
801 + 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,
802 + 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,
803 + 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,
804 + 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,
805 + 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,
806 + 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,
807 + 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,
808 + 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,
809 + 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,
810 + 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,
811 + 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,
812 + 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,
813 + 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,
814 + 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,
815 + 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,
816 + 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,
817 + 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,
818 + 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,
819 + 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,
820 + 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,
821 + 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,
822 + 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,
823 + 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,
824 + 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,
825 + 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,
826 + 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,
827 + 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,
828 + 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,
829 + 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,
830 + 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,
831 + 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,
832 + 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,
833 + 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,
834 + 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,
835 + 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,
836 + 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,
837 + 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,
838 + 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,
839 + 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,
840 + 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,
841 + 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,
842 + 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,
843 + 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,
844 + 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,
845 + 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,
846 + 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,
847 + 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,
848 + 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,
849 + 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,
850 + 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,
851 + 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,
852 + 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,
853 + 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,
854 + 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,
855 + 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,
856 + 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,
857 + 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,
858 + 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,
859 + 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,
860 + 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,
861 + 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,
862 + 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,
863 + 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,
864 + 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,
865 +};
866 +
867 +static const __device__ uint8_t ksigns_iq2xs[128] = {
868 + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,
869 + 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,
870 + 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,
871 + 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,
872 + 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,
873 + 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,
874 + 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,
875 + 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
876 +};
877 +
878 +static const __device__ uint64_t ksigns64[128] = {
879 + 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
880 + 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,
881 + 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,
882 + 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,
883 + 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,
884 + 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,
885 + 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,
886 + 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,
887 + 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,
888 + 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,
889 + 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,
890 + 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,
891 + 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,
892 + 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,
893 + 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,
894 + 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,
895 + 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,
896 + 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,
897 + 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,
898 + 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,
899 + 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,
900 + 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,
901 + 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,
902 + 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,
903 + 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,
904 + 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,
905 + 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,
906 + 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,
907 + 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,
908 + 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,
909 + 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,
910 + 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
911 +};
912 +
913 +static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};
914 +static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
915 +
916 +
917 +typedef half dfloat; // dequantize float
918 +typedef half2 dfloat2;
919 +typedef void (*dequantize_kernel_t)(const void * vx, const int ib, const int iqs, dfloat2 & v);
920 +typedef void (*to_fp16_cuda_t)(const void * __restrict__ x, dfloat * __restrict__ y, int k, cudaStream_t stream);
921 +typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs);
922 +typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc);
923 +typedef void (*load_tiles_cuda_t)(
924 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
925 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row);
926 +typedef float (*vec_dot_q_mul_mat_cuda_t)(
927 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
928 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k);
929 +
930 +// Utility function
931 +
932 +#if defined(USE_ROCM)
933 +
934 +#ifndef __has_builtin
935 + #define __has_builtin(x) 0
936 +#endif
937 +
938 +typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
939 +static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
940 + const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
941 + const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
942 +#if __has_builtin(__builtin_elementwise_sub_sat)
943 + const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);
944 + return reinterpret_cast<const int &>(c);
945 +#else
946 + int8x4_t c;
947 + int16_t tmp;
948 +#pragma unroll
949 + for (int i = 0; i < 4; i++) {
950 + tmp = va[i] - vb[i];
951 + if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();
952 + if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();
953 + c[i] = tmp;
954 + }
955 + return reinterpret_cast<int &>(c);
956 +#endif // __has_builtin(__builtin_elementwise_sub_sat)
957 +}
958 +
959 +static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
960 +#if __has_builtin(__builtin_amdgcn_sdot4)
961 + c = __builtin_amdgcn_sdot4(a, b, c, false);
962 +#else
963 + const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
964 + const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
965 + c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];
966 +#endif
967 + return c;
968 +}
969 +#endif // defined(USE_ROCM)
csrc/quantization/gguf/gguf_kernel.cu new
+242
@@ -0,0 +1,242 @@
1 +#include <cuda_fp16.h>
2 +#include <cuda_runtime.h>
3 +
4 +#include <torch/all.h>
5 +#include <c10/cuda/CUDAGuard.h>
6 +
7 +#include "ggml-common.h"
8 +#include "vecdotq.cuh"
9 +#include "dequantize.cuh"
10 +#include "mmvq.cuh"
11 +#include "mmq.cuh"
12 +
13 +// Q8 gemv
14 +static __global__ void quantize_q8_1(const half* __restrict__ x,
15 + void* __restrict__ vy, const int kx,
16 + const int kx_padded) {
17 + const int ix = blockDim.x * blockIdx.x + threadIdx.x;
18 + if (ix >= kx_padded) {
19 + return;
20 + }
21 + const int iy = blockDim.y * blockIdx.y + threadIdx.y;
22 + const int i_padded = iy * kx_padded + ix;
23 +
24 + block_q8_1* y = (block_q8_1*)vy;
25 +
26 + const int ib = i_padded / QK8_1; // block index
27 + const int iqs = i_padded % QK8_1; // quant index
28 +
29 + const float xi = ix < kx ? __half2float(x[iy * kx + ix]) : 0.0f;
30 + float amax = fabsf(xi);
31 + float sum = xi;
32 +
33 +#pragma unroll
34 + for (int mask = 16; mask > 0; mask >>= 1) {
35 + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, mask, 32));
36 + sum += __shfl_xor_sync(0xffffffff, sum, mask, 32);
37 + }
38 +
39 + const float d = amax / 127;
40 + const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
41 +
42 + y[ib].qs[iqs] = q;
43 +
44 + if (iqs > 0) {
45 + return;
46 + }
47 +
48 + y[ib].ds.x = __float2half(d);
49 + y[ib].ds.y = __float2half(sum);
50 +}
51 +
52 +static void quantize_row_q8_1_cuda(const half* x, void* vy, const int kx,
53 + const int ky, cudaStream_t stream) {
54 + const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
55 + const int block_num_x =
56 + (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
57 + const dim3 num_blocks(block_num_x, ky, 1);
58 + const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
59 + quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);
60 +}
61 +
62 +torch::Tensor ggml_dequantize(torch::Tensor W, // quant weight
63 + int8_t type, int64_t m, int64_t n) {
64 + const at::cuda::OptionalCUDAGuard device_guard(device_of(W));
65 + auto options =
66 + torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
67 + at::Tensor DW = torch::empty({m, n}, options);
68 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
69 + const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(type);
70 + to_fp16_cuda((void*)W.data_ptr(), (half*)DW.data_ptr(), m * n, stream);
71 + return DW;
72 +}
73 +
74 +torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, // quant weight
75 + torch::Tensor X, // input
76 + int8_t type, int64_t row) {
77 + int col = X.sizes()[1];
78 + const int padded = (col + 512 - 1) / 512 * 512;
79 + const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
80 + auto options =
81 + torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
82 + at::Tensor Y = torch::empty({1, row}, options);
83 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
84 + options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
85 + at::Tensor quant_X = torch::empty({1, padded / 32 * 9}, options);
86 + quantize_row_q8_1_cuda((half*)X.data_ptr(), (void*)quant_X.data_ptr(), col, 1,
87 + stream);
88 + switch (type) {
89 + case 2:
90 + mul_mat_vec_q4_0_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
91 + (half*)Y.data_ptr(), col, row, stream);
92 + break;
93 + case 3:
94 + mul_mat_vec_q4_1_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
95 + (half*)Y.data_ptr(), col, row, stream);
96 + break;
97 + case 6:
98 + mul_mat_vec_q5_0_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
99 + (half*)Y.data_ptr(), col, row, stream);
100 + break;
101 + case 7:
102 + mul_mat_vec_q5_1_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
103 + (half*)Y.data_ptr(), col, row, stream);
104 + break;
105 + case 8:
106 + mul_mat_vec_q8_0_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
107 + (half*)Y.data_ptr(), col, row, stream);
108 + break;
109 + case 10:
110 + mul_mat_vec_q2_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
111 + (half*)Y.data_ptr(), col, row, stream);
112 + break;
113 + case 11:
114 + mul_mat_vec_q3_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
115 + (half*)Y.data_ptr(), col, row, stream);
116 + break;
117 + case 12:
118 + mul_mat_vec_q4_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
119 + (half*)Y.data_ptr(), col, row, stream);
120 + break;
121 + case 13:
122 + mul_mat_vec_q5_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
123 + (half*)Y.data_ptr(), col, row, stream);
124 + break;
125 + case 14:
126 + mul_mat_vec_q6_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
127 + (half*)Y.data_ptr(), col, row, stream);
128 + break;
129 + case 16:
130 + mul_mat_vec_iq2_xxs_q8_1_cuda((void*)W.data_ptr(),
131 + (void*)quant_X.data_ptr(),
132 + (half*)Y.data_ptr(), col, row, stream);
133 + break;
134 + case 17:
135 + mul_mat_vec_iq2_xs_q8_1_cuda((void*)W.data_ptr(),
136 + (void*)quant_X.data_ptr(),
137 + (half*)Y.data_ptr(), col, row, stream);
138 + break;
139 + case 18:
140 + mul_mat_vec_iq3_xxs_q8_1_cuda((void*)W.data_ptr(),
141 + (void*)quant_X.data_ptr(),
142 + (half*)Y.data_ptr(), col, row, stream);
143 + break;
144 + case 19:
145 + mul_mat_vec_iq1_s_q8_1_cuda((void*)W.data_ptr(),
146 + (void*)quant_X.data_ptr(),
147 + (half*)Y.data_ptr(), col, row, stream);
148 + break;
149 + case 20:
150 + mul_mat_vec_iq4_nl_q8_1_cuda((void*)W.data_ptr(),
151 + (void*)quant_X.data_ptr(),
152 + (half*)Y.data_ptr(), col, row, stream);
153 + break;
154 + case 21:
155 + mul_mat_vec_iq3_s_q8_1_cuda((void*)W.data_ptr(),
156 + (void*)quant_X.data_ptr(),
157 + (half*)Y.data_ptr(), col, row, stream);
158 + break;
159 + case 22:
160 + mul_mat_vec_iq2_s_q8_1_cuda((void*)W.data_ptr(),
161 + (void*)quant_X.data_ptr(),
162 + (half*)Y.data_ptr(), col, row, stream);
163 + break;
164 + case 23:
165 + mul_mat_vec_iq4_xs_q8_1_cuda((void*)W.data_ptr(),
166 + (void*)quant_X.data_ptr(),
167 + (half*)Y.data_ptr(), col, row, stream);
168 + break;
169 + }
170 + return Y;
171 +}
172 +
173 +torch::Tensor ggml_mul_mat_a8(torch::Tensor W, // quant weight
174 + torch::Tensor X, // input
175 + int8_t type, int64_t row) {
176 + int col = X.sizes()[1];
177 + int padded = (col + 512 - 1) / 512 * 512;
178 + int batch = X.sizes()[0];
179 + const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
180 + auto options =
181 + torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
182 + at::Tensor Y = torch::empty({batch, row}, options);
183 + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
184 + options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
185 + at::Tensor quant_X = torch::empty({batch, padded / 32 * 9}, options);
186 + quantize_row_q8_1_cuda((half*)X.data_ptr(), (void*)quant_X.data_ptr(), col,
187 + batch, stream);
188 +
189 + switch (type) {
190 + case 2:
191 + ggml_mul_mat_q4_0_q8_1_cuda(
192 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
193 + col, row, batch, padded, row, stream);
194 + break;
195 + case 3:
196 + ggml_mul_mat_q4_1_q8_1_cuda(
197 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
198 + col, row, batch, padded, row, stream);
199 + break;
200 + case 6:
201 + ggml_mul_mat_q5_0_q8_1_cuda(
202 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
203 + col, row, batch, padded, row, stream);
204 + break;
205 + case 7:
206 + ggml_mul_mat_q5_1_q8_1_cuda(
207 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
208 + col, row, batch, padded, row, stream);
209 + break;
210 + case 8:
211 + ggml_mul_mat_q8_0_q8_1_cuda(
212 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
213 + col, row, batch, padded, row, stream);
214 + break;
215 + case 10:
216 + ggml_mul_mat_q2_K_q8_1_cuda(
217 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
218 + col, row, batch, padded, row, stream);
219 + break;
220 + case 11:
221 + ggml_mul_mat_q3_K_q8_1_cuda(
222 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
223 + col, row, batch, padded, row, stream);
224 + break;
225 + case 12:
226 + ggml_mul_mat_q4_K_q8_1_cuda(
227 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
228 + col, row, batch, padded, row, stream);
229 + break;
230 + case 13:
231 + ggml_mul_mat_q5_K_q8_1_cuda(
232 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
233 + col, row, batch, padded, row, stream);
234 + break;
235 + case 14:
236 + ggml_mul_mat_q6_K_q8_1_cuda(
237 + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
238 + col, row, batch, padded, row, stream);
239 + break;
240 + }
241 + return Y;
242 +}
\ No newline at end of file
csrc/quantization/gguf/mmq.cuh new
+600
@@ -0,0 +1,600 @@
1 +// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
2 +template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,
3 + allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>
4 +static __device__ __forceinline__ void mul_mat_q(
5 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
6 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
7 +
8 + const block_q_t * x = (const block_q_t *) vx;
9 + const block_q8_1 * y = (const block_q8_1 *) vy;
10 +
11 + const int blocks_per_row_x = ncols_x / qk;
12 + const int blocks_per_col_y = nrows_y / QK8_1;
13 + const int blocks_per_warp = WARP_SIZE / qi;
14 +
15 + const int & ncols_dst = ncols_y;
16 +
17 + const int row_dst_0 = blockIdx.x*mmq_y;
18 + const int & row_x_0 = row_dst_0;
19 +
20 + const int col_dst_0 = blockIdx.y*mmq_x;
21 + const int & col_y_0 = col_dst_0;
22 +
23 + int * tile_x_ql = nullptr;
24 + half2 * tile_x_dm = nullptr;
25 + int * tile_x_qh = nullptr;
26 + int * tile_x_sc = nullptr;
27 +
28 + allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
29 +
30 + __shared__ int tile_y_qs[mmq_x * WARP_SIZE];
31 + __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];
32 +
33 + float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};
34 +
35 + for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
36 +
37 + load_tiles(x + row_x_0*blocks_per_row_x + ib0, tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc,
38 + threadIdx.y, nrows_x-row_x_0-1, threadIdx.x, blocks_per_row_x);
39 +
40 +#pragma unroll
41 + for (int ir = 0; ir < qr; ++ir) {
42 + const int kqs = ir*WARP_SIZE + threadIdx.x;
43 + const int kbxd = kqs / QI8_1;
44 +
45 +#pragma unroll
46 + for (int i = 0; i < mmq_x; i += nwarps) {
47 + const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses
48 + const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];
49 + const int index_y = (threadIdx.y + i) * WARP_SIZE + kqs % WARP_SIZE;
50 + tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
51 + }
52 +
53 +#pragma unroll
54 + for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
55 + const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE/QI8_1)) % mmq_x;
56 + const int kby = threadIdx.x % (WARP_SIZE/QI8_1);
57 + const int col_y_eff = min(col_y_0 + ids, ncols_y-1);
58 +
59 + // if the sum is not needed it's faster to transform the scale to f32 ahead of time
60 + const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE/QI8_1) + kby].ds;
61 + half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE/QI8_1) + kby];
62 + if (need_sum) {
63 + *dsi_dst = *dsi_src;
64 + } else {
65 + float * dfi_dst = (float *) dsi_dst;
66 + *dfi_dst = __low2float(*dsi_src);
67 + }
68 + }
69 +
70 + __syncthreads();
71 +
72 +// #pragma unroll // unrolling this loop causes too much register pressure
73 + for (int k = ir*WARP_SIZE/qr; k < (ir+1)*WARP_SIZE/qr; k += vdr) {
74 +#pragma unroll
75 + for (int j = 0; j < mmq_x; j += nwarps) {
76 +#pragma unroll
77 + for (int i = 0; i < mmq_y; i += WARP_SIZE) {
78 + sum[i/WARP_SIZE][j/nwarps] += vec_dot(
79 + tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,
80 + threadIdx.x + i, threadIdx.y + j, k);
81 + }
82 + }
83 + }
84 + __syncthreads();
85 + }
86 + }
87 +
88 +#pragma unroll
89 + for (int j = 0; j < mmq_x; j += nwarps) {
90 + const int col_dst = col_dst_0 + j + threadIdx.y;
91 + if (col_dst >= ncols_dst) {
92 + return;
93 + }
94 +
95 +#pragma unroll
96 + for (int i = 0; i < mmq_y; i += WARP_SIZE) {
97 + const int row_dst = row_dst_0 + threadIdx.x + i;
98 + if (row_dst >= nrows_dst) {
99 + continue;
100 + }
101 + dst[col_dst*nrows_dst + row_dst] = __float2half(sum[i/WARP_SIZE][j/nwarps]);
102 + }
103 + }
104 +}
105 +
106 +#if defined(USE_ROCM)
107 +#define MMQ_X_Q4_0 64
108 +#define MMQ_Y_Q4_0 128
109 +#define NWARPS_Q4_0 8
110 +#else
111 +#define MMQ_X_Q4_0 4
112 +#define MMQ_Y_Q4_0 32
113 +#define NWARPS_Q4_0 4
114 +#endif
115 +
116 +template <bool need_check> static __global__ void
117 +#if defined(USE_ROCM)
118 +__launch_bounds__(WARP_SIZE*NWARPS_Q4_0, 2)
119 +#endif
120 +mul_mat_q4_0(
121 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
122 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
123 + const int mmq_x = MMQ_X_Q4_0;
124 + const int mmq_y = MMQ_Y_Q4_0;
125 + const int nwarps = NWARPS_Q4_0;
126 +
127 + mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,
128 + load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>
129 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
130 +}
131 +
132 +static void ggml_mul_mat_q4_0_q8_1_cuda(
133 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
134 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
135 +
136 + int mmq_x = MMQ_X_Q4_0;
137 + int mmq_y = MMQ_Y_Q4_0;
138 + int nwarps = NWARPS_Q4_0;
139 +
140 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
141 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
142 + const dim3 block_nums(block_num_x, block_num_y, 1);
143 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
144 +
145 + if (nrows_x % mmq_y == 0) {
146 + const bool need_check = false;
147 + mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>
148 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
149 + } else {
150 + const bool need_check = true;
151 + mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>
152 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
153 + }
154 +}
155 +
156 +#if defined(USE_ROCM)
157 +#define MMQ_X_Q4_1 64
158 +#define MMQ_Y_Q4_1 128
159 +#define NWARPS_Q4_1 8
160 +#else
161 +#define MMQ_X_Q4_1 4
162 +#define MMQ_Y_Q4_1 32
163 +#define NWARPS_Q4_1 4
164 +#endif
165 +
166 +template <bool need_check> static __global__ void
167 +#if defined(USE_ROCM)
168 +__launch_bounds__(WARP_SIZE*NWARPS_Q4_1, 2)
169 +#endif
170 +mul_mat_q4_1(
171 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
172 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
173 + const int mmq_x = MMQ_X_Q4_1;
174 + const int mmq_y = MMQ_Y_Q4_1;
175 + const int nwarps = NWARPS_Q4_1;
176 +
177 + mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,
178 + load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>
179 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
180 +}
181 +
182 +static void ggml_mul_mat_q4_1_q8_1_cuda(
183 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
184 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
185 +
186 + int mmq_x = MMQ_X_Q4_1;
187 + int mmq_y = MMQ_Y_Q4_1;
188 + int nwarps = NWARPS_Q4_1;
189 +
190 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
191 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
192 + const dim3 block_nums(block_num_x, block_num_y, 1);
193 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
194 +
195 + if (nrows_x % mmq_y == 0) {
196 + const bool need_check = false;
197 + mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>
198 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
199 + } else {
200 + const bool need_check = true;
201 + mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>
202 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
203 + }
204 +}
205 +
206 +#if defined(USE_ROCM)
207 +#define MMQ_X_Q5_0 64
208 +#define MMQ_Y_Q5_0 128
209 +#define NWARPS_Q5_0 8
210 +#else
211 +#define MMQ_X_Q5_0 4
212 +#define MMQ_Y_Q5_0 32
213 +#define NWARPS_Q5_0 4
214 +#endif
215 +
216 +template <bool need_check> static __global__ void
217 +#if defined(USE_ROCM)
218 +__launch_bounds__(WARP_SIZE*NWARPS_Q5_0, 2)
219 +#endif
220 +mul_mat_q5_0(
221 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
222 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
223 + const int mmq_x = MMQ_X_Q5_0;
224 + const int mmq_y = MMQ_Y_Q5_0;
225 + const int nwarps = NWARPS_Q5_0;
226 +
227 + mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,
228 + load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>
229 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
230 +}
231 +
232 +static void ggml_mul_mat_q5_0_q8_1_cuda(
233 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
234 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
235 +
236 + const int mmq_x = MMQ_X_Q5_0;
237 + const int mmq_y = MMQ_Y_Q5_0;
238 + const int nwarps = NWARPS_Q5_0;
239 +
240 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
241 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
242 + const dim3 block_nums(block_num_x, block_num_y, 1);
243 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
244 +
245 + if (nrows_x % mmq_y == 0) {
246 + const bool need_check = false;
247 + mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>
248 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
249 + } else {
250 + const bool need_check = true;
251 + mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>
252 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
253 + }
254 +}
255 +
256 +#if defined(USE_ROCM)
257 +#define MMQ_X_Q5_1 64
258 +#define MMQ_Y_Q5_1 128
259 +#define NWARPS_Q5_1 8
260 +#else
261 +#define MMQ_X_Q5_1 4
262 +#define MMQ_Y_Q5_1 32
263 +#define NWARPS_Q5_1 4
264 +#endif
265 +
266 +template <bool need_check> static __global__ void
267 +#if defined(USE_ROCM)
268 +__launch_bounds__(WARP_SIZE*NWARPS_Q5_1, 2)
269 +#endif
270 +mul_mat_q5_1(
271 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
272 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
273 + const int mmq_x = MMQ_X_Q5_1;
274 + const int mmq_y = MMQ_Y_Q5_1;
275 + const int nwarps = NWARPS_Q5_1;
276 +
277 + mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,
278 + load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>
279 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
280 +}
281 +
282 +static void ggml_mul_mat_q5_1_q8_1_cuda(
283 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
284 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
285 + const int mmq_x = MMQ_X_Q5_1;
286 + const int mmq_y = MMQ_Y_Q5_1;
287 + const int nwarps = NWARPS_Q5_1;
288 +
289 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
290 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
291 + const dim3 block_nums(block_num_x, block_num_y, 1);
292 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
293 +
294 + if (nrows_x % mmq_y == 0) {
295 + const bool need_check = false;
296 + mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>
297 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
298 + } else {
299 + const bool need_check = true;
300 + mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>
301 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
302 + }
303 +}
304 +
305 +#if defined(USE_ROCM)
306 +#define MMQ_X_Q8_0 64
307 +#define MMQ_Y_Q8_0 128
308 +#define NWARPS_Q8_0 8
309 +#else
310 +#define MMQ_X_Q8_0 4
311 +#define MMQ_Y_Q8_0 32
312 +#define NWARPS_Q8_0 4
313 +#endif
314 +
315 +template <bool need_check> static __global__ void
316 +#if defined(USE_ROCM)
317 +__launch_bounds__(WARP_SIZE*NWARPS_Q8_0, 2)
318 +#endif
319 +mul_mat_q8_0(
320 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
321 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
322 + const int mmq_x = MMQ_X_Q8_0;
323 + const int mmq_y = MMQ_Y_Q8_0;
324 + const int nwarps = NWARPS_Q8_0;
325 +
326 + mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,
327 + load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>
328 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
329 +}
330 +
331 +static void ggml_mul_mat_q8_0_q8_1_cuda(
332 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
333 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
334 + const int mmq_x = MMQ_X_Q8_0;
335 + const int mmq_y = MMQ_Y_Q8_0;
336 + const int nwarps = NWARPS_Q8_0;
337 +
338 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
339 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
340 + const dim3 block_nums(block_num_x, block_num_y, 1);
341 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
342 +
343 + if (nrows_x % mmq_y == 0) {
344 + const bool need_check = false;
345 + mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>
346 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
347 + } else {
348 + const bool need_check = true;
349 + mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>
350 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
351 + }
352 +}
353 +
354 +#if defined(USE_ROCM)
355 +#define MMQ_X_Q2_K 64
356 +#define MMQ_Y_Q2_K 128
357 +#define NWARPS_Q2_K 8
358 +#else
359 +#define MMQ_X_Q2_K 4
360 +#define MMQ_Y_Q2_K 32
361 +#define NWARPS_Q2_K 4
362 +#endif
363 +
364 +template <bool need_check> static __global__ void
365 +#if defined(USE_ROCM)
366 +__launch_bounds__(WARP_SIZE*NWARPS_Q2_K, 2)
367 +#endif
368 +mul_mat_q2_K(
369 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
370 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
371 + const int mmq_x = MMQ_X_Q2_K;
372 + const int mmq_y = MMQ_Y_Q2_K;
373 + const int nwarps = NWARPS_Q2_K;
374 +
375 + mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,
376 + load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>
377 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
378 +}
379 +
380 +static void ggml_mul_mat_q2_K_q8_1_cuda(
381 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
382 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
383 + const int mmq_x = MMQ_X_Q2_K;
384 + const int mmq_y = MMQ_Y_Q2_K;
385 + const int nwarps = NWARPS_Q2_K;
386 +
387 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
388 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
389 + const dim3 block_nums(block_num_x, block_num_y, 1);
390 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
391 +
392 + if (nrows_x % mmq_y == 0) {
393 + const bool need_check = false;
394 + mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>
395 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
396 + } else {
397 + const bool need_check = true;
398 + mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>
399 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
400 + }
401 +}
402 +
403 +#if defined(USE_ROCM)
404 +#define MMQ_X_Q3_K 64
405 +#define MMQ_Y_Q3_K 128
406 +#define NWARPS_Q3_K 8
407 +#else
408 +#define MMQ_X_Q3_K 4
409 +#define MMQ_Y_Q3_K 32
410 +#define NWARPS_Q3_K 4
411 +#endif
412 +
413 +template <bool need_check> static __global__ void
414 +#if defined(USE_ROCM)
415 +__launch_bounds__(WARP_SIZE*NWARPS_Q3_K, 2)
416 +#endif
417 +mul_mat_q3_K(
418 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
419 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
420 +
421 + const int mmq_x = MMQ_X_Q3_K;
422 + const int mmq_y = MMQ_Y_Q3_K;
423 + const int nwarps = NWARPS_Q3_K;
424 +
425 + mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,
426 + load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>
427 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
428 +}
429 +
430 +static void ggml_mul_mat_q3_K_q8_1_cuda(
431 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
432 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
433 +
434 + const int mmq_x = MMQ_X_Q3_K;
435 + const int mmq_y = MMQ_Y_Q3_K;
436 + const int nwarps = NWARPS_Q3_K;
437 +
438 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
439 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
440 + const dim3 block_nums(block_num_x, block_num_y, 1);
441 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
442 +
443 + if (nrows_x % mmq_y == 0) {
444 + const bool need_check = false;
445 + mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>
446 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
447 + } else {
448 + const bool need_check = true;
449 + mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>
450 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
451 + }
452 +}
453 +
454 +#if defined(USE_ROCM)
455 +#define MMQ_X_Q4_K 64
456 +#define MMQ_Y_Q4_K 128
457 +#define NWARPS_Q4_K 8
458 +#else
459 +#define MMQ_X_Q4_K 4
460 +#define MMQ_Y_Q4_K 32
461 +#define NWARPS_Q4_K 4
462 +#endif
463 +
464 +template <bool need_check> static __global__ void
465 +#if defined(USE_ROCM)
466 +__launch_bounds__(WARP_SIZE*NWARPS_Q4_K, 2)
467 +#endif
468 +mul_mat_q4_K(
469 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
470 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
471 + const int mmq_x = MMQ_X_Q4_K;
472 + const int mmq_y = MMQ_Y_Q4_K;
473 + const int nwarps = NWARPS_Q4_K;
474 +
475 + mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,
476 + load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>
477 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
478 +}
479 +
480 +static void ggml_mul_mat_q4_K_q8_1_cuda(
481 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
482 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
483 + const int mmq_x = MMQ_X_Q4_K;
484 + const int mmq_y = MMQ_Y_Q4_K;
485 + const int nwarps = NWARPS_Q4_K;
486 +
487 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
488 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
489 + const dim3 block_nums(block_num_x, block_num_y, 1);
490 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
491 +
492 + if (nrows_x % mmq_y == 0) {
493 + const bool need_check = false;
494 + mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>
495 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
496 + } else {
497 + const bool need_check = true;
498 + mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>
499 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
500 + }
501 +}
502 +
503 +#if defined(USE_ROCM)
504 +#define MMQ_X_Q5_K 64
505 +#define MMQ_Y_Q5_K 128
506 +#define NWARPS_Q5_K 8
507 +#else
508 +#define MMQ_X_Q5_K 4
509 +#define MMQ_Y_Q5_K 32
510 +#define NWARPS_Q5_K 4
511 +#endif
512 +
513 +template <bool need_check> static __global__ void
514 +#if defined(USE_ROCM)
515 +__launch_bounds__(WARP_SIZE*NWARPS_Q5_K, 2)
516 +#endif
517 +mul_mat_q5_K(
518 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
519 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
520 + const int mmq_x = MMQ_X_Q5_K;
521 + const int mmq_y = MMQ_Y_Q5_K;
522 + const int nwarps = NWARPS_Q5_K;
523 +
524 + mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,
525 + load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>
526 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
527 +}
528 +
529 +static void ggml_mul_mat_q5_K_q8_1_cuda(
530 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
531 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
532 +
533 + const int mmq_x = MMQ_X_Q5_K;
534 + const int mmq_y = MMQ_Y_Q5_K;
535 + const int nwarps = NWARPS_Q5_K;
536 +
537 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
538 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
539 + const dim3 block_nums(block_num_x, block_num_y, 1);
540 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
541 +
542 + if (nrows_x % mmq_y == 0) {
543 + const bool need_check = false;
544 + mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>
545 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
546 + } else {
547 + const bool need_check = true;
548 + mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>
549 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
550 + }
551 +}
552 +
553 +#if defined(USE_ROCM)
554 +#define MMQ_X_Q6_K 64
555 +#define MMQ_Y_Q6_K 128
556 +#define NWARPS_Q6_K 8
557 +#else
558 +#define MMQ_X_Q6_K 4
559 +#define MMQ_Y_Q6_K 32
560 +#define NWARPS_Q6_K 4
561 +#endif
562 +
563 +template <bool need_check> static __global__ void
564 +#if defined(USE_ROCM)
565 +__launch_bounds__(WARP_SIZE*NWARPS_Q6_K, 2)
566 +#endif
567 +mul_mat_q6_K(
568 + const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
569 + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
570 + const int mmq_x = MMQ_X_Q6_K;
571 + const int mmq_y = MMQ_Y_Q6_K;
572 + const int nwarps = NWARPS_Q6_K;
573 +
574 + mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,
575 + load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>
576 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
577 +}
578 +
579 +static void ggml_mul_mat_q6_K_q8_1_cuda(
580 + const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
581 + const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
582 + const int mmq_x = MMQ_X_Q6_K;
583 + const int mmq_y = MMQ_Y_Q6_K;
584 + const int nwarps = NWARPS_Q6_K;
585 +
586 + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
587 + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
588 + const dim3 block_nums(block_num_x, block_num_y, 1);
589 + const dim3 block_dims(WARP_SIZE, nwarps, 1);
590 +
591 + if (nrows_x % mmq_y == 0) {
592 + const bool need_check = false;
593 + mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>
594 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
595 + } else {
596 + const bool need_check = true;
597 + mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>
598 + (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
599 + }
600 +}
csrc/quantization/gguf/mmvq.cuh new
+182
@@ -0,0 +1,182 @@
1 +// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
2 +template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>
3 +static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst, const int ncols, const int nrows) {
4 + const int row = blockIdx.x*blockDim.y + threadIdx.y;
5 +
6 + if (row >= nrows) {
7 + return;
8 + }
9 +
10 + const int blocks_per_row = ncols / qk;
11 + const int blocks_per_warp = vdr * WARP_SIZE / qi;
12 +
13 +// partial sum for each thread
14 + float tmp = 0.0f;
15 +
16 + const block_q_t * x = (const block_q_t *) vx;
17 + const block_q8_1 * y = (const block_q8_1 *) vy;
18 +
19 + for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) {
20 + const int ibx = row*blocks_per_row + i; // x block index
21 +
22 + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
23 +
24 + const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int
25 +
26 + tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
27 + }
28 +
29 + // sum up partial sums and write back result
30 +#pragma unroll
31 + for (int mask = 16; mask > 0; mask >>= 1) {
32 + tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32);
33 + }
34 +
35 + if (threadIdx.x == 0) {
36 + dst[row] = __float2half(tmp);
37 + }
38 +}
39 +
40 +static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
41 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
42 + const dim3 block_nums(block_num_y, 1, 1);
43 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
44 + mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>
45 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
46 +}
47 +
48 +static void mul_mat_vec_q4_1_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
49 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
50 + const dim3 block_nums(block_num_y, 1, 1);
51 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
52 + mul_mat_vec_q<QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>
53 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
54 +}
55 +
56 +static void mul_mat_vec_q5_0_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
57 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
58 + const dim3 block_nums(block_num_y, 1, 1);
59 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
60 + mul_mat_vec_q<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>
61 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
62 +}
63 +
64 +static void mul_mat_vec_q5_1_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
65 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
66 + const dim3 block_nums(block_num_y, 1, 1);
67 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
68 + mul_mat_vec_q<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>
69 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
70 +}
71 +
72 +static void mul_mat_vec_q8_0_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
73 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
74 + const dim3 block_nums(block_num_y, 1, 1);
75 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
76 + mul_mat_vec_q<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>
77 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
78 +}
79 +
80 +static void mul_mat_vec_q2_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
81 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
82 + const dim3 block_nums(block_num_y, 1, 1);
83 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
84 + mul_mat_vec_q<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>
85 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
86 +}
87 +
88 +static void mul_mat_vec_q3_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
89 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
90 + const dim3 block_nums(block_num_y, 1, 1);
91 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
92 + mul_mat_vec_q<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>
93 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
94 +}
95 +
96 +static void mul_mat_vec_q4_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
97 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
98 + const dim3 block_nums(block_num_y, 1, 1);
99 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
100 + mul_mat_vec_q<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>
101 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
102 +}
103 +
104 +static void mul_mat_vec_q5_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
105 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
106 + const dim3 block_nums(block_num_y, 1, 1);
107 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
108 + mul_mat_vec_q<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>
109 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
110 +}
111 +
112 +static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
113 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
114 + const dim3 block_nums(block_num_y, 1, 1);
115 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
116 + mul_mat_vec_q<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>
117 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
118 +}
119 +
120 +static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
121 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
122 + const dim3 block_nums(block_num_y, 1, 1);
123 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
124 + mul_mat_vec_q<QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
125 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
126 +}
127 +
128 +static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
129 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
130 + const dim3 block_nums(block_num_y, 1, 1);
131 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
132 + mul_mat_vec_q<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
133 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
134 +}
135 +
136 +static void mul_mat_vec_iq2_s_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
137 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
138 + const dim3 block_nums(block_num_y, 1, 1);
139 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
140 + mul_mat_vec_q<QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
141 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
142 +}
143 +
144 +static void mul_mat_vec_iq3_xxs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
145 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
146 + const dim3 block_nums(block_num_y, 1, 1);
147 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
148 + mul_mat_vec_q<QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
149 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
150 +}
151 +
152 +static void mul_mat_vec_iq1_s_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
153 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
154 + const dim3 block_nums(block_num_y, 1, 1);
155 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
156 + mul_mat_vec_q<QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
157 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
158 +}
159 +
160 +static void mul_mat_vec_iq4_nl_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
161 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
162 + const dim3 block_nums(block_num_y, 1, 1);
163 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
164 + mul_mat_vec_q<QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>
165 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
166 +}
167 +
168 +static void mul_mat_vec_iq4_xs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
169 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
170 + const dim3 block_nums(block_num_y, 1, 1);
171 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
172 + mul_mat_vec_q<QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
173 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
174 +}
175 +
176 +static void mul_mat_vec_iq3_s_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
177 + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
178 + const dim3 block_nums(block_num_y, 1, 1);
179 + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
180 + mul_mat_vec_q<QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
181 + <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
182 +}
csrc/quantization/gguf/vecdotq.cuh new
+1745
@@ -0,0 +1,1745 @@
1 +// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh
2 +// and https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
3 +static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {
4 + const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment
5 + int x32 = 0;
6 + x32 |= x16[0] << 0;
7 + x32 |= x16[1] << 16;
8 + return x32;
9 +}
10 +
11 +static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {
12 + const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment
13 + int x32 = 0;
14 + x32 |= x16[0] << 0;
15 + x32 |= x16[1] << 16;
16 + return x32;
17 +}
18 +
19 +static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {
20 + return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
21 +}
22 +
23 +static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {
24 + return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
25 +}
26 +
27 +
28 +#define VDR_Q4_0_Q8_1_MMVQ 2
29 +#define VDR_Q4_0_Q8_1_MMQ 4
30 +
31 +template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
32 + const int * v, const int * u, const float & d4, const half2 & ds8) {
33 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
34 + int sumi = 0;
35 +
36 +#pragma unroll
37 + for (int i = 0; i < vdr; ++i) {
38 + const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
39 + const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
40 +
41 + // SIMD dot product of quantized values
42 + sumi = __dp4a(vi0, u[2*i+0], sumi);
43 + sumi = __dp4a(vi1, u[2*i+1], sumi);
44 + }
45 +
46 + const float2 ds8f = __half22float2(ds8);
47 +
48 + // second part effectively subtracts 8 from each quant value
49 + return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
50 +#endif
51 +}
52 +
53 +#define VDR_Q4_1_Q8_1_MMVQ 2
54 +#define VDR_Q4_1_Q8_1_MMQ 4
55 +
56 +template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
57 + const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
58 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
59 + int sumi = 0;
60 +
61 +#pragma unroll
62 + for (int i = 0; i < vdr; ++i) {
63 + const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
64 + const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
65 +
66 + // SIMD dot product of quantized values
67 + sumi = __dp4a(vi0, u[2*i+0], sumi);
68 + sumi = __dp4a(vi1, u[2*i+1], sumi);
69 + }
70 +
71 + const float2 tmp = __half22float2(__hmul2(dm4, ds8));
72 + const float d4d8 = tmp.x;
73 + const float m4s8 = tmp.y;
74 +
75 + // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
76 + return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
77 +#endif
78 +}
79 +
80 +#define VDR_Q5_0_Q8_1_MMVQ 2
81 +#define VDR_Q5_0_Q8_1_MMQ 4
82 +
83 +template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
84 + const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
85 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
86 + int sumi = 0;
87 +
88 +#pragma unroll
89 + for (int i = 0; i < vdr; ++i) {
90 + int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
91 + vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
92 + vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
93 + vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
94 + vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
95 + sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
96 +
97 + int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
98 + vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
99 + vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
100 + vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
101 + vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
102 + sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
103 + }
104 +
105 + const float2 ds8f = __half22float2(ds8);
106 +
107 + // second part effectively subtracts 16 from each quant value
108 + return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
109 +#endif
110 +}
111 +
112 +
113 +#define VDR_Q5_1_Q8_1_MMVQ 2
114 +#define VDR_Q5_1_Q8_1_MMQ 4
115 +
116 +template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
117 + const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
118 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
119 + int sumi = 0;
120 +
121 +#pragma unroll
122 + for (int i = 0; i < vdr; ++i) {
123 + int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
124 + vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
125 + vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
126 + vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
127 + vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
128 + sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
129 +
130 + int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
131 + vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
132 + vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
133 + vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
134 + vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
135 + sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
136 + }
137 +
138 + const float2 tmp = __half22float2(__hmul2(dm5, ds8));
139 + const float d5d8 = tmp.x;
140 + const float m5s8 = tmp.y;
141 +
142 + // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
143 + return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
144 +#endif
145 +}
146 +
147 +#define VDR_Q8_0_Q8_1_MMVQ 2
148 +#define VDR_Q8_0_Q8_1_MMQ 8
149 +
150 +template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(
151 + const int * v, const int * u, const float & d8_0, const float & d8_1) {
152 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
153 + int sumi = 0;
154 +
155 +#pragma unroll
156 + for (int i = 0; i < vdr; ++i) {
157 + // SIMD dot product of quantized values
158 + sumi = __dp4a(v[i], u[i], sumi);
159 + }
160 + return d8_0*d8_1 * sumi;
161 +#endif
162 +}
163 +
164 +template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
165 + const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
166 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
167 +
168 + int sumi = 0;
169 +
170 +#pragma unroll
171 + for (int i = 0; i < vdr; ++i) {
172 + // SIMD dot product of quantized values
173 + sumi = __dp4a(v[i], u[i], sumi);
174 + }
175 +
176 + const float2 tmp = __half22float2(__hmul2(dm8, ds8));
177 + const float d8d8 = tmp.x;
178 + const float m8s8 = tmp.y;
179 +
180 + // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
181 + return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
182 +#endif
183 +}
184 +
185 +#define VDR_Q2_K_Q8_1_MMVQ 1
186 +#define VDR_Q2_K_Q8_1_MMQ 2
187 +
188 +// contiguous v/x values
189 +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
190 + const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
191 + const half2 & dm2, const float * __restrict__ d8) {
192 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
193 + float sumf_d = 0.0f;
194 + float sumf_m = 0.0f;
195 +
196 +#pragma unroll
197 + for (int i = 0; i < QR2_K; ++i) {
198 + const int sc = scales[2*i];
199 +
200 + const int vi = (v >> (2*i)) & 0x03030303;
201 +
202 + sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
203 +
204 + // fill int with 4x m
205 + int m = sc >> 4;
206 + m |= m << 8;
207 + m |= m << 16;
208 + sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
209 + }
210 +
211 + const float2 dm2f = __half22float2(dm2);
212 +
213 + return dm2f.x*sumf_d - dm2f.y*sumf_m;
214 +#endif
215 +}
216 +
217 +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
218 + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
219 + const half2 & dm2, const float & d8) {
220 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
221 + int sumi_d = 0;
222 + int sumi_m = 0;
223 +
224 +#pragma unroll
225 + for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {
226 + int sumi_d_sc = 0;
227 +
228 + const int sc = scales[i0 / (QI8_1/2)];
229 +
230 + // fill int with 4x m
231 + int m = sc >> 4;
232 + m |= m << 8;
233 + m |= m << 16;
234 +
235 +#pragma unroll
236 + for (int i = i0; i < i0 + QI8_1/2; ++i) {
237 + sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product
238 + sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m
239 + }
240 +
241 + sumi_d += sumi_d_sc * (sc & 0xF);
242 + }
243 +
244 + const float2 dm2f = __half22float2(dm2);
245 +
246 + return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);
247 +#endif
248 +}
249 +
250 +#define VDR_Q3_K_Q8_1_MMVQ 1
251 +#define VDR_Q3_K_Q8_1_MMQ 2
252 +
253 +// contiguous v/x values
254 +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
255 + const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
256 + const int & scale_offset, const float & d3, const float * __restrict__ d8) {
257 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
258 +
259 + float sumf = 0.0f;
260 +
261 +#pragma unroll
262 + for (int i = 0; i < QR3_K; ++i) {
263 + const int isc = scale_offset + 2*i;
264 +
265 + const int isc_low = isc % (QK_K/32);
266 + const int sc_shift_low = 4 * (isc / (QK_K/32));
267 + const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
268 +
269 + const int isc_high = isc % (QK_K/64);
270 + const int sc_shift_high = 2 * (isc / (QK_K/64));
271 + const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
272 +
273 + const int sc = (sc_low | sc_high) - 32;
274 +
275 + const int vil = (vl >> (2*i)) & 0x03030303;
276 +
277 + const int vih = ((vh >> i) << 2) & 0x04040404;
278 +
279 + const int vi = __vsubss4(vil, vih);
280 +
281 + sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
282 + }
283 +
284 + return d3 * sumf;
285 +#endif
286 +}
287 +
288 +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
289 + const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
290 + const float & d3, const float & d8) {
291 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
292 + int sumi = 0;
293 +
294 +#pragma unroll
295 + for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
296 + int sumi_sc = 0;
297 +
298 + for (int i = i0; i < i0 + QI8_1/2; ++i) {
299 + sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product
300 + }
301 +
302 + sumi += sumi_sc * scales[i0 / (QI8_1/2)];
303 + }
304 +
305 + return d3*d8 * sumi;
306 +#endif
307 +}
308 +
309 +#define VDR_Q4_K_Q8_1_MMVQ 2
310 +#define VDR_Q4_K_Q8_1_MMQ 8
311 +
312 +// contiguous v/x values
313 +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
314 + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
315 + const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
316 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
317 +
318 + float sumf_d = 0.0f;
319 + float sumf_m = 0.0f;
320 +
321 +#pragma unroll
322 + for (int i = 0; i < QR4_K; ++i) {
323 + const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
324 + const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
325 +
326 + const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
327 + const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u
328 +
329 + sumf_d += d8[i] * (dot1 * sc[i]);
330 + sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
331 + }
332 +
333 + const float2 dm4f = __half22float2(dm4);
334 + return dm4f.x*sumf_d - dm4f.y*sumf_m;
335 +#endif
336 +}
337 +
338 +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
339 + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
340 + const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
341 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
342 + float sumf_d = 0.0f;
343 + float sumf_m = 0.0f;
344 +
345 +#pragma unroll
346 + for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
347 + int sumi_d = 0;
348 +
349 +#pragma unroll
350 + for (int j = 0; j < QI8_1; ++j) {
351 + sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
352 + }
353 +
354 + const float2 ds8f = __half22float2(ds8[i]);
355 +
356 + sumf_d += ds8f.x * (sc[i] * sumi_d);
357 + sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
358 + }
359 +
360 + const float2 dm4f = __half22float2(dm4);
361 +
362 + return dm4f.x*sumf_d - dm4f.y*sumf_m;
363 +#endif
364 +}
365 +
366 +#define VDR_Q5_K_Q8_1_MMVQ 2
367 +#define VDR_Q5_K_Q8_1_MMQ 8
368 +
369 +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
370 + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
371 + const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
372 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
373 +
374 + float sumf_d = 0.0f;
375 + float sumf_m = 0.0f;
376 +
377 +#pragma unroll
378 + for (int i = 0; i < QR5_K; ++i) {
379 + const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
380 + const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
381 +
382 + const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
383 + const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
384 +
385 + const int v0i = vl0i | vh0i;
386 + const int v1i = vl1i | vh1i;
387 +
388 + const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
389 + const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u
390 +
391 + sumf_d += d8[i] * (dot1 * sc[i]);
392 + sumf_m += d8[i] * (dot2 * m[i]);
393 + }
394 +
395 + const float2 dm5f = __half22float2(dm5);
396 + return dm5f.x*sumf_d - dm5f.y*sumf_m;
397 +#endif
398 +}
399 +
400 +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
401 + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
402 + const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
403 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
404 + float sumf_d = 0.0f;
405 + float sumf_m = 0.0f;
406 +
407 +#pragma unroll
408 + for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
409 + int sumi_d = 0;
410 +
411 +#pragma unroll
412 + for (int j = 0; j < QI8_1; ++j) {
413 + sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
414 + }
415 +
416 + const float2 ds8f = __half22float2(ds8[i]);
417 +
418 + sumf_d += ds8f.x * (sc[i] * sumi_d);
419 + sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
420 + }
421 +
422 + const float2 dm4f = __half22float2(dm4);
423 +
424 + return dm4f.x*sumf_d - dm4f.y*sumf_m;
425 +#endif
426 +}
427 +
428 +#define VDR_Q6_K_Q8_1_MMVQ 1
429 +#define VDR_Q6_K_Q8_1_MMQ 8
430 +
431 +// contiguous v/x values
432 +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
433 + const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
434 + const float & d, const float * __restrict__ d8) {
435 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
436 + float sumf = 0.0f;
437 +
438 +#pragma unroll
439 + for (int i = 0; i < QR6_K; ++i) {
440 + const int sc = scales[4*i];
441 + const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
442 + const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
443 + const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
444 +
445 + sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
446 + }
447 +
448 + return d*sumf;
449 +#endif
450 +}
451 +
452 +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
453 + const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
454 + const float & d6, const float * __restrict__ d8) {
455 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
456 + float sumf_d = 0.0f;
457 +
458 +#pragma unroll
459 + for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
460 + int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
461 +
462 +#pragma unroll
463 + for (int i = i0; i < i0 + 2; ++i) {
464 + sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
465 + sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
466 +
467 + sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
468 + sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
469 + }
470 +
471 + sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);
472 + }
473 +
474 + return d6 * sumf_d;
475 +#endif
476 +}
477 +
478 +static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
479 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
480 +
481 + const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;
482 +
483 + int v[VDR_Q4_0_Q8_1_MMVQ];
484 + int u[2*VDR_Q4_0_Q8_1_MMVQ];
485 +
486 +#pragma unroll
487 + for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
488 + v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);
489 + u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
490 + u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);
491 + }
492 +
493 + return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, __half2float(bq4_0->d), bq8_1->ds);
494 +}
495 +
496 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
497 + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
498 + __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];
499 + *x_ql = tile_x_qs;
500 + *x_dm = (half2 *) tile_x_d;
501 +}
502 +
503 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(
504 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
505 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
506 + const int kbx = k / QI4_0;
507 + const int kqsx = k % QI4_0;
508 +
509 + const block_q4_0 * bx0 = (const block_q4_0 *) vx;
510 + float * x_dmf = (float *) x_dm;
511 +
512 +#pragma unroll
513 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
514 + int i = i0 + i_offset;
515 + if (need_check) {
516 + i = min(i, i_max);
517 + }
518 + const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;
519 + x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
520 + // x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbx] = bxi->d;
521 + }
522 +
523 + const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;
524 + const int kbxd = k % blocks_per_tile_x_row;
525 +
526 +#pragma unroll
527 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {
528 + int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;
529 + if (need_check) {
530 + i = min(i, i_max);
531 + }
532 + const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;
533 + x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
534 + }
535 +}
536 +
537 +static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(
538 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
539 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
540 + (void)x_qh; (void)x_sc;
541 +
542 + const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
543 + const float * x_dmf = (const float *) x_dm;
544 +
545 + int u[2*VDR_Q4_0_Q8_1_MMQ];
546 +
547 +#pragma unroll
548 + for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
549 + u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
550 + u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_0) % WARP_SIZE];
551 + }
552 +
553 + return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
554 + (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dmf[i * (WARP_SIZE/QI4_0) + i/QI4_0 + k/QI4_0],
555 + y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
556 +}
557 +
558 +static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
559 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
560 +
561 + const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;
562 +
563 + int v[VDR_Q4_1_Q8_1_MMVQ];
564 + int u[2*VDR_Q4_1_Q8_1_MMVQ];
565 +
566 +#pragma unroll
567 + for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
568 + v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);
569 + u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
570 + u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);
571 + }
572 +
573 + return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
574 +}
575 +
576 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
577 + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];
578 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];
579 + *x_ql = tile_x_qs;
580 + *x_dm = tile_x_dm;
581 +}
582 +
583 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(
584 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
585 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
586 + const int kbx = k / QI4_1;
587 + const int kqsx = k % QI4_1;
588 +
589 + const block_q4_1 * bx0 = (const block_q4_1 *) vx;
590 +
591 +#pragma unroll
592 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
593 + int i = i0 + i_offset;
594 + if (need_check) {
595 + i = min(i, i_max);
596 + }
597 + const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;
598 + x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
599 + }
600 +
601 + const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;
602 + const int kbxd = k % blocks_per_tile_x_row;
603 +
604 +#pragma unroll
605 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {
606 + int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;
607 + if (need_check) {
608 + i = min(i, i_max);
609 + }
610 + const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;
611 + x_dm[i * (WARP_SIZE/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
612 + }
613 +}
614 +
615 +static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(
616 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
617 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
618 + const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
619 +
620 + int u[2*VDR_Q4_1_Q8_1_MMQ];
621 +
622 +#pragma unroll
623 + for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
624 + u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
625 + u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_1) % WARP_SIZE];
626 + }
627 +
628 + return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
629 + (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dm[i * (WARP_SIZE/QI4_1) + i/QI4_1 + k/QI4_1],
630 + y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
631 +}
632 +
633 +static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
634 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
635 +
636 + const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;
637 +
638 + int vl[VDR_Q5_0_Q8_1_MMVQ];
639 + int vh[VDR_Q5_0_Q8_1_MMVQ];
640 + int u[2*VDR_Q5_0_Q8_1_MMVQ];
641 +
642 +#pragma unroll
643 + for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
644 + vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);
645 + vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));
646 + u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
647 + u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);
648 + }
649 +
650 + return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, __half2float(bq5_0->d), bq8_1->ds);
651 +}
652 +
653 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
654 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
655 + __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];
656 +
657 + *x_ql = tile_x_ql;
658 + *x_dm = (half2 *) tile_x_d;
659 +}
660 +
661 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(
662 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
663 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
664 + const int kbx = k / QI5_0;
665 + const int kqsx = k % QI5_0;
666 +
667 + const block_q5_0 * bx0 = (const block_q5_0 *) vx;
668 +
669 +#pragma unroll
670 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
671 + int i = i0 + i_offset;
672 +
673 + if (need_check) {
674 + i = min(i, i_max);
675 + }
676 + const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbx;
677 + const int ql = get_int_from_uint8(bxi->qs, kqsx);
678 + const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));
679 +
680 + int qs0 = (ql >> 0) & 0x0F0F0F0F;
681 + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
682 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
683 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
684 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
685 + qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
686 +
687 + x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
688 +
689 + int qs1 = (ql >> 4) & 0x0F0F0F0F;
690 + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
691 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
692 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
693 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
694 + qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
695 +
696 + x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
697 + }
698 +
699 + const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;
700 + const int kbxd = k % blocks_per_tile_x_row;
701 + float * x_dmf = (float *) x_dm;
702 +
703 +#pragma unroll
704 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {
705 + int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;
706 +
707 + if (need_check) {
708 + i = min(i, i_max);
709 + }
710 +
711 + const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;
712 + x_dmf[i * (WARP_SIZE/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
713 + }
714 +}
715 +
716 +static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
717 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
718 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
719 + const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
720 + const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;
721 + const float * x_dmf = (const float *) x_dm;
722 + const float * y_df = (const float *) y_ds;
723 +
724 + int u[2*VDR_Q5_0_Q8_1_MMQ];
725 +
726 +#pragma unroll
727 + for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {
728 + u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
729 + u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_0) % WARP_SIZE];
730 + }
731 +
732 + return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>
733 + (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
734 +}
735 +
736 +static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
737 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
738 +
739 + const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;
740 +
741 + int vl[VDR_Q5_1_Q8_1_MMVQ];
742 + int vh[VDR_Q5_1_Q8_1_MMVQ];
743 + int u[2*VDR_Q5_1_Q8_1_MMVQ];
744 +
745 +#pragma unroll
746 + for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
747 + vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);
748 + vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));
749 + u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
750 + u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);
751 + }
752 +
753 + return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
754 +}
755 +
756 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
757 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
758 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];
759 +
760 + *x_ql = tile_x_ql;
761 + *x_dm = tile_x_dm;
762 +}
763 +
764 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(
765 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
766 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
767 + const int kbx = k / QI5_1;
768 + const int kqsx = k % QI5_1;
769 +
770 + const block_q5_1 * bx0 = (const block_q5_1 *) vx;
771 +
772 +#pragma unroll
773 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
774 + int i = i0 + i_offset;
775 +
776 + if (need_check) {
777 + i = min(i, i_max);
778 + }
779 +
780 + const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbx;
781 +
782 + const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
783 + const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));
784 +
785 + int qs0 = (ql >> 0) & 0x0F0F0F0F;
786 + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
787 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
788 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
789 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
790 +
791 + x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
792 +
793 + int qs1 = (ql >> 4) & 0x0F0F0F0F;
794 + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
795 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
796 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
797 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
798 +
799 + x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
800 + }
801 +
802 + const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;
803 + const int kbxd = k % blocks_per_tile_x_row;
804 +
805 +#pragma unroll
806 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {
807 + int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;
808 +
809 + if (need_check) {
810 + i = min(i, i_max);
811 + }
812 +
813 + const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;
814 +
815 + x_dm[i * (WARP_SIZE/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
816 + }
817 +}
818 +
819 +static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(
820 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
821 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
822 + const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
823 + const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;
824 +
825 + int u[2*VDR_Q5_1_Q8_1_MMQ];
826 +
827 +#pragma unroll
828 + for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {
829 + u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
830 + u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_1) % WARP_SIZE];
831 + }
832 +
833 + return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
834 + (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
835 +}
836 +
837 +static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
838 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
839 +
840 + const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;
841 +
842 + int v[VDR_Q8_0_Q8_1_MMVQ];
843 + int u[VDR_Q8_0_Q8_1_MMVQ];
844 +
845 +#pragma unroll
846 + for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
847 + v[i] = get_int_from_int8(bq8_0->qs, iqs + i);
848 + u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
849 + }
850 +
851 + return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, __half2float(bq8_0->d), __low2float(bq8_1->ds));
852 +}
853 +
854 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
855 + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
856 + __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];
857 +
858 + *x_ql = tile_x_qs;
859 + *x_dm = (half2 *) tile_x_d;
860 +}
861 +
862 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(
863 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
864 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
865 + const int kbx = k / QI8_0;
866 + const int kqsx = k % QI8_0;
867 + float * x_dmf = (float *) x_dm;
868 +
869 + const block_q8_0 * bx0 = (const block_q8_0 *) vx;
870 +
871 +#pragma unroll
872 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
873 + int i = i0 + i_offset;
874 +
875 + if (need_check) {
876 + i = min(i, i_max);
877 + }
878 + const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;
879 + x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
880 + }
881 +
882 + const int blocks_per_tile_x_row = WARP_SIZE / QI8_0;
883 + const int kbxd = k % blocks_per_tile_x_row;
884 +
885 +#pragma unroll
886 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {
887 + int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;
888 +
889 + if (need_check) {
890 + i = min(i, i_max);
891 + }
892 + const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;
893 + x_dmf[i * (WARP_SIZE/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
894 + }
895 +}
896 +
897 +static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(
898 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
899 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
900 + const float * x_dmf = (const float *) x_dm;
901 + const float * y_df = (const float *) y_ds;
902 +
903 + return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>
904 + (&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[j * WARP_SIZE + k], x_dmf[i * (WARP_SIZE/QI8_0) + i/QI8_0 + k/QI8_0],
905 + y_df[j * (WARP_SIZE/QI8_1) + k/QI8_1]);
906 +}
907 +
908 +static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
909 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
910 +
911 + const block_q2_K * bq2_K = (const block_q2_K *) vbq;
912 +
913 + const int bq8_offset = QR2_K * (iqs / QI8_1);
914 + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
915 +
916 + const uint8_t * scales = bq2_K->scales + scale_offset;
917 +
918 + const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);
919 + int u[QR2_K];
920 + float d8[QR2_K];
921 +
922 +#pragma unroll
923 + for (int i = 0; i < QR2_K; ++ i) {
924 + u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
925 + d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
926 + }
927 +
928 + return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
929 +}
930 +
931 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
932 + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
933 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];
934 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
935 +
936 + *x_ql = tile_x_ql;
937 + *x_dm = tile_x_dm;
938 + *x_sc = tile_x_sc;
939 +}
940 +
941 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(
942 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
943 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
944 + const int kbx = k / QI2_K;
945 + const int kqsx = k % QI2_K;
946 +
947 + const block_q2_K * bx0 = (const block_q2_K *) vx;
948 +
949 +#pragma unroll
950 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
951 + int i = i0 + i_offset;
952 +
953 + if (need_check) {
954 + i = min(i, i_max);
955 + }
956 + const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;
957 + x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
958 + }
959 +
960 + const int blocks_per_tile_x_row = WARP_SIZE / QI2_K;
961 + const int kbxd = k % blocks_per_tile_x_row;
962 +
963 +#pragma unroll
964 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {
965 + int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;
966 +
967 + if (need_check) {
968 + i = min(i, i_max);
969 + }
970 + const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;
971 + x_dm[i * (WARP_SIZE/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
972 + }
973 +
974 +#pragma unroll
975 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
976 + int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
977 +
978 + if (need_check) {
979 + i = min(i, i_max);
980 + }
981 + const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI2_K/4);
982 + x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
983 + }
984 +}
985 +
986 +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
987 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
988 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
989 + const int kbx = k / QI2_K;
990 + const int ky = (k % QI2_K) * QR2_K;
991 + const float * y_df = (const float *) y_ds;
992 +
993 + int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];
994 +
995 + const int kqsx = i * (WARP_SIZE + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
996 + const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));
997 +
998 +#pragma unroll
999 + for (int l = 0; l < QR2_K*VDR_Q2_K_Q8_1_MMQ; ++l) {
1000 + v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;
1001 + }
1002 +
1003 + const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE/4) + i/4 + kbx*4]) + ky/4;
1004 +
1005 + const int index_y = j * WARP_SIZE + (QR2_K*k) % WARP_SIZE;
1006 + return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1007 +}
1008 +
1009 +static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
1010 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1011 +
1012 + const block_q3_K * bq3_K = (const block_q3_K *) vbq;
1013 +
1014 + const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
1015 + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
1016 +
1017 + const float d = __half2float(bq3_K->d);
1018 +
1019 + const int vl = get_int_from_uint8(bq3_K->qs, iqs);
1020 +
1021 + // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1022 + const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
1023 +
1024 + int u[QR3_K];
1025 + float d8[QR3_K];
1026 +
1027 +#pragma unroll
1028 + for (int i = 0; i < QR3_K; ++i) {
1029 + u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
1030 + d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
1031 + }
1032 +
1033 + return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
1034 +}
1035 +
1036 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1037 + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1038 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI3_K) + mmq_y/QI3_K];
1039 + __shared__ int tile_x_qh[mmq_y * (WARP_SIZE/2) + mmq_y/2];
1040 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
1041 +
1042 + *x_ql = tile_x_ql;
1043 + *x_dm = tile_x_dm;
1044 + *x_qh = tile_x_qh;
1045 + *x_sc = tile_x_sc;
1046 +}
1047 +
1048 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(
1049 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1050 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1051 + const int kbx = k / QI3_K;
1052 + const int kqsx = k % QI3_K;
1053 +
1054 + const block_q3_K * bx0 = (const block_q3_K *) vx;
1055 +
1056 +#pragma unroll
1057 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1058 + int i = i0 + i_offset;
1059 + if (need_check) {
1060 + i = min(i, i_max);
1061 + }
1062 + const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;
1063 + x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1064 + }
1065 +
1066 + const int blocks_per_tile_x_row = WARP_SIZE / QI3_K;
1067 + const int kbxd = k % blocks_per_tile_x_row;
1068 + float * x_dmf = (float *) x_dm;
1069 +
1070 +#pragma unroll
1071 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {
1072 + int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;
1073 + if (need_check) {
1074 + i = min(i, i_max);
1075 + }
1076 + const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;
1077 + x_dmf[i * (WARP_SIZE/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1078 + }
1079 +
1080 +#pragma unroll
1081 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {
1082 + int i = i0 + i_offset * 2 + k / (WARP_SIZE/2);
1083 + if (need_check) {
1084 + i = min(i, i_max);
1085 + }
1086 + const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/2)) / (QI3_K/2);
1087 + // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1088 + x_qh[i * (WARP_SIZE/2) + i / 2 + k % (WARP_SIZE/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1089 + }
1090 +
1091 +#pragma unroll
1092 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
1093 + int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
1094 + if (need_check) {
1095 + i = min(i, i_max);
1096 + }
1097 + const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI3_K/4);
1098 +
1099 + const int ksc = k % (QI3_K/4);
1100 +
1101 + const int ksc_low = ksc % (QI3_K/8);
1102 + const int shift_low = 4 * (ksc / (QI3_K/8));
1103 + const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;
1104 +
1105 + const int ksc_high = QI3_K/8;
1106 + const int shift_high = 2 * ksc;
1107 + const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;
1108 +
1109 + const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
1110 +
1111 + x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = sc;
1112 + }
1113 +}
1114 +
1115 +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(
1116 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1117 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1118 +
1119 + const int kbx = k / QI3_K;
1120 + const int ky = (k % QI3_K) * QR3_K;
1121 + const float * x_dmf = (const float *) x_dm;
1122 + const float * y_df = (const float *) y_ds;
1123 +
1124 + const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;
1125 +
1126 + int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];
1127 +
1128 +#pragma unroll
1129 + for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {
1130 + const int kqsx = i * (WARP_SIZE + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1131 + const int shift = 2 * ((ky % 32) / 8);
1132 + const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;
1133 +
1134 + const int vh = x_qh[i * (WARP_SIZE/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1135 + const int vlh = (vh << 2) & 0x04040404;
1136 +
1137 + v[l] = __vsubss4(vll, vlh);
1138 + }
1139 +
1140 + const int index_y = j * WARP_SIZE + (k*QR3_K) % WARP_SIZE;
1141 + return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1142 +}
1143 +
1144 +static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
1145 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1146 + const block_q4_K * bq4_K = (const block_q4_K *) vbq;
1147 +
1148 + int v[2];
1149 + int u[2*QR4_K];
1150 + float d8[QR4_K];
1151 +
1152 + // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
1153 + const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
1154 +
1155 + // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
1156 + // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
1157 + // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
1158 + // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
1159 +
1160 + const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1161 + v[0] = q4[0];
1162 + v[1] = q4[4];
1163 +
1164 + const uint16_t * scales = (const uint16_t *)bq4_K->scales;
1165 + uint16_t aux[2];
1166 + const int j = bq8_offset/2;
1167 + if (j < 2) {
1168 + aux[0] = scales[j+0] & 0x3f3f;
1169 + aux[1] = scales[j+2] & 0x3f3f;
1170 + } else {
1171 + aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1172 + aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1173 + }
1174 + const uint8_t * sc = (const uint8_t *)aux;
1175 + const uint8_t * m = sc + 2;
1176 +
1177 + for (int i = 0; i < QR4_K; ++i) {
1178 + const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1179 + d8[i] = __low2float(bq8i->ds);
1180 +
1181 + const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1182 + u[2*i+0] = q8[0];
1183 + u[2*i+1] = q8[4];
1184 + }
1185 +
1186 + return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
1187 +}
1188 +
1189 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1190 + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1191 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];
1192 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1193 +
1194 + *x_ql = tile_x_ql;
1195 + *x_dm = tile_x_dm;
1196 + *x_sc = tile_x_sc;
1197 +}
1198 +
1199 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(
1200 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1201 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1202 + const int kbx = k / QI4_K; // == 0 if QK_K == 256
1203 + const int kqsx = k % QI4_K; // == k if QK_K == 256
1204 +
1205 + const block_q4_K * bx0 = (const block_q4_K *) vx;
1206 +
1207 +#pragma unroll
1208 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1209 + int i = i0 + i_offset;
1210 +
1211 + if (need_check) {
1212 + i = min(i, i_max);
1213 + }
1214 + const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;
1215 + x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1216 + }
1217 +
1218 + const int blocks_per_tile_x_row = WARP_SIZE / QI4_K; // == 1 if QK_K == 256
1219 + const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1220 +
1221 +#pragma unroll
1222 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {
1223 + int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;
1224 + if (need_check) {
1225 + i = min(i, i_max);
1226 + }
1227 + const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;
1228 + x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1229 + }
1230 +
1231 +#pragma unroll
1232 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1233 + int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1234 +
1235 + if (need_check) {
1236 + i = min(i, i_max);
1237 + }
1238 +
1239 + const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);
1240 +
1241 + const int * scales = (const int *) bxi->scales;
1242 +
1243 + const int ksc = k % (WARP_SIZE/8);
1244 + // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1245 + int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1246 + scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1247 +
1248 + x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1249 + }
1250 +}
1251 +
1252 +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(
1253 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1254 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1255 + (void)x_qh;
1256 +
1257 + const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1258 +
1259 + const int index_y = j * WARP_SIZE + (QR4_K*k) % WARP_SIZE;
1260 + return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[index_y], sc, sc+8,
1261 + x_dm[i * (WARP_SIZE/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1262 +}
1263 +
1264 +static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
1265 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1266 +
1267 + const block_q5_K * bq5_K = (const block_q5_K *) vbq;
1268 +
1269 + int vl[2];
1270 + int vh[2];
1271 + int u[2*QR5_K];
1272 + float d8[QR5_K];
1273 +
1274 + const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
1275 + const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1276 + const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
1277 +
1278 + vl[0] = ql[0];
1279 + vl[1] = ql[4];
1280 +
1281 + vh[0] = qh[0] >> bq8_offset;
1282 + vh[1] = qh[4] >> bq8_offset;
1283 +
1284 + const uint16_t * scales = (const uint16_t *)bq5_K->scales;
1285 + uint16_t aux[2];
1286 + const int j = bq8_offset/2;
1287 + if (j < 2) {
1288 + aux[0] = scales[j+0] & 0x3f3f;
1289 + aux[1] = scales[j+2] & 0x3f3f;
1290 + } else {
1291 + aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1292 + aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1293 + }
1294 + const uint8_t * sc = (const uint8_t *)aux;
1295 + const uint8_t * m = sc + 2;
1296 +
1297 +#pragma unroll
1298 + for (int i = 0; i < QR5_K; ++i) {
1299 + const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1300 + d8[i] = __low2float(bq8i->ds);
1301 +
1302 + const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1303 + u[2*i+0] = q8[0];
1304 + u[2*i+1] = q8[4];
1305 + }
1306 +
1307 + return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
1308 +}
1309 +
1310 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1311 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1312 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];
1313 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1314 +
1315 + *x_ql = tile_x_ql;
1316 + *x_dm = tile_x_dm;
1317 + *x_sc = tile_x_sc;
1318 +}
1319 +
1320 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(
1321 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1322 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1323 + const int kbx = k / QI5_K; // == 0 if QK_K == 256
1324 + const int kqsx = k % QI5_K; // == k if QK_K == 256
1325 +
1326 + const block_q5_K * bx0 = (const block_q5_K *) vx;
1327 +
1328 +#pragma unroll
1329 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1330 + int i = i0 + i_offset;
1331 +
1332 + if (need_check) {
1333 + i = min(i, i_max);
1334 + }
1335 +
1336 + const block_q5_K * bxi = bx0 + i*blocks_per_row + kbx;
1337 + const int ky = QR5_K*kqsx;
1338 +
1339 + const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
1340 + const int ql0 = (ql >> 0) & 0x0F0F0F0F;
1341 + const int ql1 = (ql >> 4) & 0x0F0F0F0F;
1342 +
1343 + const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K/4));
1344 + const int qh0 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 0)) << 4) & 0x10101010;
1345 + const int qh1 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 1)) << 4) & 0x10101010;
1346 +
1347 + const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;
1348 + const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);
1349 +
1350 + x_ql[i * (2*WARP_SIZE + 1) + kq0] = ql0 | qh0;
1351 + x_ql[i * (2*WARP_SIZE + 1) + kq1] = ql1 | qh1;
1352 + }
1353 +
1354 + const int blocks_per_tile_x_row = WARP_SIZE / QI5_K; // == 1 if QK_K == 256
1355 + const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1356 +
1357 +#pragma unroll
1358 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {
1359 + int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;
1360 +
1361 + if (need_check) {
1362 + i = min(i, i_max);
1363 + }
1364 +
1365 + const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;
1366 + x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1367 + }
1368 +
1369 +#pragma unroll
1370 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1371 + int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1372 +
1373 + if (need_check) {
1374 + i = min(i, i_max);
1375 + }
1376 +
1377 + const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);
1378 +
1379 + const int * scales = (const int *) bxi->scales;
1380 +
1381 + const int ksc = k % (WARP_SIZE/8);
1382 +
1383 + // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1384 + int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1385 + scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1386 +
1387 + x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1388 + }
1389 +}
1390 +
1391 +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(
1392 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1393 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1394 + const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1395 +
1396 + const int index_x = i * (QR5_K*WARP_SIZE + 1) + QR5_K*k;
1397 + const int index_y = j * WARP_SIZE + (QR5_K*k) % WARP_SIZE;
1398 + return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,
1399 + x_dm[i * (WARP_SIZE/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1400 +}
1401 +
1402 +static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
1403 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1404 +
1405 + const block_q6_K * bq6_K = (const block_q6_K *) vbq;
1406 +
1407 + const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
1408 + const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
1409 + const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
1410 +
1411 + const int vl = get_int_from_uint8(bq6_K->ql, iqs);
1412 + const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
1413 +
1414 + const int8_t * scales = bq6_K->scales + scale_offset;
1415 +
1416 + int u[QR6_K];
1417 + float d8[QR6_K];
1418 +
1419 +#pragma unroll
1420 + for (int i = 0; i < QR6_K; ++i) {
1421 + u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
1422 + d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
1423 + }
1424 +
1425 + return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, __half2float(bq6_K->d), d8);
1426 +}
1427 +
1428 +template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1429 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1430 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];
1431 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1432 +
1433 + *x_ql = tile_x_ql;
1434 + *x_dm = tile_x_dm;
1435 + *x_sc = tile_x_sc;
1436 +}
1437 +
1438 +template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(
1439 + const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1440 + int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1441 + const int kbx = k / QI6_K; // == 0 if QK_K == 256
1442 + const int kqsx = k % QI6_K; // == k if QK_K == 256
1443 +
1444 + const block_q6_K * bx0 = (const block_q6_K *) vx;
1445 +
1446 +#pragma unroll
1447 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1448 + int i = i0 + i_offset;
1449 +
1450 + if (need_check) {
1451 + i = min(i, i_max);
1452 + }
1453 +
1454 + const block_q6_K * bxi = bx0 + i*blocks_per_row + kbx;
1455 + const int ky = QR6_K*kqsx;
1456 +
1457 + const int ql = get_int_from_uint8(bxi->ql, kqsx);
1458 + const int ql0 = (ql >> 0) & 0x0F0F0F0F;
1459 + const int ql1 = (ql >> 4) & 0x0F0F0F0F;
1460 +
1461 + const int qh = get_int_from_uint8(bxi->qh, (QI6_K/4) * (kqsx / (QI6_K/2)) + kqsx % (QI6_K/4));
1462 + const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) << 4) & 0x30303030;
1463 + const int qh1 = (qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) & 0x30303030;
1464 +
1465 + const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;
1466 + const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);
1467 +
1468 + x_ql[i * (2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1469 + x_ql[i * (2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1470 + }
1471 +
1472 + const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256
1473 + const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1474 + float * x_dmf = (float *) x_dm;
1475 +
1476 +#pragma unroll
1477 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {
1478 + int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;
1479 +
1480 + if (need_check) {
1481 + i = min(i, i_max);
1482 + }
1483 +
1484 + const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;
1485 +
1486 + x_dmf[i * (WARP_SIZE/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1487 + }
1488 +
1489 +#pragma unroll
1490 + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1491 + int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1492 +
1493 + if (need_check) {
1494 + i = min(i, i_max);
1495 + }
1496 +
1497 + const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / 4;
1498 +
1499 + x_sc[i * (WARP_SIZE/8) + i / 8 + k % (WARP_SIZE/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1500 + }
1501 +}
1502 +
1503 +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(
1504 + const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1505 + const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1506 + const float * x_dmf = (const float *) x_dm;
1507 + const float * y_df = (const float *) y_ds;
1508 +
1509 + const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/8]);
1510 +
1511 + const int index_x = i * (QR6_K*WARP_SIZE + 1) + QR6_K*k;
1512 + const int index_y = j * WARP_SIZE + (QR6_K*k) % WARP_SIZE;
1513 + return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1514 +}
1515 +
1516 +static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
1517 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1518 + const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;
1519 +
1520 + const int ib32 = iqs;
1521 + const uint16_t * q2 = bq2->qs + 4*ib32;
1522 + const uint8_t * aux8 = (const uint8_t *)q2;
1523 + const int8_t * q8 = bq8_1[ib32].qs;
1524 + uint32_t aux32 = q2[2] | (q2[3] << 16);
1525 + int sumi = 0;
1526 + for (int l = 0; l < 4; ++l) {
1527 + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
1528 + const uint8_t signs = ksigns_iq2xs[aux32 & 127];
1529 + for (int j = 0; j < 8; ++j) {
1530 + sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1531 + }
1532 + q8 += 8;
1533 + aux32 >>= 7;
1534 + }
1535 + const float d = __half2float(bq2->d) * (0.5f + aux32) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
1536 + return d * sumi;
1537 +}
1538 +
1539 +static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
1540 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1541 + const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;
1542 +
1543 + const int ib32 = iqs;
1544 + const uint16_t * q2 = bq2->qs + 4*ib32;
1545 + const int8_t * q8 = bq8_1[ib32].qs;
1546 + const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1547 + const uint8_t ls2 = bq2->scales[ib32] >> 4;
1548 + int sumi1 = 0;
1549 + for (int l = 0; l < 2; ++l) {
1550 + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1551 + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1552 + for (int j = 0; j < 8; ++j) {
1553 + sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1554 + }
1555 + q8 += 8;
1556 + }
1557 + int sumi2 = 0;
1558 + for (int l = 2; l < 4; ++l) {
1559 + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1560 + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1561 + for (int j = 0; j < 8; ++j) {
1562 + sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1563 + }
1564 + q8 += 8;
1565 + }
1566 + const float d = __half2float(bq2->d) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
1567 + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1568 +}
1569 +
1570 +static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1571 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1572 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1573 + const block_iq2_s * bq2 = (const block_iq2_s *) vbq;
1574 +
1575 + const int ib32 = iqs;
1576 + const int8_t * q8 = bq8_1[ib32].qs;
1577 + const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;
1578 + const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1579 + const uint8_t ls2 = bq2->scales[ib32] >> 4;
1580 + int sumi1 = 0;
1581 + for (int l = 0; l < 2; ++l) {
1582 + const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1583 + const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1584 + const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1585 + const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
1586 + const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
1587 + sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);
1588 + sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);
1589 + q8 += 8;
1590 + }
1591 + int sumi2 = 0;
1592 + for (int l = 2; l < 4; ++l) {
1593 + const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1594 + const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1595 + const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1596 + const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
1597 + const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
1598 + sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);
1599 + sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);
1600 + q8 += 8;
1601 + }
1602 + const float d = __half2float(bq2->d) * __low2float(bq8_1[ib32].ds) * 0.25f;
1603 + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1604 +#endif
1605 +}
1606 +
1607 +static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1608 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1609 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1610 + const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;
1611 +
1612 + const int ib32 = iqs;
1613 + const uint8_t * q3 = bq2->qs + 8*ib32;
1614 + const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;
1615 + const int8_t * q8 = bq8_1[ib32].qs;
1616 + uint32_t aux32 = gas[0] | (gas[1] << 16);
1617 + int sumi = 0;
1618 + for (int l = 0; l < 4; ++l) {
1619 + const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];
1620 + const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];
1621 + const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));
1622 + const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);
1623 + const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);
1624 + sumi = __dp4a(grid_l, *((int *)q8+0), sumi);
1625 + sumi = __dp4a(grid_h, *((int *)q8+1), sumi);
1626 + q8 += 8;
1627 + aux32 >>= 7;
1628 + }
1629 + const float d = __half2float(bq2->d) * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;
1630 + return d * sumi;
1631 +#endif
1632 +}
1633 +
1634 +static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1635 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1636 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1637 + const block_iq3_s * bq2 = (const block_iq3_s *) vbq;
1638 +
1639 + const int ib32 = iqs;
1640 + const uint8_t * qs = bq2->qs + 8*ib32;
1641 + const int8_t * q8 = bq8_1[ib32].qs;
1642 + int sumi = 0;
1643 + for (int l = 0; l < 4; ++l) {
1644 + const uint32_t * grid1 = iq3xs_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));
1645 + const uint32_t * grid2 = iq3xs_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));
1646 + uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1647 + uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1648 + const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);
1649 + const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);
1650 + sumi = __dp4a(grid_l, *((int *)q8+0), sumi);
1651 + sumi = __dp4a(grid_h, *((int *)q8+1), sumi);
1652 + q8 += 8;
1653 + }
1654 + const float d = __half2float(bq2->d) * (0.5f + ((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds) * 0.5f;
1655 + return d * sumi;
1656 +#endif
1657 +}
1658 +
1659 +static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1660 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1661 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1662 + const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1663 +
1664 + const int ib32 = iqs;
1665 + int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;
1666 + const uint8_t h1 = bq1->scales[2*ib32+0];
1667 + const uint8_t h2 = bq1->scales[2*ib32+1];
1668 + const int * q8 = (const int *)bq8_1[ib32].qs;
1669 + const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5)));
1670 + const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1)));
1671 + const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5)));
1672 + const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1)));
1673 + for (int j = 0; j < 2; ++j) {
1674 + sumi1 = __dp4a(q8[j+0], grid1[j], sumi1);
1675 + sumi2 = __dp4a(q8[j+2], grid2[j], sumi2);
1676 + sumi3 = __dp4a(q8[j+4], grid3[j], sumi3);
1677 + sumi4 = __dp4a(q8[j+6], grid4[j], sumi4);
1678 + }
1679 + const float d = __half2float(bq1->d) * __low2float(bq8_1[ib32].ds);
1680 + return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) +
1681 + sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1));
1682 +#endif
1683 +}
1684 +
1685 +static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,
1686 + int & val1, int & val2) {
1687 +
1688 + uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;
1689 + aux32 = q4 & 0x0f0f0f0f;
1690 + uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);
1691 + uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);
1692 + val1 = v1 | (v2 << 16);
1693 + aux32 = (q4 >> 4) & 0x0f0f0f0f;
1694 + v1 = values[q8[0]] | (values[q8[1]] << 8);
1695 + v2 = values[q8[2]] | (values[q8[3]] << 8);
1696 + val2 = v1 | (v2 << 16);
1697 +}
1698 +
1699 +static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1700 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1701 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1702 +
1703 + const block_iq4_nl * bq = (const block_iq4_nl *) vbq;
1704 +
1705 + const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;
1706 + const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;
1707 +
1708 + const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1709 +
1710 + int v1, v2;
1711 + int sumi1 = 0, sumi2 = 0;
1712 + for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
1713 + const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);
1714 + get_int_from_table_16(aux, values, v1, v2);
1715 + sumi1 = __dp4a(v1, q8[l+0], sumi1);
1716 + sumi2 = __dp4a(v2, q8[l+4], sumi2);
1717 + }
1718 + const float d = __half2float(bq->d) * __low2float(bq8_1->ds);
1719 + return d * (sumi1 + sumi2);
1720 +#endif
1721 +}
1722 +
1723 +
1724 +static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
1725 + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1726 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1727 + const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;
1728 + const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1729 +
1730 + // iqs is 0...7
1731 + const int ib32 = iqs;
1732 + const int32_t * q8 = (const int *)bq8_1[ib32].qs;
1733 + const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;
1734 + const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);
1735 + const float d = __half2float(bq4->d) * (ls - 32) * __low2float(bq8_1[ib32].ds);
1736 + int v1, v2;
1737 + int sumi1 = 0, sumi2 = 0;
1738 + for (int j = 0; j < 4; ++j) {
1739 + get_int_from_table_16(q4[j], values, v1, v2);
1740 + sumi1 = __dp4a(v1, q8[j+0], sumi1);
1741 + sumi2 = __dp4a(v2, q8[j+4], sumi2);
1742 + }
1743 + return d * (sumi1 + sumi2);
1744 +#endif
1745 +}
\ No newline at end of file
csrc/torch_bindings.cpp
+12
@@ -145,6 +145,18 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
145 ops.def("awq_marlin_repack", &awq_marlin_repack);
146 ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
147
148 + // Dequantization for GGML.
149 + ops.def("ggml_dequantize", &ggml_dequantize);
150 + ops.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize);
151 +
152 + // mmvq kernel for GGML.
153 + ops.def("ggml_mul_mat_vec_a8", &ggml_mul_mat_vec_a8);
154 + ops.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8);
155 +
156 + // mmq kernel for GGML.
157 + ops.def("ggml_mul_mat_a8", &ggml_mul_mat_a8);
158 + ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
159 +
160 // fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
161 ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
162 ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
examples/gguf_inference.py new
+38
@@ -0,0 +1,38 @@
1 +from huggingface_hub import hf_hub_download
2 +
3 +from vllm import LLM, SamplingParams
4 +
5 +
6 +def run_gguf_inference(model_path):
7 + PROMPT_TEMPLATE = "<|system|>\n{system_message}</s>\n<|user|>\n{prompt}</s>\n<|assistant|>\n" # noqa: E501
8 + system_message = "You are a friendly chatbot who always responds in the style of a pirate." # noqa: E501
9 + # Sample prompts.
10 + prompts = [
11 + "How many helicopters can a human eat in one sitting?",
12 + "What's the future of AI?",
13 + ]
14 + prompts = [
15 + PROMPT_TEMPLATE.format(system_message=system_message, prompt=prompt)
16 + for prompt in prompts
17 + ]
18 + # Create a sampling params object.
19 + sampling_params = SamplingParams(temperature=0, max_tokens=128)
20 +
21 + # Create an LLM.
22 + llm = LLM(model=model_path,
23 + tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
24 + gpu_memory_utilization=0.95)
25 +
26 + outputs = llm.generate(prompts, sampling_params)
27 + # Print the outputs.
28 + for output in outputs:
29 + prompt = output.prompt
30 + generated_text = output.outputs[0].text
31 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
32 +
33 +
34 +if __name__ == "__main__":
35 + repo_id = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
36 + filename = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
37 + model = hf_hub_download(repo_id, filename=filename)
38 + run_gguf_inference(model)
format.sh
+5
@@ -242,6 +242,11 @@ echo 'vLLM isort: Done'
242 # NOTE: Keep up to date with .github/workflows/clang-format.yml
243 CLANG_FORMAT_EXCLUDES=(
244 'csrc/moe/topk_softmax_kernels.cu'
245 + 'csrc/quantization/gguf/ggml-common.h'
246 + 'csrc/quantization/gguf/dequantize.cuh'
247 + 'csrc/quantization/gguf/vecdotq.cuh'
248 + 'csrc/quantization/gguf/mmq.cuh'
249 + 'csrc/quantization/gguf/mmvq.cuh'
250 )
251
252 # Format specified files with clang-format
requirements-common.txt
+1
@@ -22,3 +22,4 @@ outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
22 typing_extensions
23 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
24 pyzmq
25 +gguf == 0.9.1
tests/models/test_gguf.py new
+76
@@ -0,0 +1,76 @@
1 +"""
2 +Tests gguf models against unquantized models generations
3 +Note: To pass the test, quantization higher than Q4 should be used
4 +"""
5 +
6 +import os
7 +
8 +import pytest
9 +from huggingface_hub import hf_hub_download
10 +
11 +from tests.quantization.utils import is_quant_method_supported
12 +
13 +from .utils import check_logprobs_close
14 +
15 +os.environ["TOKENIZERS_PARALLELISM"] = "true"
16 +
17 +MAX_MODEL_LEN = 1024
18 +
19 +# FIXME: Move this to confest
20 +MODELS = [
21 + ("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
22 + hf_hub_download("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
23 + filename="tinyllama-1.1b-chat-v1.0.Q4_0.gguf")),
24 + ("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
25 + hf_hub_download("duyntnet/TinyLlama-1.1B-Chat-v1.0-imatrix-GGUF",
26 + filename="TinyLlama-1.1B-Chat-v1.0-IQ4_XS.gguf")),
27 + ("Qwen/Qwen2-1.5B-Instruct",
28 + hf_hub_download("Qwen/Qwen2-1.5B-Instruct-GGUF",
29 + filename="qwen2-1_5b-instruct-q4_k_m.gguf")),
30 + ("Qwen/Qwen2-1.5B-Instruct",
31 + hf_hub_download("legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
32 + filename="Qwen2-1.5B-Instruct.IQ4_XS.gguf")),
33 +]
34 +
35 +
36 +@pytest.mark.skipif(not is_quant_method_supported("gguf"),
37 + reason="gguf is not supported on this GPU type.")
38 +@pytest.mark.parametrize("model", MODELS)
39 +@pytest.mark.parametrize("dtype", ["half"])
40 +@pytest.mark.parametrize("max_tokens", [32])
41 +@pytest.mark.parametrize("num_logprobs", [5])
42 +def test_models(
43 + vllm_runner,
44 + example_prompts,
45 + model,
46 + dtype: str,
47 + max_tokens: int,
48 + num_logprobs: int,
49 +) -> None:
50 + original_model, gguf_model = model
51 +
52 + # Run unquantized model.
53 + with vllm_runner(model_name=original_model,
54 + dtype=dtype,
55 + max_model_len=MAX_MODEL_LEN,
56 + enforce_eager=True,
57 + tensor_parallel_size=1) as original_model:
58 +
59 + original_outputs = original_model.generate_greedy_logprobs(
60 + example_prompts[:-1], max_tokens, num_logprobs)
61 +
62 + # Run gguf model.
63 + with vllm_runner(model_name=gguf_model,
64 + dtype=dtype,
65 + max_model_len=MAX_MODEL_LEN,
66 + enforce_eager=True,
67 + tensor_parallel_size=1) as gguf_model:
68 + gguf_outputs = gguf_model.generate_greedy_logprobs(
69 + example_prompts[:-1], max_tokens, num_logprobs)
70 +
71 + check_logprobs_close(
72 + outputs_0_lst=original_outputs,
73 + outputs_1_lst=gguf_outputs,
74 + name_0="original",
75 + name_1="gguf",
76 + )
tests/quantization/test_lm_head.py
+4 -2
@@ -7,11 +7,12 @@ from typing import Tuple
7 import pytest
8 import torch
9
10 -from vllm.model_executor.layers.linear import UnquantizedLinearMethod
10 from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
11 from vllm.model_executor.layers.quantization.gptq_marlin import (
12 GPTQMarlinLinearMethod)
13 from vllm.model_executor.layers.quantization.marlin import MarlinLinearMethod
14 +from vllm.model_executor.layers.vocab_parallel_embedding import (
15 + UnquantizedEmbeddingMethod)
16
17 PROMPT = "On the surface of Mars, we found"
18
@@ -37,7 +38,8 @@ def test_lm_head(
38 lm_head_layer.linear_method,
39 (GPTQLinearMethod, GPTQMarlinLinearMethod, MarlinLinearMethod))
40 else:
40 - assert isinstance(lm_head_layer.linear_method, UnquantizedLinearMethod)
41 + assert isinstance(lm_head_layer.linear_method,
42 + UnquantizedEmbeddingMethod)
43
44 print(
45 vllm_model.generate_greedy(prompts=["Hello my name is"],
vllm/_custom_ops.py
+32
@@ -404,6 +404,38 @@ def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
404 workspace, size_m, size_n, size_k)
405
406
407 +# gguf
408 +def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int, n: int):
409 + return torch.ops._C.ggml_dequantize(W, quant_type, m, n)
410 +
411 +
412 +def ggml_mul_mat_vec(
413 + W: torch.Tensor,
414 + X: torch.Tensor,
415 + quant_type: int,
416 + row: int,
417 +):
418 + return torch.ops._C.ggml_mul_mat_vec(W, X, quant_type, row)
419 +
420 +
421 +def ggml_mul_mat_vec_a8(
422 + W: torch.Tensor,
423 + X: torch.Tensor,
424 + quant_type: int,
425 + row: int,
426 +):
427 + return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
428 +
429 +
430 +def ggml_mul_mat_a8(
431 + W: torch.Tensor,
432 + X: torch.Tensor,
433 + quant_type: int,
434 + row: int,
435 +):
436 + return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
437 +
438 +
439 # moe
440 def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
441 block_size: int, sorted_token_ids: torch.Tensor,
vllm/config.py
+1
@@ -582,6 +582,7 @@ class LoadFormat(str, enum.Enum):
582 DUMMY = "dummy"
583 TENSORIZER = "tensorizer"
584 SHARDED_STATE = "sharded_state"
585 + GGUF = "gguf"
586 BITSANDBYTES = "bitsandbytes"
587
588
vllm/engine/arg_utils.py
+3
@@ -672,6 +672,9 @@ class EngineArgs:
672 return engine_args
673
674 def create_engine_config(self, ) -> EngineConfig:
675 + # gguf file needs a specific model loader and doesn't use hf_repo
676 + if self.model.endswith(".gguf"):
677 + self.quantization = self.load_format = "gguf"
678
679 # bitsandbytes quantization needs a specific model loader
680 # so we make sure the quant method and the load format are consistent
vllm/model_executor/layers/linear.py
+81 -1
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Tuple
3
4 import torch
5 import torch.nn.functional as F
6 -from torch.nn.parameter import Parameter
6 +from torch.nn.parameter import Parameter, UninitializedParameter
7
8 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9 get_tensor_model_parallel_world_size,
@@ -311,6 +311,17 @@ class ColumnParallelLinear(LinearBase):
311 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
312 tp_rank = get_tensor_model_parallel_rank()
313 output_dim = getattr(param, "output_dim", None)
314 +
315 + # Special case for GGUF
316 + is_gguf_weight = getattr(param, "is_gguf_weight", False)
317 + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
318 + if is_gguf_weight_type:
319 + param.weight_type = loaded_weight.item()
320 +
321 + # Materialize GGUF UninitializedParameter
322 + if is_gguf_weight and isinstance(param, UninitializedParameter):
323 + param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
324 +
325 param_data = param.data
326 if output_dim is not None:
327 shard_size = param_data.shape[output_dim]
@@ -398,6 +409,27 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
409 loaded_weight: torch.Tensor,
410 loaded_shard_id: Optional[int] = None):
411
412 + # Special case for GGUF
413 + # initialize GGUF param after we know the quantize type
414 + is_gguf_weight = getattr(param, "is_gguf_weight", False)
415 + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
416 + if is_gguf_weight_type:
417 + param.data[loaded_shard_id].copy_(loaded_weight)
418 + param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
419 + return
420 +
421 + if is_gguf_weight and isinstance(param, UninitializedParameter):
422 + from gguf.constants import GGML_QUANT_SIZES
423 +
424 + ori_shape = param.tensor_shape
425 + weight_types = self.qweight_type.shard_weight_type.values()
426 + row_size = []
427 + for weight_type in weight_types:
428 + block_size, type_size = GGML_QUANT_SIZES[weight_type]
429 + row_size.append(ori_shape[1] // block_size * type_size)
430 + q_shape = (ori_shape[0], max(row_size))
431 + param.materialize(q_shape, dtype=loaded_weight.dtype)
432 +
433 param_data = param.data
434 output_dim = getattr(param, "output_dim", None)
435 # Special case for AQLM codebooks.
@@ -460,6 +492,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
492 shard_offset = loaded_weight.shape[output_dim] * \
493 loaded_shard_id
494
495 + if is_gguf_weight:
496 + shard_size = loaded_weight.shape[output_dim]
497 + shard_offset = loaded_weight.shape[output_dim] * \
498 + loaded_shard_id
499 + param.shard_id.append(loaded_shard_id)
500 + param.shard_size[loaded_shard_id] = loaded_weight.shape
501 +
502 param_data = param_data.narrow(output_dim, shard_offset,
503 shard_size)
504 start_idx = tp_rank * shard_size
@@ -563,6 +602,29 @@ class QKVParallelLinear(ColumnParallelLinear):
602 param: Parameter,
603 loaded_weight: torch.Tensor,
604 loaded_shard_id: Optional[str] = None):
605 +
606 + # Special case for GGUF
607 + # initialize GGUF param after we know the quantize type
608 + is_gguf_weight = getattr(param, "is_gguf_weight", False)
609 + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
610 + if is_gguf_weight_type and loaded_shard_id is not None:
611 + idx_map = {"q": 0, "k": 1, "v": 2}
612 + param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
613 + param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
614 + return
615 +
616 + if is_gguf_weight and isinstance(param, UninitializedParameter):
617 + from gguf.constants import GGML_QUANT_SIZES
618 +
619 + ori_shape = param.tensor_shape
620 + weight_types = self.qweight_type.shard_weight_type.values()
621 + row_size = []
622 + for weight_type in weight_types:
623 + block_size, type_size = GGML_QUANT_SIZES[weight_type]
624 + row_size.append(ori_shape[1] // block_size * type_size)
625 + q_shape = (ori_shape[0], max(row_size))
626 + param.materialize(q_shape, dtype=loaded_weight.dtype)
627 +
628 param_data = param.data
629 output_dim = getattr(param, "output_dim", None)
630 # Special case for AQLM codebooks.
@@ -650,6 +712,13 @@ class QKVParallelLinear(ColumnParallelLinear):
712 shard_size, shard_offset = adjust_bitsandbytes_shard(
713 param, orig_qkv_offsets, loaded_shard_id)
714
715 + if is_gguf_weight:
716 + param.shard_id.append(loaded_shard_id)
717 + param.shard_size[loaded_shard_id] = loaded_weight.shape
718 + input_dim = getattr(param, "input_dim", None)
719 + input_size = loaded_weight.shape[input_dim]
720 + param_data = param_data.narrow(input_dim, 0, input_size)
721 +
722 param_data = param_data.narrow(output_dim, shard_offset,
723 shard_size)
724 if loaded_shard_id == "q":
@@ -755,6 +824,17 @@ class RowParallelLinear(LinearBase):
824 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
825 tp_rank = get_tensor_model_parallel_rank()
826 input_dim = getattr(param, "input_dim", None)
827 +
828 + # Special case for GGUF
829 + is_gguf_weight = getattr(param, "is_gguf_weight", False)
830 + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
831 + if is_gguf_weight_type:
832 + param.weight_type = loaded_weight.item()
833 +
834 + # Materialize GGUF UninitializedParameter
835 + if is_gguf_weight and isinstance(param, UninitializedParameter):
836 + param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
837 +
838 param_data = param.data
839 if input_dim is not None:
840 shard_size = param_data.shape[input_dim]
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.deepspeedfp import (
13 DeepSpeedFPConfig)
14 from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
15 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
16 +from vllm.model_executor.layers.quantization.gguf import GGUFConfig
17 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
18 from vllm.model_executor.layers.quantization.gptq_marlin import (
19 GPTQMarlinConfig)
@@ -31,6 +32,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
32 # The order of gptq methods is important for config.py iteration over
33 # override_quantization_method(..)
34 "marlin": MarlinConfig,
35 + "gguf": GGUFConfig,
36 "gptq_marlin_24": GPTQMarlin24Config,
37 "gptq_marlin": GPTQMarlinConfig,
38 "awq_marlin": AWQMarlinConfig,
vllm/model_executor/layers/quantization/base_config.py
+25 -1
@@ -1,5 +1,6 @@
1 +import inspect
2 from abc import ABC, abstractmethod
2 -from typing import Any, Dict, List, Optional
3 +from typing import Any, Dict, List, Optional, Type
4
5 import torch
6 from torch import nn
@@ -23,6 +24,14 @@ class QuantizeMethodBase(ABC):
24 Expects create_weights to have been called before on the layer."""
25 raise NotImplementedError
26
27 + # Not required functions
28 + def embedding(self, layer: torch.nn.Module, *args,
29 + **kwargs) -> torch.Tensor:
30 + """Gather embeddings in the layer based on indices in the input tensor.
31 +
32 + Expects create_weights to have been called before on the layer."""
33 + raise NotImplementedError
34 +
35 def process_weights_after_loading(self, layer: nn.Module) -> None:
36 """Process the weight after loading.
37
@@ -31,6 +40,21 @@ class QuantizeMethodBase(ABC):
40 return
41
42
43 +def method_has_implemented_embedding(
44 + method_class: Type[QuantizeMethodBase]) -> bool:
45 + """
46 + Not all quant methods have embedding implemented, so we need to check that
47 + it exists for our given method. We check this by making sure the function
48 + has been changed from the base implementation.
49 + """
50 + base_embedding = inspect.getattr_static(QuantizeMethodBase, "embedding",
51 + None)
52 + class_embedding = inspect.getattr_static(method_class, "embedding", None)
53 +
54 + return (class_embedding is not None
55 + and class_embedding is not base_embedding)
56 +
57 +
58 class QuantizationConfig(ABC):
59 """Base class for quantization configs."""
60
vllm/model_executor/layers/quantization/gguf.py new
+165
@@ -0,0 +1,165 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import gguf
4 +import torch
5 +from torch.nn.parameter import Parameter, UninitializedParameter
6 +
7 +from vllm import _custom_ops as ops
8 +from vllm.distributed import get_tensor_model_parallel_world_size
9 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 +from vllm.model_executor.layers.quantization.base_config import (
11 + QuantizationConfig, QuantizeMethodBase)
12 +from vllm.model_executor.layers.vocab_parallel_embedding import (
13 + VocabParallelEmbedding)
14 +from vllm.model_executor.utils import set_weight_attrs
15 +
16 +
17 +class GGUFConfig(QuantizationConfig):
18 + """Config class for GGUF."""
19 +
20 + def __init__(self, ) -> None:
21 + pass
22 +
23 + def __repr__(self) -> str:
24 + return ("GGUFConfig()")
25 +
26 + def get_name(self) -> str:
27 + return "gguf"
28 +
29 + def get_supported_act_dtypes(self) -> List[torch.dtype]:
30 + return [torch.half, torch.bfloat16]
31 +
32 + @classmethod
33 + def get_min_capability(cls) -> int:
34 + return 60
35 +
36 + @classmethod
37 + def get_config_filenames(cls) -> List[str]:
38 + return [] # no extra configs.
39 +
40 + @classmethod
41 + def from_config(cls, config: Dict[str, Any]) -> "GGUFConfig":
42 + if get_tensor_model_parallel_world_size() > 1:
43 + raise ValueError(
44 + "GGUF quantization hasn't supported tensor parallelism yet.")
45 + return cls()
46 +
47 + def get_quant_method(self, layer: torch.nn.Module,
48 + prefix: str) -> Optional["QuantizeMethodBase"]:
49 + if isinstance(layer, LinearBase):
50 + return GGUFLinearMethod(self)
51 + elif isinstance(layer, VocabParallelEmbedding):
52 + return GGUFEmbeddingMethod(self)
53 + return None
54 +
55 + def get_scaled_act_names(self) -> List[str]:
56 + return []
57 +
58 +
59 +def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
60 + qweight_type: int) -> torch.Tensor:
61 + # use dequantize mulmat for IQmatrix, mmq for k-quants
62 + if qweight_type >= 16:
63 + block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
64 + shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
65 + weight = ops.ggml_dequantize(qweight, qweight_type, *shape)
66 + y = x @ weight.T
67 + else:
68 + y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
69 + return y
70 +
71 +
72 +class GGUFLinearMethod(LinearMethodBase):
73 + """Linear method for GGUF.
74 +
75 + Args:
76 + quant_config: The GGUF quantization config.
77 + """
78 +
79 + def __init__(self, quant_config: GGUFConfig):
80 + self.quant_config = quant_config
81 +
82 + def create_weights(self, layer: torch.nn.Module,
83 + input_size_per_partition: int,
84 + output_partition_sizes: List[int], input_size: int,
85 + output_size: int, params_dtype: torch.dtype,
86 + **extra_weight_attrs):
87 + output_size_per_partition = sum(output_partition_sizes)
88 +
89 + tensor_shape = (output_size_per_partition, input_size_per_partition)
90 + qweight = UninitializedParameter(requires_grad=False)
91 + set_weight_attrs(
92 + qweight, {
93 + "input_dim": 1,
94 + "output_dim": 0,
95 + "tensor_shape": tensor_shape,
96 + "is_gguf_weight": True,
97 + "shard_size": {},
98 + "shard_id": [],
99 + })
100 + set_weight_attrs(qweight, extra_weight_attrs)
101 + layer.register_parameter("qweight", qweight)
102 +
103 + qweight_type = Parameter(torch.empty(len(output_partition_sizes),
104 + dtype=torch.uint8),
105 + requires_grad=False)
106 + set_weight_attrs(
107 + qweight_type, {
108 + "is_gguf_weight_type": True,
109 + "weight_type": 0,
110 + "shard_weight_type": {},
111 + "ignore_warning": True
112 + })
113 + set_weight_attrs(qweight_type, extra_weight_attrs)
114 + layer.register_parameter("qweight_type", qweight_type)
115 +
116 + def apply(self,
117 + layer: torch.nn.Module,
118 + x: torch.Tensor,
119 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
120 + shard_size = getattr(layer.qweight, "shard_size", None)
121 + shard_id = getattr(layer.qweight, "shard_id", None)
122 +
123 + if shard_id and shard_size:
124 + result = []
125 + offset = 0
126 + # dequantize shard weights respectively
127 + shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id
128 + for id in shard_id:
129 + shard_weight = layer.qweight[
130 + offset:offset +
131 + shard_size[id][0], :shard_size[id][1]].contiguous()
132 + qweight_type = layer.qweight_type.shard_weight_type[id]
133 + result.append(_fuse_mul_mat(x, shard_weight, qweight_type))
134 + offset += shard_size[id][0]
135 + out = torch.cat(result, axis=1)
136 + else:
137 + qweight = layer.qweight
138 + qweight_type = layer.qweight_type.weight_type
139 + out = _fuse_mul_mat(x, qweight, qweight_type)
140 + if bias is not None:
141 + out.add_(bias)
142 + return out
143 +
144 +
145 +class GGUFEmbeddingMethod(GGUFLinearMethod):
146 + """Embedding method for GGUF.
147 +
148 + Args:
149 + quant_config: The GGUF quantization config.
150 + """
151 +
152 + def embedding(self, layer: torch.nn.Module,
153 + x: torch.Tensor) -> torch.Tensor:
154 + qweight = layer.qweight
155 + qweight_type = layer.qweight_type.weight_type
156 +
157 + block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
158 + hidden_size = qweight.shape[1] // type_size * block_size
159 + if qweight_type < 2:
160 + return torch.embedding(qweight, x)
161 + x_flat = x.flatten()
162 + quant = torch.index_select(qweight, dim=0, index=x_flat)
163 + dequant = ops.ggml_dequantize(quant, qweight_type, hidden_size,
164 + x_flat.shape[0])
165 + return dequant.view(*x.shape, hidden_size)
vllm/model_executor/layers/vocab_parallel_embedding.py
+54 -5
@@ -3,19 +3,46 @@ from typing import List, Optional, Sequence, Tuple
3
4 import torch
5 import torch.nn.functional as F
6 -from torch.nn.parameter import Parameter
6 +from torch.nn.parameter import Parameter, UninitializedParameter
7
8 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9 get_tensor_model_parallel_world_size,
10 tensor_model_parallel_all_reduce)
11 -from vllm.model_executor.layers.linear import UnquantizedLinearMethod
11 from vllm.model_executor.layers.quantization.base_config import (
13 - QuantizationConfig, QuantizeMethodBase)
12 + QuantizationConfig, QuantizeMethodBase, method_has_implemented_embedding)
13 from vllm.model_executor.utils import set_weight_attrs
14
15 DEFAULT_VOCAB_PADDING_SIZE = 64
16
17
18 +class UnquantizedEmbeddingMethod(QuantizeMethodBase):
19 + """Unquantized method for embeddings."""
20 +
21 + def create_weights(self, layer: torch.nn.Module,
22 + input_size_per_partition: int,
23 + output_partition_sizes: List[int], input_size: int,
24 + output_size: int, params_dtype: torch.dtype,
25 + **extra_weight_attrs):
26 + """Create weights for embedding layer."""
27 + weight = Parameter(torch.empty(sum(output_partition_sizes),
28 + input_size_per_partition,
29 + dtype=params_dtype),
30 + requires_grad=False)
31 + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
32 + layer.register_parameter("weight", weight)
33 + set_weight_attrs(weight, extra_weight_attrs)
34 +
35 + def apply(self,
36 + layer: torch.nn.Module,
37 + x: torch.Tensor,
38 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
39 + return F.linear(x, layer.weight, bias)
40 +
41 + def embedding(self, layer: torch.nn.Module,
42 + input_: torch.Tensor) -> torch.Tensor:
43 + return F.embedding(input_, layer.weight)
44 +
45 +
46 def pad_vocab_size(vocab_size: int,
47 pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
48 """Pad the vocab size to the given value."""
@@ -199,7 +226,19 @@ class VocabParallelEmbedding(torch.nn.Module):
226 if quant_config is not None:
227 linear_method = quant_config.get_quant_method(self, prefix=prefix)
228 if linear_method is None:
202 - linear_method = UnquantizedLinearMethod()
229 + linear_method = UnquantizedEmbeddingMethod()
230 +
231 + # If we are making an embedding layer, then our quantization linear
232 + # method must implement the embedding operation. If we are another
233 + # layer type like ParallelLMHead, this is not important.
234 + is_embedding_layer = type(self.__class__) is VocabParallelEmbedding
235 + linear_method_implements_embedding = method_has_implemented_embedding(
236 + type(linear_method))
237 + if is_embedding_layer and not linear_method_implements_embedding:
238 + raise NotImplementedError(
239 + f"The class {type(linear_method).__name__} must implement "
240 + "the 'embedding' method, see UnquantizedEmbeddingMethod.")
241 +
242 self.linear_method: QuantizeMethodBase = linear_method
243
244 if params_dtype is None:
@@ -306,6 +345,14 @@ class VocabParallelEmbedding(torch.nn.Module):
345 output_dim = getattr(param, "output_dim", None)
346 packed_dim = getattr(param, "packed_dim", None)
347
348 + # If the parameter is a gguf weight, then load it directly.
349 + if getattr(param, "is_gguf_weight_type", None):
350 + param.data.copy_(loaded_weight)
351 + param.weight_type = loaded_weight.item()
352 + return
353 + elif isinstance(param, UninitializedParameter):
354 + param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
355 +
356 # If parameter does not have output dim, then it should
357 # be copied onto all gpus (e.g. g_idx for act_order gptq).
358 if output_dim is None:
@@ -344,7 +391,8 @@ class VocabParallelEmbedding(torch.nn.Module):
391 else:
392 masked_input = input_
393 # Get the embeddings.
347 - output_parallel = F.embedding(masked_input.long(), self.weight)
394 + output_parallel = self.linear_method.embedding(self,
395 + masked_input.long())
396 # Mask the output embedding.
397 if self.tp_size > 1:
398 output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
@@ -389,6 +437,7 @@ class ParallelLMHead(VocabParallelEmbedding):
437 super().__init__(num_embeddings, embedding_dim, params_dtype,
438 org_num_embeddings, padding_size, quant_config,
439 prefix)
440 +
441 if bias:
442 self.bias = Parameter(
443 torch.empty(self.num_embeddings_per_partition,
vllm/model_executor/model_loader/loader.py
+92 -2
@@ -10,11 +10,13 @@ from abc import ABC, abstractmethod
10 from contextlib import contextmanager
11 from typing import Any, Dict, Generator, List, Optional, Tuple, Type
12
13 +import gguf
14 import huggingface_hub
15 import numpy as np
16 import torch
17 from huggingface_hub import HfApi, hf_hub_download
18 from torch import nn
19 +from transformers import AutoModelForCausalLM
20
21 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
22 LoRAConfig, ModelConfig, MultiModalConfig,
@@ -31,8 +33,9 @@ from vllm.model_executor.model_loader.utils import (get_model_architecture,
33 from vllm.model_executor.model_loader.weight_utils import (
34 download_safetensors_index_file_from_hf, download_weights_from_hf,
35 filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
34 - get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
35 - pt_weights_iterator, safetensors_weights_iterator)
36 + get_gguf_extra_tensor_names, get_quant_config, gguf_quant_weights_iterator,
37 + initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
38 + safetensors_weights_iterator)
39 from vllm.model_executor.models.interfaces import (has_inner_state,
40 supports_lora,
41 supports_vision)
@@ -948,6 +951,90 @@ class BitsAndBytesModelLoader(BaseModelLoader):
951 return model.eval()
952
953
954 +class GGUFModelLoader(BaseModelLoader):
955 + """
956 + Model loader that can load GGUF files. This is useful for loading models
957 + that are quantized with GGUF and saved in the GGUF format. This loader
958 + supports loading both full models and sharded models.
959 + """
960 +
961 + def __init__(self, load_config: LoadConfig):
962 + super().__init__(load_config)
963 + if load_config.model_loader_extra_config:
964 + raise ValueError(f"Model loader extra config is not supported for "
965 + f"load format {load_config.load_format}")
966 +
967 + def _prepare_weights(self, model_name_or_path: str):
968 + if os.path.isfile(model_name_or_path):
969 + return model_name_or_path
970 + else:
971 + raise ValueError(f"{model_name_or_path} is not a file.")
972 +
973 + def _get_gguf_weights_map(self, model_config: ModelConfig):
974 + """
975 + GGUF uses this naming convention for their tensors from HF checkpoint:
976 + `blk.N.BB.weight` and `blk.N.BB.bias`
977 + where N signifies the block number of a layer, and BB signifies the
978 + attention/mlp layer components.
979 + See "Standardized tensor names" in
980 + https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details.
981 + """
982 + config = model_config.hf_config
983 + model_type = config.model_type
984 + # hack: ggufs have a different name than transformers
985 + if model_type == "cohere":
986 + model_type = "command-r"
987 + arch = None
988 + for key, value in gguf.MODEL_ARCH_NAMES.items():
989 + if value == model_type:
990 + arch = key
991 + break
992 + if arch is None:
993 + raise RuntimeError(f"Unknown gguf model_type: {model_type}")
994 + num_layers = config.num_hidden_layers
995 + name_map = gguf.get_tensor_name_map(arch, num_layers)
996 + with torch.device("meta"):
997 + dummy_model = AutoModelForCausalLM.from_config(config)
998 + state_dict = dummy_model.state_dict()
999 +
1000 + gguf_to_hf_name_map = {}
1001 + for hf_name in state_dict:
1002 + name, suffix = hf_name.rsplit(".", 1)
1003 + gguf_name = name_map.get_name(name)
1004 + gguf_to_hf_name_map[f"{gguf_name}.{suffix}"] = hf_name
1005 + return gguf_to_hf_name_map
1006 +
1007 + def _get_weights_iterator(
1008 + self, model_name_or_path: str, gguf_to_hf_name_map: Dict[str, str]
1009 + ) -> Generator[Tuple[str, torch.Tensor], None, None]:
1010 + return gguf_quant_weights_iterator(model_name_or_path,
1011 + gguf_to_hf_name_map)
1012 +
1013 + def load_model(self, *, model_config: ModelConfig,
1014 + device_config: DeviceConfig,
1015 + lora_config: Optional[LoRAConfig],
1016 + multimodal_config: Optional[MultiModalConfig],
1017 + parallel_config: ParallelConfig,
1018 + scheduler_config: SchedulerConfig,
1019 + cache_config: CacheConfig) -> nn.Module:
1020 +
1021 + local_model_path = self._prepare_weights(model_config.model)
1022 + gguf_weights_map = self._get_gguf_weights_map(model_config)
1023 + # we can only know if tie word embeddings after mapping weights
1024 + if "lm_head.weight" in get_gguf_extra_tensor_names(
1025 + local_model_path, gguf_weights_map):
1026 + model_config.hf_config.update({"tie_word_embeddings": True})
1027 +
1028 + with set_default_torch_dtype(model_config.dtype):
1029 + with torch.device(device_config.device):
1030 + model = _initialize_model(model_config, self.load_config,
1031 + lora_config, multimodal_config,
1032 + cache_config)
1033 + model.load_weights(
1034 + self._get_weights_iterator(local_model_path, gguf_weights_map))
1035 + return model
1036 +
1037 +
1038 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1039 """Get a model loader based on the load format."""
1040
@@ -966,4 +1053,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1053 if load_config.load_format == LoadFormat.BITSANDBYTES:
1054 return BitsAndBytesModelLoader(load_config)
1055
1056 + if load_config.load_format == LoadFormat.GGUF:
1057 + return GGUFModelLoader(load_config)
1058 +
1059 return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/weight_utils.py
+46 -1
@@ -6,9 +6,10 @@ import json
6 import os
7 import tempfile
8 from collections import defaultdict
9 -from typing import Any, Generator, Iterable, List, Optional, Tuple, Union
9 +from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union
10
11 import filelock
12 +import gguf
13 import huggingface_hub.constants
14 import numpy as np
15 import torch
@@ -121,6 +122,11 @@ def get_quant_config(model_config: ModelConfig,
122 load_config: LoadConfig) -> QuantizationConfig:
123
124 quant_cls = get_quantization_config(model_config.quantization)
125 +
126 + # GGUF doesn't have config file
127 + if model_config.quantization == "gguf":
128 + return quant_cls.from_config({})
129 +
130 # Read the quantization config from the HF model config, if available.
131 hf_quant_config = getattr(model_config.hf_config, "quantization_config",
132 None)
@@ -409,6 +415,45 @@ def pt_weights_iterator(
415 torch.cuda.empty_cache()
416
417
418 +def get_gguf_extra_tensor_names(
419 + gguf_file: str, gguf_to_hf_name_map: Dict[str, str]) -> List[str]:
420 + reader = gguf.GGUFReader(gguf_file)
421 + expected_gguf_keys = set(gguf_to_hf_name_map.keys())
422 + exact_gguf_keys = set([tensor.name for tensor in reader.tensors])
423 + extra_keys = expected_gguf_keys - exact_gguf_keys
424 + return [gguf_to_hf_name_map[key] for key in extra_keys]
425 +
426 +
427 +def gguf_quant_weights_iterator(
428 + gguf_file: str, gguf_to_hf_name_map: Dict[str, str]
429 +) -> Generator[Tuple[str, torch.Tensor], None, None]:
430 + """
431 + Iterate over the quant weights in the model gguf files and convert
432 + them to torch tensors
433 + """
434 +
435 + reader = gguf.GGUFReader(gguf_file)
436 +
437 + for tensor in reader.tensors:
438 + weight_type = tensor.tensor_type
439 + name = gguf_to_hf_name_map[tensor.name]
440 +
441 + if weight_type.name != "F32":
442 + weight_type_name = name.replace("weight", "qweight_type")
443 + weight_type = torch.tensor(weight_type)
444 + yield weight_type_name, weight_type
445 +
446 + for tensor in reader.tensors:
447 + weight = tensor.data
448 + weight_type = tensor.tensor_type
449 + name = gguf_to_hf_name_map[tensor.name]
450 +
451 + if weight_type.name != "F32":
452 + name = name.replace("weight", "qweight")
453 + param = torch.tensor(weight)
454 + yield name, param
455 +
456 +
457 def kv_cache_scales_loader(
458 filename: str, tp_rank: int, tp_size: int, num_hidden_layers: int,
459 model_type: Optional[str]) -> Iterable[Tuple[int, float]]:
vllm/model_executor/models/llama.py
+7
@@ -140,6 +140,7 @@ class LlamaAttention(nn.Module):
140 quant_config=quant_config,
141 prefix=f"{prefix}.qkv_proj",
142 )
143 +
144 self.o_proj = RowParallelLinear(
145 input_size=self.total_num_heads * self.head_dim,
146 output_size=hidden_size,
@@ -148,12 +149,17 @@ class LlamaAttention(nn.Module):
149 prefix=f"{prefix}.o_proj",
150 )
151
152 + is_neox_style = True
153 + if quant_config is not None and quant_config.get_name() == "gguf":
154 + is_neox_style = False
155 +
156 self.rotary_emb = get_rope(
157 self.head_dim,
158 rotary_dim=self.head_dim,
159 max_position=max_position_embeddings,
160 base=rope_theta,
161 rope_scaling=rope_scaling,
162 + is_neox_style=is_neox_style,
163 )
164 self.attn = Attention(self.num_heads,
165 self.head_dim,
@@ -279,6 +285,7 @@ class LlamaModel(nn.Module):
285 self.vocab_size,
286 config.hidden_size,
287 org_num_embeddings=config.vocab_size,
288 + quant_config=quant_config,
289 )
290 else:
291 self.embed_tokens = PPMissingLayer()
vllm/model_executor/models/qwen2.py
+1
@@ -238,6 +238,7 @@ class Qwen2Model(nn.Module):
238 self.embed_tokens = VocabParallelEmbedding(
239 config.vocab_size,
240 config.hidden_size,
241 + quant_config=quant_config,
242 )
243 self.start_layer, self.end_layer, self.layers = make_layers(
244 config.num_hidden_layers,
vllm/transformers_utils/config.py
+32 -8
@@ -1,7 +1,10 @@
1 import contextlib
2 -from typing import Dict, Optional, Type
2 +from pathlib import Path
3 +from typing import Dict, Optional, Type, Union
4
5 from transformers import GenerationConfig, PretrainedConfig
6 +from transformers.models.auto.modeling_auto import (
7 + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
8
9 from vllm.envs import VLLM_USE_MODELSCOPE
10 from vllm.logger import init_logger
@@ -36,18 +39,29 @@ for name, cls in _CONFIG_REGISTRY.items():
39 AutoConfig.register(name, cls)
40
41
39 -def get_config(model: str,
40 - trust_remote_code: bool,
41 - revision: Optional[str] = None,
42 - code_revision: Optional[str] = None,
43 - rope_scaling: Optional[dict] = None,
44 - rope_theta: Optional[float] = None) -> PretrainedConfig:
42 +def get_config(
43 + model: Union[str, Path],
44 + trust_remote_code: bool,
45 + revision: Optional[str] = None,
46 + code_revision: Optional[str] = None,
47 + rope_scaling: Optional[dict] = None,
48 + rope_theta: Optional[float] = None,
49 + **kwargs,
50 +) -> PretrainedConfig:
51 +
52 + # Separate model folder from file path for GGUF models
53 + is_gguf = Path(model).is_file() and Path(model).suffix == ".gguf"
54 + if is_gguf:
55 + kwargs["gguf_file"] = Path(model).name
56 + model = Path(model).parent
57 +
58 try:
59 config = AutoConfig.from_pretrained(
60 model,
61 trust_remote_code=trust_remote_code,
62 revision=revision,
50 - code_revision=code_revision)
63 + code_revision=code_revision,
64 + **kwargs)
65 except ValueError as e:
66 if (not trust_remote_code and
67 "requires you to execute the configuration file" in str(e)):
@@ -64,12 +78,22 @@ def get_config(model: str,
78 config = config_class.from_pretrained(model,
79 revision=revision,
80 code_revision=code_revision)
81 +
82 + # Special architecture mapping check for GGUF models
83 + if is_gguf:
84 + if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
85 + raise RuntimeError(
86 + f"Can't get gguf config for {config.model_type}.")
87 + model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
88 + config.update({"architectures": [model_type]})
89 +
90 for key, value in [("rope_scaling", rope_scaling),
91 ("rope_theta", rope_theta)]:
92 if value is not None:
93 logger.info("Updating %s from %r to %r", key,
94 getattr(config, key, None), value)
95 config.update({key: value})
96 +
97 return config
98
99
vllm/transformers_utils/tokenizer.py
+9 -1
@@ -1,4 +1,5 @@
1 import os
2 +from pathlib import Path
3 from typing import Optional, Union
4
5 import huggingface_hub
@@ -55,7 +56,7 @@ def get_cached_tokenizer(
56
57
58 def get_tokenizer(
58 - tokenizer_name: str,
59 + tokenizer_name: Union[str, Path],
60 *args,
61 tokenizer_mode: str = "auto",
62 trust_remote_code: bool = False,
@@ -91,6 +92,13 @@ def get_tokenizer(
92 if "truncation_side" not in kwargs:
93 kwargs["truncation_side"] = "left"
94
95 + # Separate model folder from file path for GGUF models
96 + is_gguf = Path(tokenizer_name).is_file() and Path(
97 + tokenizer_name).suffix == ".gguf"
98 + if is_gguf:
99 + kwargs["gguf_file"] = Path(tokenizer_name).name
100 + tokenizer_name = Path(tokenizer_name).parent
101 +
102 try:
103 tokenizer = AutoTokenizer.from_pretrained(
104 tokenizer_name,