A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Support loading GGUF model (#5191)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Isotr0py committed
Aug 6, 2024 at 07:54 UTC
360bd67cf0ea4a79a59c1aae736cc495a5a63ec5
29 files changed
+4970
-21
.github/workflows/clang-format.yml
+5
@@ -30,6 +30,11 @@ jobs:
30
run: |
31
EXCLUDES=(
32
'csrc/moe/topk_softmax_kernels.cu'
33
+ 'csrc/quantization/gguf/ggml-common.h'
34
+ 'csrc/quantization/gguf/dequantize.cuh'
35
+ 'csrc/quantization/gguf/vecdotq.cuh'
36
+ 'csrc/quantization/gguf/mmq.cuh'
37
+ 'csrc/quantization/gguf/mmvq.cuh'
38
)
39
find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
40
| grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
CMakeLists.txt
+1
@@ -208,6 +208,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
208
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
209
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
210
"csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
211
+ "csrc/quantization/gguf/gguf_kernel.cu"
212
"csrc/quantization/fp8/fp8_marlin.cu"
213
"csrc/custom_all_reduce.cu"
214
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
csrc/ops.h
+9
@@ -107,6 +107,15 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
107
torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
108
int64_t size_n, int64_t num_bits);
109
110
+torch::Tensor ggml_dequantize(torch::Tensor W, int8_t type, int64_t m,
111
+ int64_t n);
112
+
113
+torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X, int8_t type,
114
+ int64_t row);
115
+
116
+torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int8_t type,
117
+ int64_t row);
118
+
119
torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
120
torch::Tensor& b_scales, torch::Tensor& workspace,
121
int64_t num_bits, int64_t size_m, int64_t size_n,
csrc/quantization/gguf/dequantize.cuh
new
+531
@@ -0,0 +1,531 @@
1
+// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu
2
+// Dequant functions
3
+static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
4
+ const block_q4_0 * x = (const block_q4_0 *) vx;
5
+
6
+ const dfloat d = x[ib].d;
7
+
8
+ const int vui = x[ib].qs[iqs];
9
+
10
+ v.x = __int2half_rn(vui & 0xF);
11
+ v.y = __int2half_rn(vui >> 4);
12
+
13
+ v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f));
14
+ v = __hmul2(v, {d, d});
15
+}
16
+
17
+static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
18
+ const block_q4_1 * x = (const block_q4_1 *) vx;
19
+
20
+ const dfloat d = __low2half(x[ib].dm);
21
+ const dfloat m = __high2half(x[ib].dm);
22
+
23
+ const int vui = x[ib].qs[iqs];
24
+
25
+ v.x = __int2half_rn(vui & 0xF);
26
+ v.y = __int2half_rn(vui >> 4);
27
+
28
+ v = __hmul2(v, {d, d});
29
+ v = __hadd2(v, {m, m});
30
+}
31
+
32
+static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
33
+ const block_q5_0 * x = (const block_q5_0 *) vx;
34
+
35
+ const dfloat d = x[ib].d;
36
+
37
+ uint32_t qh;
38
+ memcpy(&qh, x[ib].qh, sizeof(qh));
39
+
40
+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
41
+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
42
+
43
+ v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
44
+ v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
45
+
46
+ v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f));
47
+ v = __hmul2(v, {d, d});
48
+}
49
+
50
+static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
51
+ const block_q5_1 * x = (const block_q5_1 *) vx;
52
+
53
+ const dfloat d = __low2half(x[ib].dm);
54
+ const dfloat m = __high2half(x[ib].dm);
55
+
56
+ uint32_t qh;
57
+ memcpy(&qh, x[ib].qh, sizeof(qh));
58
+
59
+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
60
+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
61
+
62
+ v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
63
+ v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
64
+
65
+ v = __hmul2(v, {d, d});
66
+ v = __hadd2(v, {m, m});
67
+}
68
+
69
+static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
70
+ const block_q8_0 * x = (const block_q8_0 *) vx;
71
+
72
+ const dfloat d = x[ib].d;
73
+
74
+ v.x = __int2half_rn(x[ib].qs[iqs + 0]);
75
+ v.y = __int2half_rn(x[ib].qs[iqs + 1]);
76
+
77
+ v = __hmul2(v, {d, d});
78
+}
79
+
80
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
81
+static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int k) {
82
+ const int i = 2*(blockDim.x*blockIdx.x + threadIdx.x);
83
+
84
+ if (i >= k) {
85
+ return;
86
+ }
87
+
88
+ const int ib = i/qk; // block index
89
+ const int iqs = (i%qk)/qr; // quant index
90
+ const int iybs = i - i%qk; // y block start index
91
+ const int y_offset = qr == 1 ? 1 : qk/2;
92
+
93
+ // dequantize
94
+ dfloat2 v;
95
+ dequantize_kernel(vx, ib, iqs, v);
96
+
97
+ y[iybs + iqs + 0] = v.x;
98
+ y[iybs + iqs + y_offset] = v.y;
99
+}
100
+
101
+template<typename dst_t>
102
+static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
103
+
104
+ const int i = blockIdx.x;
105
+ const block_q2_K * x = (const block_q2_K *) vx;
106
+
107
+ const int tid = threadIdx.x;
108
+ const int n = tid/32;
109
+ const int l = tid - 32*n;
110
+ const int is = 8*n + l/16;
111
+
112
+ const uint8_t q = x[i].qs[32*n + l];
113
+ dst_t * y = yy + i*QK_K + 128*n;
114
+
115
+ half dall = __low2half(x[i].dm);
116
+ half dmin = __high2half(x[i].dm);
117
+ y[l+ 0] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+0] & 0xF) * ((q >> 0) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+0] >> 4)));
118
+ y[l+32] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+2] & 0xF) * ((q >> 2) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+2] >> 4)));
119
+ y[l+64] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+4] & 0xF) * ((q >> 4) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+4] >> 4)));
120
+ y[l+96] = __hsub(__hmul(dall, __int2half_rn((x[i].scales[is+6] & 0xF) * ((q >> 6) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+6] >> 4)));
121
+}
122
+
123
+template<typename dst_t>
124
+static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
125
+
126
+ const int i = blockIdx.x;
127
+ const block_q3_K * x = (const block_q3_K *) vx;
128
+
129
+ const int r = threadIdx.x/4;
130
+ const int tid = r/2;
131
+ const int is0 = r%2;
132
+ const int l0 = 16*is0 + 4*(threadIdx.x%4);
133
+ const int n = tid / 4;
134
+ const int j = tid - 4*n;
135
+
136
+ uint8_t m = 1 << (4*n + j);
137
+ int is = 8*n + 2*j + is0;
138
+ int shift = 2*j;
139
+
140
+ int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :
141
+ is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :
142
+ is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :
143
+ (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);
144
+ half d_all = x[i].d;
145
+ half dl = __hmul(d_all, __int2half_rn(us - 32));
146
+
147
+ dst_t * y = yy + i*QK_K + 128*n + 32*j;
148
+ const uint8_t * q = x[i].qs + 32*n;
149
+ const uint8_t * hm = x[i].hmask;
150
+
151
+ for (int l = l0; l < l0+4; ++l) y[l] = __hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)));
152
+}
153
+
154
+static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {
155
+ if (j < 4) {
156
+ d = q[j] & 63; m = q[j + 4] & 63;
157
+ } else {
158
+ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
159
+ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
160
+ }
161
+}
162
+
163
+template<typename dst_t>
164
+static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
165
+ const block_q4_K * x = (const block_q4_K *) vx;
166
+
167
+ const int i = blockIdx.x;
168
+
169
+ // assume 32 threads
170
+ const int tid = threadIdx.x;
171
+ const int il = tid/8;
172
+ const int ir = tid%8;
173
+ const int is = 2*il;
174
+ const int n = 4;
175
+
176
+ dst_t * y = yy + i*QK_K + 64*il + n*ir;
177
+
178
+ const half dall = __low2half(x[i].dm);
179
+ const half dmin = __high2half(x[i].dm);
180
+
181
+ const uint8_t * q = x[i].qs + 32*il + n*ir;
182
+
183
+ uint8_t sc, m;
184
+ get_scale_min_k4(is + 0, x[i].scales, sc, m);
185
+ const half d1 = __hmul(dall, __int2half_rn(sc));
186
+ const half m1 = __hmul(dmin, __int2half_rn(m));
187
+ get_scale_min_k4(is + 1, x[i].scales, sc, m);
188
+ const half d2 = __hmul(dall, __int2half_rn(sc));
189
+ const half m2 = __hmul(dmin, __int2half_rn(m));
190
+ for (int l = 0; l < n; ++l) {
191
+ y[l + 0] = __hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1);
192
+ y[l +32] = __hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2);
193
+ }
194
+}
195
+
196
+template<typename dst_t>
197
+static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
198
+ const block_q5_K * x = (const block_q5_K *) vx;
199
+
200
+ const int i = blockIdx.x;
201
+
202
+ // assume 64 threads - this is very slightly better than the one below
203
+ const int tid = threadIdx.x;
204
+ const int il = tid/16; // il is in 0...3
205
+ const int ir = tid%16; // ir is in 0...15
206
+ const int is = 2*il; // is is in 0...6
207
+
208
+ dst_t * y = yy + i*QK_K + 64*il + 2*ir;
209
+
210
+ const half dall = __low2half(x[i].dm);
211
+ const half dmin = __high2half(x[i].dm);
212
+
213
+ const uint8_t * ql = x[i].qs + 32*il + 2*ir;
214
+ const uint8_t * qh = x[i].qh + 2*ir;
215
+
216
+ uint8_t sc, m;
217
+ get_scale_min_k4(is + 0, x[i].scales, sc, m);
218
+ const half d1 = __hmul(dall, __int2half_rn(sc)); const half m1 = __hmul(dmin, __int2half_rn(m));
219
+ get_scale_min_k4(is + 1, x[i].scales, sc, m);
220
+ const half d2 = __hmul(dall, __int2half_rn(sc)); const half m2 = __hmul(dmin, __int2half_rn(m));
221
+
222
+ uint8_t hm = 1 << (2*il);
223
+ y[ 0] = __hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1);
224
+ y[ 1] = __hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1);
225
+ hm <<= 1;
226
+ y[32] = __hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2);
227
+ y[33] = __hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2);
228
+}
229
+
230
+template<typename dst_t>
231
+static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
232
+ const block_q6_K * x = (const block_q6_K *) vx;
233
+
234
+ const int i = blockIdx.x;
235
+
236
+ // assume 64 threads - this is very slightly better than the one below
237
+ const int tid = threadIdx.x;
238
+ const int ip = tid/32; // ip is 0 or 1
239
+ const int il = tid - 32*ip; // 0...32
240
+ const int is = 8*ip + il/16;
241
+
242
+ dst_t * y = yy + i*QK_K + 128*ip + il;
243
+
244
+ const half d = x[i].d;
245
+
246
+ const uint8_t * ql = x[i].ql + 64*ip + il;
247
+ const uint8_t qh = x[i].qh[32*ip + il];
248
+ const int8_t * sc = x[i].scales + is;
249
+
250
+ y[ 0] = __hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32)));
251
+ y[32] = __hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32)));
252
+ y[64] = __hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32)));
253
+ y[96] = __hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32)));
254
+}
255
+
256
+template<typename dst_t>
257
+static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
258
+
259
+ const int i = blockIdx.x;
260
+ const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
261
+
262
+ const int tid = threadIdx.x;
263
+ const int il = tid/8; // 0...3
264
+ const int ib = tid%8; // 0...7
265
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
266
+ const uint16_t * q2 = x[i].qs + 4*ib;
267
+ const uint8_t * aux8 = (const uint8_t *)q2;
268
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);
269
+ const uint32_t aux32 = q2[2] | (q2[3] << 16);
270
+ const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f;
271
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
272
+ for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
273
+}
274
+
275
+template<typename dst_t>
276
+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
277
+
278
+ const int i = blockIdx.x;
279
+ const block_iq2_xs * x = (const block_iq2_xs *) vx;
280
+
281
+ const int tid = threadIdx.x;
282
+ const int il = tid/8; // 0...3
283
+ const int ib = tid%8; // 0...7
284
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
285
+ const uint16_t * q2 = x[i].qs + 4*ib;
286
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
287
+ const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
288
+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
289
+ for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
290
+
291
+}
292
+
293
+template<typename dst_t>
294
+static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
295
+
296
+ const int i = blockIdx.x;
297
+ const block_iq2_s * x = (const block_iq2_s *) vx;
298
+
299
+ const int tid = threadIdx.x;
300
+ const int il = tid/8; // 0...3
301
+ const int ib = tid%8; // 0...7
302
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
303
+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));
304
+ const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
305
+ const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];
306
+ for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
307
+}
308
+
309
+template<typename dst_t>
310
+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
311
+
312
+ const int i = blockIdx.x;
313
+ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
314
+
315
+ const int tid = threadIdx.x;
316
+ const int il = tid/8; // 0...3
317
+ const int ib = tid%8; // 0...7
318
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
319
+ const uint8_t * q3 = x[i].qs + 8*ib;
320
+ const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;
321
+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);
322
+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);
323
+ const uint32_t aux32 = gas[0] | (gas[1] << 16);
324
+ const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f;
325
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
326
+ for (int j = 0; j < 4; ++j) {
327
+ y[j+0] = __float2half(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));
328
+ y[j+4] = __float2half(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));
329
+ }
330
+}
331
+
332
+template<typename dst_t>
333
+static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
334
+
335
+ const int i = blockIdx.x;
336
+ const block_iq3_s * x = (const block_iq3_s *) vx;
337
+
338
+ const int tid = threadIdx.x;
339
+ const int il = tid/8; // 0...3
340
+ const int ib = tid%8; // 0...7
341
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
342
+ const uint8_t * qs = x[i].qs + 8*ib;
343
+ const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));
344
+ const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));
345
+ const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f;
346
+ const uint8_t signs = x[i].signs[4*ib + il];
347
+ for (int j = 0; j < 4; ++j) {
348
+ y[j+0] = __float2half(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));
349
+ y[j+4] = __float2half(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));
350
+ }
351
+}
352
+
353
+template<typename dst_t>
354
+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
355
+
356
+ const int i = blockIdx.x;
357
+ const block_iq1_s * x = (const block_iq1_s *) vx;
358
+
359
+ const int tid = threadIdx.x;
360
+ const int il = tid/8; // 0...3
361
+ const int ib = tid%8; // 0...7
362
+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;
363
+ const int i8 = 4*ib+il;
364
+ uint8_t h = x[i].scales[i8/2] >> 4*(i8%2);
365
+ const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5)));
366
+ const float d = __half2float(x[i].d) * (2*(h & 7) + 1);
367
+ for (int j = 0; j < 8; ++j) y[j] = __float2half(d * grid[j]);
368
+}
369
+
370
+template<typename dst_t>
371
+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {
372
+
373
+ const int i = blockIdx.x;
374
+ const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);
375
+
376
+ const int tid = threadIdx.x;
377
+ const int il = tid/8; // 0...3
378
+ const int ib = tid%8; // 0...7
379
+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;
380
+ const uint8_t * q4 = x[ib].qs + 4*il;
381
+ const float d = __half2float(x[ib].d);
382
+ for (int j = 0; j < 4; ++j) {
383
+ y[j+ 0] = __float2half(d * kvalues_iq4nl[q4[j] & 0xf]);
384
+ y[j+16] = __float2half(d * kvalues_iq4nl[q4[j] >> 4]);
385
+ }
386
+
387
+}
388
+
389
+template<typename dst_t>
390
+static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
391
+ const int i = blockIdx.x;
392
+ const block_iq4_xs * x = (const block_iq4_xs *)vx;
393
+
394
+ const int tid = threadIdx.x;
395
+ const int il = tid/8; // 0...3
396
+ const int ib = tid%8; // 0...7
397
+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;
398
+ const uint8_t * q4 = x[i].qs + 16*ib + 4*il;
399
+ const float d = __half2float(x[i].d) * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);
400
+ for (int j = 0; j < 4; ++j) {
401
+ y[j+ 0] = __float2half(d * kvalues_iq4nl[q4[j] & 0xf]);
402
+ y[j+16] = __float2half(d * kvalues_iq4nl[q4[j] >> 4]);
403
+ }
404
+}
405
+
406
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
407
+static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {
408
+ const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);
409
+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
410
+}
411
+
412
+template<typename dst_t>
413
+static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
414
+ const int nb = k / QK_K;
415
+ dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
416
+}
417
+
418
+template<typename dst_t>
419
+static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
420
+ const int nb = k / QK_K;
421
+ dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
422
+}
423
+
424
+template<typename dst_t>
425
+static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
426
+ const int nb = k / QK_K;
427
+ dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
428
+}
429
+
430
+template<typename dst_t>
431
+static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
432
+ const int nb = k / QK_K;
433
+ dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
434
+}
435
+
436
+template<typename dst_t>
437
+static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
438
+ const int nb = k / QK_K;
439
+ dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
440
+}
441
+
442
+template<typename dst_t>
443
+static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
444
+ const int nb = k / QK_K;
445
+ dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
446
+}
447
+
448
+template<typename dst_t>
449
+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
450
+ const int nb = k / QK_K;
451
+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
452
+}
453
+
454
+template<typename dst_t>
455
+static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
456
+ const int nb = k / QK_K;
457
+ dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
458
+}
459
+
460
+template<typename dst_t>
461
+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
462
+ const int nb = k / QK_K;
463
+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
464
+}
465
+
466
+template<typename dst_t>
467
+static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
468
+ const int nb = k / QK_K;
469
+ dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
470
+}
471
+
472
+template<typename dst_t>
473
+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
474
+ const int nb = k / QK_K;
475
+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
476
+}
477
+
478
+template<typename dst_t>
479
+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
480
+ const int nb = (k + QK_K - 1) / QK_K;
481
+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);
482
+}
483
+
484
+template<typename dst_t>
485
+static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {
486
+ const int nb = (k + QK_K - 1) / QK_K;
487
+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
488
+}
489
+
490
+static to_fp16_cuda_t ggml_get_to_fp16_cuda(int type) {
491
+ switch (type) {
492
+ case 2:
493
+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
494
+ case 3:
495
+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
496
+ case 6:
497
+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
498
+ case 7:
499
+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
500
+ case 8:
501
+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
502
+ case 10:
503
+ return dequantize_row_q2_K_cuda;
504
+ case 11:
505
+ return dequantize_row_q3_K_cuda;
506
+ case 12:
507
+ return dequantize_row_q4_K_cuda;
508
+ case 13:
509
+ return dequantize_row_q5_K_cuda;
510
+ case 14:
511
+ return dequantize_row_q6_K_cuda;
512
+ case 16:
513
+ return dequantize_row_iq2_xxs_cuda;
514
+ case 17:
515
+ return dequantize_row_iq2_xs_cuda;
516
+ case 18:
517
+ return dequantize_row_iq3_xxs_cuda;
518
+ case 19:
519
+ return dequantize_row_iq1_s_cuda;
520
+ case 20:
521
+ return dequantize_row_iq4_nl_cuda;
522
+ case 21:
523
+ return dequantize_row_iq3_s_cuda;
524
+ case 22:
525
+ return dequantize_row_iq2_s_cuda;
526
+ case 23:
527
+ return dequantize_row_iq4_xs_cuda;
528
+ default:
529
+ return nullptr;
530
+ }
531
+}
\ No newline at end of file
csrc/quantization/gguf/ggml-common.h
new
+969
@@ -0,0 +1,969 @@
1
+// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
2
+#define QK_K 256
3
+#define K_QUANTS_PER_ITERATION 2
4
+#define WARP_SIZE 32
5
+#define K_SCALE_SIZE 12
6
+#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
7
+#define CUDA_QUANTIZE_BLOCK_SIZE 256
8
+#define GGML_CUDA_DMMV_X 32
9
+#define GGML_CUDA_MMV_Y 1
10
+
11
+
12
+// Data Structures
13
+// QK = number of values after dequantization
14
+// QR = QK / number of values before dequantization
15
+// QI = number of 32 bit integers before dequantization
16
+
17
+#define QK4_0 32
18
+#define QR4_0 2
19
+#define QI4_0 (QK4_0 / (4 * QR4_0))
20
+typedef struct {
21
+ half d; // delta
22
+ uint8_t qs[QK4_0 / 2]; // nibbles / quants
23
+} block_q4_0;
24
+
25
+#define QK4_1 32
26
+#define QR4_1 2
27
+#define QI4_1 (QK4_1 / (4 * QR4_1))
28
+typedef struct {
29
+ half2 dm; // dm.x = delta, dm.y = min
30
+ uint8_t qs[QK4_1 / 2]; // nibbles / quants
31
+} block_q4_1;
32
+
33
+#define QK5_0 32
34
+#define QR5_0 2
35
+#define QI5_0 (QK5_0 / (4 * QR5_0))
36
+typedef struct {
37
+ half d; // delta
38
+ uint8_t qh[4]; // 5-th bit of quants
39
+ uint8_t qs[QK5_0 / 2]; // nibbles / quants
40
+} block_q5_0;
41
+
42
+#define QK5_1 32
43
+#define QR5_1 2
44
+#define QI5_1 (QK5_1 / (4 * QR5_1))
45
+typedef struct {
46
+ half2 dm; // dm.x = delta, dm.y = min
47
+ uint8_t qh[4]; // 5-th bit of quants
48
+ uint8_t qs[QK5_1 / 2]; // nibbles / quants
49
+} block_q5_1;
50
+
51
+#define QK8_0 32
52
+#define QR8_0 1
53
+#define QI8_0 (QK8_0 / (4 * QR8_0))
54
+typedef struct {
55
+ half d; // delta
56
+ int8_t qs[QK8_0]; // quants
57
+} block_q8_0;
58
+
59
+#define QK8_1 32
60
+#define QR8_1 1
61
+#define QI8_1 (QK8_1 / (4 * QR8_1))
62
+typedef struct {
63
+ half2 ds; // ds.x = delta, ds.y = sum
64
+ int8_t qs[QK8_0]; // quants
65
+} block_q8_1;
66
+
67
+#define QR2_K 4
68
+#define QI2_K (QK_K / (4*QR2_K))
69
+typedef struct {
70
+ uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
71
+ uint8_t qs[QK_K/4]; // quants
72
+ half2 dm; // super-block scale for quantized scales/mins
73
+} block_q2_K;
74
+
75
+#define QR3_K 4
76
+#define QI3_K (QK_K / (4*QR3_K))
77
+typedef struct {
78
+ uint8_t hmask[QK_K/8]; // quants - high bit
79
+ uint8_t qs[QK_K/4]; // quants - low 2 bits
80
+ uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits
81
+ half d; // super-block scale
82
+} block_q3_K;
83
+
84
+#define QR4_K 2
85
+#define QI4_K (QK_K / (4*QR4_K))
86
+typedef struct {
87
+ half2 dm; // super-block scale for quantized scales/mins
88
+ uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits
89
+ uint8_t qs[QK_K/2]; // 4--bit quants
90
+} block_q4_K;
91
+
92
+#define QR5_K 2
93
+#define QI5_K (QK_K / (4*QR5_K))
94
+typedef struct {
95
+ half2 dm; // super-block scale for quantized scales/mins
96
+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
97
+ uint8_t qh[QK_K/8]; // quants, high bit
98
+ uint8_t qs[QK_K/2]; // quants, low 4 bits
99
+} block_q5_K;
100
+
101
+#define QR6_K 2
102
+#define QI6_K (QK_K / (4*QR6_K))
103
+typedef struct {
104
+ uint8_t ql[QK_K/2]; // quants, lower 4 bits
105
+ uint8_t qh[QK_K/4]; // quants, upper 2 bits
106
+ int8_t scales[QK_K/16]; // scales
107
+ half d; // delta
108
+} block_q6_K;
109
+
110
+#define QR2_XXS 8
111
+#define QI2_XXS (QK_K / (4*QR2_XXS))
112
+typedef struct {
113
+ half d;
114
+ uint16_t qs[QK_K/8];
115
+} block_iq2_xxs;
116
+
117
+#define QR2_XS 8
118
+#define QI2_XS (QK_K / (4*QR2_XS))
119
+typedef struct {
120
+ half d;
121
+ uint16_t qs[QK_K/8];
122
+ uint8_t scales[QK_K/32];
123
+} block_iq2_xs;
124
+
125
+#define QR2_S 8
126
+#define QI2_S (QK_K / (4*QR2_S))
127
+typedef struct {
128
+ half d;
129
+ uint8_t qs[QK_K/4];
130
+ uint8_t qh[QK_K/32];
131
+ uint8_t scales[QK_K/32];
132
+} block_iq2_s;
133
+
134
+#define QR3_XXS 8
135
+#define QI3_XXS (QK_K / (4*QR3_XXS))
136
+typedef struct {
137
+ half d;
138
+ uint8_t qs[3*(QK_K/8)];
139
+} block_iq3_xxs;
140
+
141
+#define QR3_XS 8
142
+#define QI3_XS (QK_K / (4*QR3_XS))
143
+#define IQ3S_N_SCALE QK_K/64
144
+typedef struct {
145
+ half d;
146
+ uint8_t qs[QK_K/4];
147
+ uint8_t qh[QK_K/32];
148
+ uint8_t signs[QK_K/8];
149
+ uint8_t scales[IQ3S_N_SCALE];
150
+} block_iq3_s;
151
+
152
+#define QR1_S 8
153
+#define QI1_S (QK_K / (4*QR1_S))
154
+typedef struct {
155
+ half d;
156
+ uint8_t qs[QK_K/8];
157
+ uint8_t scales[QK_K/16];
158
+} block_iq1_s;
159
+
160
+#define QK4_NL 32
161
+#define QR4_NL 2
162
+#define QI4_NL (QK4_NL / (4*QR4_NL))
163
+typedef struct {
164
+ half d;
165
+ uint8_t qs[QK4_NL/2];
166
+} block_iq4_nl;
167
+
168
+#define QR4_XS 8
169
+#define QI4_XS (QK_K / (4*QR4_XS))
170
+typedef struct {
171
+ half d;
172
+ uint16_t scales_h;
173
+ uint8_t scales_l[QK_K/64];
174
+ uint8_t qs[QK_K/2];
175
+} block_iq4_xs;
176
+
177
+static const __device__ uint64_t iq2xxs_grid[256] = {
178
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
179
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,
180
+ 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,
181
+ 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,
182
+ 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,
183
+ 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,
184
+ 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,
185
+ 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,
186
+ 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,
187
+ 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,
188
+ 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,
189
+ 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,
190
+ 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,
191
+ 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,
192
+ 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,
193
+ 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,
194
+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,
195
+ 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,
196
+ 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,
197
+ 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,
198
+ 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,
199
+ 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,
200
+ 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,
201
+ 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,
202
+ 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,
203
+ 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,
204
+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,
205
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,
206
+ 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,
207
+ 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,
208
+ 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,
209
+ 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,
210
+ 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,
211
+ 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,
212
+ 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,
213
+ 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,
214
+ 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,
215
+ 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,
216
+ 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,
217
+ 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,
218
+ 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,
219
+ 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,
220
+ 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,
221
+ 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,
222
+ 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,
223
+ 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,
224
+ 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,
225
+ 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,
226
+ 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,
227
+ 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,
228
+ 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,
229
+ 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,
230
+ 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,
231
+ 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,
232
+ 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,
233
+ 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,
234
+ 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,
235
+ 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,
236
+ 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,
237
+ 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,
238
+ 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,
239
+ 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,
240
+ 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,
241
+ 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,
242
+};
243
+
244
+static const __device__ uint64_t iq2xs_grid[512] = {
245
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
246
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
247
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
248
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
249
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
250
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,
251
+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,
252
+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,
253
+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,
254
+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,
255
+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,
256
+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,
257
+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,
258
+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,
259
+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,
260
+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,
261
+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,
262
+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,
263
+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,
264
+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,
265
+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,
266
+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,
267
+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,
268
+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,
269
+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,
270
+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,
271
+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,
272
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,
273
+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,
274
+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,
275
+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,
276
+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,
277
+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,
278
+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,
279
+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,
280
+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,
281
+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,
282
+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,
283
+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,
284
+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,
285
+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,
286
+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,
287
+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,
288
+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,
289
+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,
290
+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,
291
+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,
292
+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,
293
+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,
294
+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,
295
+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,
296
+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,
297
+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,
298
+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,
299
+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,
300
+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,
301
+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,
302
+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,
303
+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,
304
+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,
305
+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,
306
+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,
307
+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,
308
+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,
309
+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,
310
+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,
311
+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,
312
+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,
313
+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,
314
+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,
315
+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,
316
+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,
317
+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,
318
+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,
319
+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,
320
+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,
321
+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,
322
+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,
323
+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,
324
+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,
325
+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,
326
+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,
327
+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,
328
+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,
329
+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,
330
+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,
331
+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,
332
+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,
333
+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,
334
+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,
335
+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,
336
+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,
337
+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,
338
+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,
339
+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,
340
+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,
341
+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,
342
+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,
343
+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,
344
+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,
345
+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,
346
+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,
347
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,
348
+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,
349
+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,
350
+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,
351
+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,
352
+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,
353
+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,
354
+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,
355
+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,
356
+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,
357
+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,
358
+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,
359
+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,
360
+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,
361
+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,
362
+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,
363
+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,
364
+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,
365
+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,
366
+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,
367
+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,
368
+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,
369
+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,
370
+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,
371
+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,
372
+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,
373
+};
374
+
375
+static const __device__ uint64_t iq2s_grid[1024] = {
376
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
377
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
378
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
379
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
380
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
381
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,
382
+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,
383
+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,
384
+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,
385
+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
386
+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,
387
+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,
388
+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,
389
+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,
390
+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
391
+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,
392
+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,
393
+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,
394
+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,
395
+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
396
+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,
397
+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,
398
+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,
399
+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,
400
+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
401
+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,
402
+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,
403
+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,
404
+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,
405
+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
406
+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,
407
+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,
408
+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,
409
+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,
410
+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
411
+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,
412
+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,
413
+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,
414
+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,
415
+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
416
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,
417
+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,
418
+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,
419
+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,
420
+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
421
+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,
422
+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,
423
+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,
424
+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,
425
+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
426
+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,
427
+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,
428
+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,
429
+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,
430
+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
431
+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,
432
+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,
433
+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,
434
+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,
435
+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
436
+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,
437
+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,
438
+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,
439
+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,
440
+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
441
+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,
442
+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,
443
+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,
444
+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,
445
+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
446
+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,
447
+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,
448
+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,
449
+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,
450
+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
451
+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,
452
+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,
453
+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,
454
+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,
455
+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
456
+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,
457
+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,
458
+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,
459
+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,
460
+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
461
+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,
462
+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,
463
+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,
464
+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,
465
+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
466
+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,
467
+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,
468
+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,
469
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,
470
+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
471
+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,
472
+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,
473
+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,
474
+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,
475
+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
476
+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,
477
+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,
478
+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,
479
+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,
480
+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
481
+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,
482
+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,
483
+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,
484
+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,
485
+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
486
+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,
487
+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,
488
+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,
489
+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,
490
+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
491
+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,
492
+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,
493
+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
494
+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,
495
+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
496
+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,
497
+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,
498
+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,
499
+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,
500
+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
501
+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,
502
+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,
503
+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,
504
+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,
505
+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
506
+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,
507
+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,
508
+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,
509
+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,
510
+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
511
+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,
512
+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,
513
+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,
514
+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,
515
+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
516
+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,
517
+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,
518
+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,
519
+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,
520
+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
521
+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,
522
+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,
523
+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,
524
+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,
525
+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
526
+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,
527
+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,
528
+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,
529
+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,
530
+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
531
+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,
532
+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,
533
+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,
534
+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,
535
+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
536
+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,
537
+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,
538
+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,
539
+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,
540
+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
541
+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,
542
+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,
543
+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,
544
+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,
545
+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
546
+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,
547
+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,
548
+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,
549
+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,
550
+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
551
+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,
552
+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,
553
+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,
554
+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,
555
+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
556
+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,
557
+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,
558
+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,
559
+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,
560
+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
561
+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,
562
+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,
563
+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,
564
+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,
565
+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
566
+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,
567
+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,
568
+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,
569
+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,
570
+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
571
+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,
572
+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,
573
+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,
574
+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,
575
+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
576
+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,
577
+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,
578
+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,
579
+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,
580
+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
581
+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
582
+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,
583
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,
584
+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,
585
+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
586
+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,
587
+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,
588
+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,
589
+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,
590
+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
591
+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,
592
+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,
593
+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,
594
+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,
595
+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
596
+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,
597
+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,
598
+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,
599
+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,
600
+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
601
+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,
602
+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,
603
+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,
604
+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,
605
+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
606
+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,
607
+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,
608
+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,
609
+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,
610
+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
611
+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,
612
+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,
613
+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,
614
+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,
615
+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
616
+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,
617
+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,
618
+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,
619
+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,
620
+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
621
+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,
622
+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,
623
+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,
624
+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,
625
+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
626
+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,
627
+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,
628
+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,
629
+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,
630
+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
631
+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
632
+};
633
+
634
+static const __device__ uint32_t iq3xxs_grid[256] = {
635
+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
636
+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
637
+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
638
+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
639
+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
640
+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
641
+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
642
+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
643
+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
644
+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
645
+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
646
+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
647
+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
648
+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
649
+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
650
+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
651
+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
652
+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
653
+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
654
+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
655
+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
656
+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
657
+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
658
+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
659
+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
660
+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
661
+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
662
+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
663
+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
664
+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
665
+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
666
+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,
667
+};
668
+
669
+static const __device__ uint32_t iq3xs_grid[512] = {
670
+ 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14,
671
+ 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414,
672
+ 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24,
673
+ 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c,
674
+ 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c,
675
+ 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34,
676
+ 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c,
677
+ 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414,
678
+ 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,
679
+ 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404,
680
+ 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434,
681
+ 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c,
682
+ 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404,
683
+ 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414,
684
+ 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414,
685
+ 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404,
686
+ 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c,
687
+ 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,
688
+ 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404,
689
+ 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e,
690
+ 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14,
691
+ 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c,
692
+ 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424,
693
+ 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c,
694
+ 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c,
695
+ 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e,
696
+ 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,
697
+ 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e,
698
+ 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424,
699
+ 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e,
700
+ 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424,
701
+ 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404,
702
+ 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c,
703
+ 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e,
704
+ 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c,
705
+ 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,
706
+ 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c,
707
+ 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404,
708
+ 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04,
709
+ 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c,
710
+ 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414,
711
+ 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c,
712
+ 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c,
713
+ 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424,
714
+ 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,
715
+ 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c,
716
+ 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414,
717
+ 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c,
718
+ 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e,
719
+ 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04,
720
+ 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424,
721
+ 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14,
722
+ 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34,
723
+ 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,
724
+ 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434,
725
+ 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c,
726
+ 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424,
727
+ 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24,
728
+ 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24,
729
+ 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e,
730
+ 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c,
731
+ 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c,
732
+ 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,
733
+ 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,
734
+};
735
+
736
+static const __device__ uint64_t iq1s_grid[512] = {
737
+ 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,
738
+ 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,
739
+ 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,
740
+ 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,
741
+ 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,
742
+ 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,
743
+ 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,
744
+ 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,
745
+ 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,
746
+ 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,
747
+ 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,
748
+ 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,
749
+ 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,
750
+ 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,
751
+ 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,
752
+ 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,
753
+ 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,
754
+ 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,
755
+ 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,
756
+ 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,
757
+ 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,
758
+ 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,
759
+ 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,
760
+ 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,
761
+ 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,
762
+ 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,
763
+ 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,
764
+ 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,
765
+ 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,
766
+ 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,
767
+ 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,
768
+ 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,
769
+ 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,
770
+ 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,
771
+ 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,
772
+ 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,
773
+ 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,
774
+ 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,
775
+ 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,
776
+ 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,
777
+ 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,
778
+ 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,
779
+ 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,
780
+ 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,
781
+ 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,
782
+ 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,
783
+ 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,
784
+ 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,
785
+ 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,
786
+ 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,
787
+ 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,
788
+ 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,
789
+ 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,
790
+ 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,
791
+ 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,
792
+ 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,
793
+ 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,
794
+ 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,
795
+ 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,
796
+ 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,
797
+ 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,
798
+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,
799
+ 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,
800
+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,
801
+ 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,
802
+ 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,
803
+ 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,
804
+ 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,
805
+ 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,
806
+ 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,
807
+ 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,
808
+ 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,
809
+ 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,
810
+ 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,
811
+ 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,
812
+ 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,
813
+ 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,
814
+ 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,
815
+ 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,
816
+ 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,
817
+ 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,
818
+ 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,
819
+ 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,
820
+ 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,
821
+ 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,
822
+ 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,
823
+ 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,
824
+ 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,
825
+ 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,
826
+ 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,
827
+ 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,
828
+ 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,
829
+ 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,
830
+ 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,
831
+ 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,
832
+ 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,
833
+ 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,
834
+ 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,
835
+ 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,
836
+ 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,
837
+ 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,
838
+ 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,
839
+ 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,
840
+ 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,
841
+ 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,
842
+ 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,
843
+ 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,
844
+ 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,
845
+ 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,
846
+ 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,
847
+ 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,
848
+ 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,
849
+ 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,
850
+ 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,
851
+ 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,
852
+ 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,
853
+ 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,
854
+ 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,
855
+ 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,
856
+ 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,
857
+ 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,
858
+ 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,
859
+ 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,
860
+ 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,
861
+ 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,
862
+ 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,
863
+ 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,
864
+ 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,
865
+};
866
+
867
+static const __device__ uint8_t ksigns_iq2xs[128] = {
868
+ 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,
869
+ 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,
870
+ 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,
871
+ 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,
872
+ 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,
873
+ 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,
874
+ 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,
875
+ 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
876
+};
877
+
878
+static const __device__ uint64_t ksigns64[128] = {
879
+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
880
+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,
881
+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,
882
+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,
883
+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,
884
+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,
885
+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,
886
+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,
887
+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,
888
+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,
889
+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,
890
+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,
891
+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,
892
+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,
893
+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,
894
+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,
895
+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,
896
+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,
897
+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,
898
+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,
899
+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,
900
+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,
901
+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,
902
+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,
903
+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,
904
+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,
905
+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,
906
+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,
907
+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,
908
+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,
909
+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,
910
+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
911
+};
912
+
913
+static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};
914
+static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
915
+
916
+
917
+typedef half dfloat; // dequantize float
918
+typedef half2 dfloat2;
919
+typedef void (*dequantize_kernel_t)(const void * vx, const int ib, const int iqs, dfloat2 & v);
920
+typedef void (*to_fp16_cuda_t)(const void * __restrict__ x, dfloat * __restrict__ y, int k, cudaStream_t stream);
921
+typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs);
922
+typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc);
923
+typedef void (*load_tiles_cuda_t)(
924
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
925
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row);
926
+typedef float (*vec_dot_q_mul_mat_cuda_t)(
927
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
928
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k);
929
+
930
+// Utility function
931
+
932
+#if defined(USE_ROCM)
933
+
934
+#ifndef __has_builtin
935
+ #define __has_builtin(x) 0
936
+#endif
937
+
938
+typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
939
+static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
940
+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
941
+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
942
+#if __has_builtin(__builtin_elementwise_sub_sat)
943
+ const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);
944
+ return reinterpret_cast<const int &>(c);
945
+#else
946
+ int8x4_t c;
947
+ int16_t tmp;
948
+#pragma unroll
949
+ for (int i = 0; i < 4; i++) {
950
+ tmp = va[i] - vb[i];
951
+ if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();
952
+ if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();
953
+ c[i] = tmp;
954
+ }
955
+ return reinterpret_cast<int &>(c);
956
+#endif // __has_builtin(__builtin_elementwise_sub_sat)
957
+}
958
+
959
+static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
960
+#if __has_builtin(__builtin_amdgcn_sdot4)
961
+ c = __builtin_amdgcn_sdot4(a, b, c, false);
962
+#else
963
+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
964
+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
965
+ c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];
966
+#endif
967
+ return c;
968
+}
969
+#endif // defined(USE_ROCM)
csrc/quantization/gguf/gguf_kernel.cu
new
+242
@@ -0,0 +1,242 @@
1
+#include <cuda_fp16.h>
2
+#include <cuda_runtime.h>
3
+
4
+#include <torch/all.h>
5
+#include <c10/cuda/CUDAGuard.h>
6
+
7
+#include "ggml-common.h"
8
+#include "vecdotq.cuh"
9
+#include "dequantize.cuh"
10
+#include "mmvq.cuh"
11
+#include "mmq.cuh"
12
+
13
+// Q8 gemv
14
+static __global__ void quantize_q8_1(const half* __restrict__ x,
15
+ void* __restrict__ vy, const int kx,
16
+ const int kx_padded) {
17
+ const int ix = blockDim.x * blockIdx.x + threadIdx.x;
18
+ if (ix >= kx_padded) {
19
+ return;
20
+ }
21
+ const int iy = blockDim.y * blockIdx.y + threadIdx.y;
22
+ const int i_padded = iy * kx_padded + ix;
23
+
24
+ block_q8_1* y = (block_q8_1*)vy;
25
+
26
+ const int ib = i_padded / QK8_1; // block index
27
+ const int iqs = i_padded % QK8_1; // quant index
28
+
29
+ const float xi = ix < kx ? __half2float(x[iy * kx + ix]) : 0.0f;
30
+ float amax = fabsf(xi);
31
+ float sum = xi;
32
+
33
+#pragma unroll
34
+ for (int mask = 16; mask > 0; mask >>= 1) {
35
+ amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, mask, 32));
36
+ sum += __shfl_xor_sync(0xffffffff, sum, mask, 32);
37
+ }
38
+
39
+ const float d = amax / 127;
40
+ const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
41
+
42
+ y[ib].qs[iqs] = q;
43
+
44
+ if (iqs > 0) {
45
+ return;
46
+ }
47
+
48
+ y[ib].ds.x = __float2half(d);
49
+ y[ib].ds.y = __float2half(sum);
50
+}
51
+
52
+static void quantize_row_q8_1_cuda(const half* x, void* vy, const int kx,
53
+ const int ky, cudaStream_t stream) {
54
+ const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
55
+ const int block_num_x =
56
+ (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
57
+ const dim3 num_blocks(block_num_x, ky, 1);
58
+ const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
59
+ quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);
60
+}
61
+
62
+torch::Tensor ggml_dequantize(torch::Tensor W, // quant weight
63
+ int8_t type, int64_t m, int64_t n) {
64
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(W));
65
+ auto options =
66
+ torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
67
+ at::Tensor DW = torch::empty({m, n}, options);
68
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
69
+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(type);
70
+ to_fp16_cuda((void*)W.data_ptr(), (half*)DW.data_ptr(), m * n, stream);
71
+ return DW;
72
+}
73
+
74
+torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, // quant weight
75
+ torch::Tensor X, // input
76
+ int8_t type, int64_t row) {
77
+ int col = X.sizes()[1];
78
+ const int padded = (col + 512 - 1) / 512 * 512;
79
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
80
+ auto options =
81
+ torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
82
+ at::Tensor Y = torch::empty({1, row}, options);
83
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
84
+ options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
85
+ at::Tensor quant_X = torch::empty({1, padded / 32 * 9}, options);
86
+ quantize_row_q8_1_cuda((half*)X.data_ptr(), (void*)quant_X.data_ptr(), col, 1,
87
+ stream);
88
+ switch (type) {
89
+ case 2:
90
+ mul_mat_vec_q4_0_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
91
+ (half*)Y.data_ptr(), col, row, stream);
92
+ break;
93
+ case 3:
94
+ mul_mat_vec_q4_1_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
95
+ (half*)Y.data_ptr(), col, row, stream);
96
+ break;
97
+ case 6:
98
+ mul_mat_vec_q5_0_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
99
+ (half*)Y.data_ptr(), col, row, stream);
100
+ break;
101
+ case 7:
102
+ mul_mat_vec_q5_1_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
103
+ (half*)Y.data_ptr(), col, row, stream);
104
+ break;
105
+ case 8:
106
+ mul_mat_vec_q8_0_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
107
+ (half*)Y.data_ptr(), col, row, stream);
108
+ break;
109
+ case 10:
110
+ mul_mat_vec_q2_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
111
+ (half*)Y.data_ptr(), col, row, stream);
112
+ break;
113
+ case 11:
114
+ mul_mat_vec_q3_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
115
+ (half*)Y.data_ptr(), col, row, stream);
116
+ break;
117
+ case 12:
118
+ mul_mat_vec_q4_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
119
+ (half*)Y.data_ptr(), col, row, stream);
120
+ break;
121
+ case 13:
122
+ mul_mat_vec_q5_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
123
+ (half*)Y.data_ptr(), col, row, stream);
124
+ break;
125
+ case 14:
126
+ mul_mat_vec_q6_K_q8_1_cuda((void*)W.data_ptr(), (void*)quant_X.data_ptr(),
127
+ (half*)Y.data_ptr(), col, row, stream);
128
+ break;
129
+ case 16:
130
+ mul_mat_vec_iq2_xxs_q8_1_cuda((void*)W.data_ptr(),
131
+ (void*)quant_X.data_ptr(),
132
+ (half*)Y.data_ptr(), col, row, stream);
133
+ break;
134
+ case 17:
135
+ mul_mat_vec_iq2_xs_q8_1_cuda((void*)W.data_ptr(),
136
+ (void*)quant_X.data_ptr(),
137
+ (half*)Y.data_ptr(), col, row, stream);
138
+ break;
139
+ case 18:
140
+ mul_mat_vec_iq3_xxs_q8_1_cuda((void*)W.data_ptr(),
141
+ (void*)quant_X.data_ptr(),
142
+ (half*)Y.data_ptr(), col, row, stream);
143
+ break;
144
+ case 19:
145
+ mul_mat_vec_iq1_s_q8_1_cuda((void*)W.data_ptr(),
146
+ (void*)quant_X.data_ptr(),
147
+ (half*)Y.data_ptr(), col, row, stream);
148
+ break;
149
+ case 20:
150
+ mul_mat_vec_iq4_nl_q8_1_cuda((void*)W.data_ptr(),
151
+ (void*)quant_X.data_ptr(),
152
+ (half*)Y.data_ptr(), col, row, stream);
153
+ break;
154
+ case 21:
155
+ mul_mat_vec_iq3_s_q8_1_cuda((void*)W.data_ptr(),
156
+ (void*)quant_X.data_ptr(),
157
+ (half*)Y.data_ptr(), col, row, stream);
158
+ break;
159
+ case 22:
160
+ mul_mat_vec_iq2_s_q8_1_cuda((void*)W.data_ptr(),
161
+ (void*)quant_X.data_ptr(),
162
+ (half*)Y.data_ptr(), col, row, stream);
163
+ break;
164
+ case 23:
165
+ mul_mat_vec_iq4_xs_q8_1_cuda((void*)W.data_ptr(),
166
+ (void*)quant_X.data_ptr(),
167
+ (half*)Y.data_ptr(), col, row, stream);
168
+ break;
169
+ }
170
+ return Y;
171
+}
172
+
173
+torch::Tensor ggml_mul_mat_a8(torch::Tensor W, // quant weight
174
+ torch::Tensor X, // input
175
+ int8_t type, int64_t row) {
176
+ int col = X.sizes()[1];
177
+ int padded = (col + 512 - 1) / 512 * 512;
178
+ int batch = X.sizes()[0];
179
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
180
+ auto options =
181
+ torch::TensorOptions().dtype(torch::kFloat16).device(W.device());
182
+ at::Tensor Y = torch::empty({batch, row}, options);
183
+ cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
184
+ options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
185
+ at::Tensor quant_X = torch::empty({batch, padded / 32 * 9}, options);
186
+ quantize_row_q8_1_cuda((half*)X.data_ptr(), (void*)quant_X.data_ptr(), col,
187
+ batch, stream);
188
+
189
+ switch (type) {
190
+ case 2:
191
+ ggml_mul_mat_q4_0_q8_1_cuda(
192
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
193
+ col, row, batch, padded, row, stream);
194
+ break;
195
+ case 3:
196
+ ggml_mul_mat_q4_1_q8_1_cuda(
197
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
198
+ col, row, batch, padded, row, stream);
199
+ break;
200
+ case 6:
201
+ ggml_mul_mat_q5_0_q8_1_cuda(
202
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
203
+ col, row, batch, padded, row, stream);
204
+ break;
205
+ case 7:
206
+ ggml_mul_mat_q5_1_q8_1_cuda(
207
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
208
+ col, row, batch, padded, row, stream);
209
+ break;
210
+ case 8:
211
+ ggml_mul_mat_q8_0_q8_1_cuda(
212
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
213
+ col, row, batch, padded, row, stream);
214
+ break;
215
+ case 10:
216
+ ggml_mul_mat_q2_K_q8_1_cuda(
217
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
218
+ col, row, batch, padded, row, stream);
219
+ break;
220
+ case 11:
221
+ ggml_mul_mat_q3_K_q8_1_cuda(
222
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
223
+ col, row, batch, padded, row, stream);
224
+ break;
225
+ case 12:
226
+ ggml_mul_mat_q4_K_q8_1_cuda(
227
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
228
+ col, row, batch, padded, row, stream);
229
+ break;
230
+ case 13:
231
+ ggml_mul_mat_q5_K_q8_1_cuda(
232
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
233
+ col, row, batch, padded, row, stream);
234
+ break;
235
+ case 14:
236
+ ggml_mul_mat_q6_K_q8_1_cuda(
237
+ (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (half*)Y.data_ptr(),
238
+ col, row, batch, padded, row, stream);
239
+ break;
240
+ }
241
+ return Y;
242
+}
\ No newline at end of file
csrc/quantization/gguf/mmq.cuh
new
+600
@@ -0,0 +1,600 @@
1
+// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
2
+template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,
3
+ allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>
4
+static __device__ __forceinline__ void mul_mat_q(
5
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
6
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
7
+
8
+ const block_q_t * x = (const block_q_t *) vx;
9
+ const block_q8_1 * y = (const block_q8_1 *) vy;
10
+
11
+ const int blocks_per_row_x = ncols_x / qk;
12
+ const int blocks_per_col_y = nrows_y / QK8_1;
13
+ const int blocks_per_warp = WARP_SIZE / qi;
14
+
15
+ const int & ncols_dst = ncols_y;
16
+
17
+ const int row_dst_0 = blockIdx.x*mmq_y;
18
+ const int & row_x_0 = row_dst_0;
19
+
20
+ const int col_dst_0 = blockIdx.y*mmq_x;
21
+ const int & col_y_0 = col_dst_0;
22
+
23
+ int * tile_x_ql = nullptr;
24
+ half2 * tile_x_dm = nullptr;
25
+ int * tile_x_qh = nullptr;
26
+ int * tile_x_sc = nullptr;
27
+
28
+ allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
29
+
30
+ __shared__ int tile_y_qs[mmq_x * WARP_SIZE];
31
+ __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];
32
+
33
+ float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};
34
+
35
+ for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
36
+
37
+ load_tiles(x + row_x_0*blocks_per_row_x + ib0, tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc,
38
+ threadIdx.y, nrows_x-row_x_0-1, threadIdx.x, blocks_per_row_x);
39
+
40
+#pragma unroll
41
+ for (int ir = 0; ir < qr; ++ir) {
42
+ const int kqs = ir*WARP_SIZE + threadIdx.x;
43
+ const int kbxd = kqs / QI8_1;
44
+
45
+#pragma unroll
46
+ for (int i = 0; i < mmq_x; i += nwarps) {
47
+ const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses
48
+ const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];
49
+ const int index_y = (threadIdx.y + i) * WARP_SIZE + kqs % WARP_SIZE;
50
+ tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
51
+ }
52
+
53
+#pragma unroll
54
+ for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
55
+ const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE/QI8_1)) % mmq_x;
56
+ const int kby = threadIdx.x % (WARP_SIZE/QI8_1);
57
+ const int col_y_eff = min(col_y_0 + ids, ncols_y-1);
58
+
59
+ // if the sum is not needed it's faster to transform the scale to f32 ahead of time
60
+ const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE/QI8_1) + kby].ds;
61
+ half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE/QI8_1) + kby];
62
+ if (need_sum) {
63
+ *dsi_dst = *dsi_src;
64
+ } else {
65
+ float * dfi_dst = (float *) dsi_dst;
66
+ *dfi_dst = __low2float(*dsi_src);
67
+ }
68
+ }
69
+
70
+ __syncthreads();
71
+
72
+// #pragma unroll // unrolling this loop causes too much register pressure
73
+ for (int k = ir*WARP_SIZE/qr; k < (ir+1)*WARP_SIZE/qr; k += vdr) {
74
+#pragma unroll
75
+ for (int j = 0; j < mmq_x; j += nwarps) {
76
+#pragma unroll
77
+ for (int i = 0; i < mmq_y; i += WARP_SIZE) {
78
+ sum[i/WARP_SIZE][j/nwarps] += vec_dot(
79
+ tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,
80
+ threadIdx.x + i, threadIdx.y + j, k);
81
+ }
82
+ }
83
+ }
84
+ __syncthreads();
85
+ }
86
+ }
87
+
88
+#pragma unroll
89
+ for (int j = 0; j < mmq_x; j += nwarps) {
90
+ const int col_dst = col_dst_0 + j + threadIdx.y;
91
+ if (col_dst >= ncols_dst) {
92
+ return;
93
+ }
94
+
95
+#pragma unroll
96
+ for (int i = 0; i < mmq_y; i += WARP_SIZE) {
97
+ const int row_dst = row_dst_0 + threadIdx.x + i;
98
+ if (row_dst >= nrows_dst) {
99
+ continue;
100
+ }
101
+ dst[col_dst*nrows_dst + row_dst] = __float2half(sum[i/WARP_SIZE][j/nwarps]);
102
+ }
103
+ }
104
+}
105
+
106
+#if defined(USE_ROCM)
107
+#define MMQ_X_Q4_0 64
108
+#define MMQ_Y_Q4_0 128
109
+#define NWARPS_Q4_0 8
110
+#else
111
+#define MMQ_X_Q4_0 4
112
+#define MMQ_Y_Q4_0 32
113
+#define NWARPS_Q4_0 4
114
+#endif
115
+
116
+template <bool need_check> static __global__ void
117
+#if defined(USE_ROCM)
118
+__launch_bounds__(WARP_SIZE*NWARPS_Q4_0, 2)
119
+#endif
120
+mul_mat_q4_0(
121
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
122
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
123
+ const int mmq_x = MMQ_X_Q4_0;
124
+ const int mmq_y = MMQ_Y_Q4_0;
125
+ const int nwarps = NWARPS_Q4_0;
126
+
127
+ mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,
128
+ load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>
129
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
130
+}
131
+
132
+static void ggml_mul_mat_q4_0_q8_1_cuda(
133
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
134
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
135
+
136
+ int mmq_x = MMQ_X_Q4_0;
137
+ int mmq_y = MMQ_Y_Q4_0;
138
+ int nwarps = NWARPS_Q4_0;
139
+
140
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
141
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
142
+ const dim3 block_nums(block_num_x, block_num_y, 1);
143
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
144
+
145
+ if (nrows_x % mmq_y == 0) {
146
+ const bool need_check = false;
147
+ mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>
148
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
149
+ } else {
150
+ const bool need_check = true;
151
+ mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>
152
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
153
+ }
154
+}
155
+
156
+#if defined(USE_ROCM)
157
+#define MMQ_X_Q4_1 64
158
+#define MMQ_Y_Q4_1 128
159
+#define NWARPS_Q4_1 8
160
+#else
161
+#define MMQ_X_Q4_1 4
162
+#define MMQ_Y_Q4_1 32
163
+#define NWARPS_Q4_1 4
164
+#endif
165
+
166
+template <bool need_check> static __global__ void
167
+#if defined(USE_ROCM)
168
+__launch_bounds__(WARP_SIZE*NWARPS_Q4_1, 2)
169
+#endif
170
+mul_mat_q4_1(
171
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
172
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
173
+ const int mmq_x = MMQ_X_Q4_1;
174
+ const int mmq_y = MMQ_Y_Q4_1;
175
+ const int nwarps = NWARPS_Q4_1;
176
+
177
+ mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,
178
+ load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>
179
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
180
+}
181
+
182
+static void ggml_mul_mat_q4_1_q8_1_cuda(
183
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
184
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
185
+
186
+ int mmq_x = MMQ_X_Q4_1;
187
+ int mmq_y = MMQ_Y_Q4_1;
188
+ int nwarps = NWARPS_Q4_1;
189
+
190
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
191
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
192
+ const dim3 block_nums(block_num_x, block_num_y, 1);
193
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
194
+
195
+ if (nrows_x % mmq_y == 0) {
196
+ const bool need_check = false;
197
+ mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>
198
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
199
+ } else {
200
+ const bool need_check = true;
201
+ mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>
202
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
203
+ }
204
+}
205
+
206
+#if defined(USE_ROCM)
207
+#define MMQ_X_Q5_0 64
208
+#define MMQ_Y_Q5_0 128
209
+#define NWARPS_Q5_0 8
210
+#else
211
+#define MMQ_X_Q5_0 4
212
+#define MMQ_Y_Q5_0 32
213
+#define NWARPS_Q5_0 4
214
+#endif
215
+
216
+template <bool need_check> static __global__ void
217
+#if defined(USE_ROCM)
218
+__launch_bounds__(WARP_SIZE*NWARPS_Q5_0, 2)
219
+#endif
220
+mul_mat_q5_0(
221
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
222
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
223
+ const int mmq_x = MMQ_X_Q5_0;
224
+ const int mmq_y = MMQ_Y_Q5_0;
225
+ const int nwarps = NWARPS_Q5_0;
226
+
227
+ mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,
228
+ load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>
229
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
230
+}
231
+
232
+static void ggml_mul_mat_q5_0_q8_1_cuda(
233
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
234
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
235
+
236
+ const int mmq_x = MMQ_X_Q5_0;
237
+ const int mmq_y = MMQ_Y_Q5_0;
238
+ const int nwarps = NWARPS_Q5_0;
239
+
240
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
241
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
242
+ const dim3 block_nums(block_num_x, block_num_y, 1);
243
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
244
+
245
+ if (nrows_x % mmq_y == 0) {
246
+ const bool need_check = false;
247
+ mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>
248
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
249
+ } else {
250
+ const bool need_check = true;
251
+ mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>
252
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
253
+ }
254
+}
255
+
256
+#if defined(USE_ROCM)
257
+#define MMQ_X_Q5_1 64
258
+#define MMQ_Y_Q5_1 128
259
+#define NWARPS_Q5_1 8
260
+#else
261
+#define MMQ_X_Q5_1 4
262
+#define MMQ_Y_Q5_1 32
263
+#define NWARPS_Q5_1 4
264
+#endif
265
+
266
+template <bool need_check> static __global__ void
267
+#if defined(USE_ROCM)
268
+__launch_bounds__(WARP_SIZE*NWARPS_Q5_1, 2)
269
+#endif
270
+mul_mat_q5_1(
271
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
272
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
273
+ const int mmq_x = MMQ_X_Q5_1;
274
+ const int mmq_y = MMQ_Y_Q5_1;
275
+ const int nwarps = NWARPS_Q5_1;
276
+
277
+ mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,
278
+ load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>
279
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
280
+}
281
+
282
+static void ggml_mul_mat_q5_1_q8_1_cuda(
283
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
284
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
285
+ const int mmq_x = MMQ_X_Q5_1;
286
+ const int mmq_y = MMQ_Y_Q5_1;
287
+ const int nwarps = NWARPS_Q5_1;
288
+
289
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
290
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
291
+ const dim3 block_nums(block_num_x, block_num_y, 1);
292
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
293
+
294
+ if (nrows_x % mmq_y == 0) {
295
+ const bool need_check = false;
296
+ mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>
297
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
298
+ } else {
299
+ const bool need_check = true;
300
+ mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>
301
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
302
+ }
303
+}
304
+
305
+#if defined(USE_ROCM)
306
+#define MMQ_X_Q8_0 64
307
+#define MMQ_Y_Q8_0 128
308
+#define NWARPS_Q8_0 8
309
+#else
310
+#define MMQ_X_Q8_0 4
311
+#define MMQ_Y_Q8_0 32
312
+#define NWARPS_Q8_0 4
313
+#endif
314
+
315
+template <bool need_check> static __global__ void
316
+#if defined(USE_ROCM)
317
+__launch_bounds__(WARP_SIZE*NWARPS_Q8_0, 2)
318
+#endif
319
+mul_mat_q8_0(
320
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
321
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
322
+ const int mmq_x = MMQ_X_Q8_0;
323
+ const int mmq_y = MMQ_Y_Q8_0;
324
+ const int nwarps = NWARPS_Q8_0;
325
+
326
+ mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,
327
+ load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>
328
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
329
+}
330
+
331
+static void ggml_mul_mat_q8_0_q8_1_cuda(
332
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
333
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
334
+ const int mmq_x = MMQ_X_Q8_0;
335
+ const int mmq_y = MMQ_Y_Q8_0;
336
+ const int nwarps = NWARPS_Q8_0;
337
+
338
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
339
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
340
+ const dim3 block_nums(block_num_x, block_num_y, 1);
341
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
342
+
343
+ if (nrows_x % mmq_y == 0) {
344
+ const bool need_check = false;
345
+ mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>
346
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
347
+ } else {
348
+ const bool need_check = true;
349
+ mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>
350
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
351
+ }
352
+}
353
+
354
+#if defined(USE_ROCM)
355
+#define MMQ_X_Q2_K 64
356
+#define MMQ_Y_Q2_K 128
357
+#define NWARPS_Q2_K 8
358
+#else
359
+#define MMQ_X_Q2_K 4
360
+#define MMQ_Y_Q2_K 32
361
+#define NWARPS_Q2_K 4
362
+#endif
363
+
364
+template <bool need_check> static __global__ void
365
+#if defined(USE_ROCM)
366
+__launch_bounds__(WARP_SIZE*NWARPS_Q2_K, 2)
367
+#endif
368
+mul_mat_q2_K(
369
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
370
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
371
+ const int mmq_x = MMQ_X_Q2_K;
372
+ const int mmq_y = MMQ_Y_Q2_K;
373
+ const int nwarps = NWARPS_Q2_K;
374
+
375
+ mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,
376
+ load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>
377
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
378
+}
379
+
380
+static void ggml_mul_mat_q2_K_q8_1_cuda(
381
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
382
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
383
+ const int mmq_x = MMQ_X_Q2_K;
384
+ const int mmq_y = MMQ_Y_Q2_K;
385
+ const int nwarps = NWARPS_Q2_K;
386
+
387
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
388
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
389
+ const dim3 block_nums(block_num_x, block_num_y, 1);
390
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
391
+
392
+ if (nrows_x % mmq_y == 0) {
393
+ const bool need_check = false;
394
+ mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>
395
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
396
+ } else {
397
+ const bool need_check = true;
398
+ mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>
399
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
400
+ }
401
+}
402
+
403
+#if defined(USE_ROCM)
404
+#define MMQ_X_Q3_K 64
405
+#define MMQ_Y_Q3_K 128
406
+#define NWARPS_Q3_K 8
407
+#else
408
+#define MMQ_X_Q3_K 4
409
+#define MMQ_Y_Q3_K 32
410
+#define NWARPS_Q3_K 4
411
+#endif
412
+
413
+template <bool need_check> static __global__ void
414
+#if defined(USE_ROCM)
415
+__launch_bounds__(WARP_SIZE*NWARPS_Q3_K, 2)
416
+#endif
417
+mul_mat_q3_K(
418
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
419
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
420
+
421
+ const int mmq_x = MMQ_X_Q3_K;
422
+ const int mmq_y = MMQ_Y_Q3_K;
423
+ const int nwarps = NWARPS_Q3_K;
424
+
425
+ mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,
426
+ load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>
427
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
428
+}
429
+
430
+static void ggml_mul_mat_q3_K_q8_1_cuda(
431
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
432
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
433
+
434
+ const int mmq_x = MMQ_X_Q3_K;
435
+ const int mmq_y = MMQ_Y_Q3_K;
436
+ const int nwarps = NWARPS_Q3_K;
437
+
438
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
439
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
440
+ const dim3 block_nums(block_num_x, block_num_y, 1);
441
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
442
+
443
+ if (nrows_x % mmq_y == 0) {
444
+ const bool need_check = false;
445
+ mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>
446
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
447
+ } else {
448
+ const bool need_check = true;
449
+ mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>
450
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
451
+ }
452
+}
453
+
454
+#if defined(USE_ROCM)
455
+#define MMQ_X_Q4_K 64
456
+#define MMQ_Y_Q4_K 128
457
+#define NWARPS_Q4_K 8
458
+#else
459
+#define MMQ_X_Q4_K 4
460
+#define MMQ_Y_Q4_K 32
461
+#define NWARPS_Q4_K 4
462
+#endif
463
+
464
+template <bool need_check> static __global__ void
465
+#if defined(USE_ROCM)
466
+__launch_bounds__(WARP_SIZE*NWARPS_Q4_K, 2)
467
+#endif
468
+mul_mat_q4_K(
469
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
470
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
471
+ const int mmq_x = MMQ_X_Q4_K;
472
+ const int mmq_y = MMQ_Y_Q4_K;
473
+ const int nwarps = NWARPS_Q4_K;
474
+
475
+ mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,
476
+ load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>
477
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
478
+}
479
+
480
+static void ggml_mul_mat_q4_K_q8_1_cuda(
481
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
482
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
483
+ const int mmq_x = MMQ_X_Q4_K;
484
+ const int mmq_y = MMQ_Y_Q4_K;
485
+ const int nwarps = NWARPS_Q4_K;
486
+
487
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
488
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
489
+ const dim3 block_nums(block_num_x, block_num_y, 1);
490
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
491
+
492
+ if (nrows_x % mmq_y == 0) {
493
+ const bool need_check = false;
494
+ mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>
495
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
496
+ } else {
497
+ const bool need_check = true;
498
+ mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>
499
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
500
+ }
501
+}
502
+
503
+#if defined(USE_ROCM)
504
+#define MMQ_X_Q5_K 64
505
+#define MMQ_Y_Q5_K 128
506
+#define NWARPS_Q5_K 8
507
+#else
508
+#define MMQ_X_Q5_K 4
509
+#define MMQ_Y_Q5_K 32
510
+#define NWARPS_Q5_K 4
511
+#endif
512
+
513
+template <bool need_check> static __global__ void
514
+#if defined(USE_ROCM)
515
+__launch_bounds__(WARP_SIZE*NWARPS_Q5_K, 2)
516
+#endif
517
+mul_mat_q5_K(
518
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
519
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
520
+ const int mmq_x = MMQ_X_Q5_K;
521
+ const int mmq_y = MMQ_Y_Q5_K;
522
+ const int nwarps = NWARPS_Q5_K;
523
+
524
+ mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,
525
+ load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>
526
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
527
+}
528
+
529
+static void ggml_mul_mat_q5_K_q8_1_cuda(
530
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
531
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
532
+
533
+ const int mmq_x = MMQ_X_Q5_K;
534
+ const int mmq_y = MMQ_Y_Q5_K;
535
+ const int nwarps = NWARPS_Q5_K;
536
+
537
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
538
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
539
+ const dim3 block_nums(block_num_x, block_num_y, 1);
540
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
541
+
542
+ if (nrows_x % mmq_y == 0) {
543
+ const bool need_check = false;
544
+ mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>
545
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
546
+ } else {
547
+ const bool need_check = true;
548
+ mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>
549
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
550
+ }
551
+}
552
+
553
+#if defined(USE_ROCM)
554
+#define MMQ_X_Q6_K 64
555
+#define MMQ_Y_Q6_K 128
556
+#define NWARPS_Q6_K 8
557
+#else
558
+#define MMQ_X_Q6_K 4
559
+#define MMQ_Y_Q6_K 32
560
+#define NWARPS_Q6_K 4
561
+#endif
562
+
563
+template <bool need_check> static __global__ void
564
+#if defined(USE_ROCM)
565
+__launch_bounds__(WARP_SIZE*NWARPS_Q6_K, 2)
566
+#endif
567
+mul_mat_q6_K(
568
+ const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
569
+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
570
+ const int mmq_x = MMQ_X_Q6_K;
571
+ const int mmq_y = MMQ_Y_Q6_K;
572
+ const int nwarps = NWARPS_Q6_K;
573
+
574
+ mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,
575
+ load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>
576
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
577
+}
578
+
579
+static void ggml_mul_mat_q6_K_q8_1_cuda(
580
+ const void * vx, const void * vy, half * dst, const int ncols_x, const int nrows_x,
581
+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
582
+ const int mmq_x = MMQ_X_Q6_K;
583
+ const int mmq_y = MMQ_Y_Q6_K;
584
+ const int nwarps = NWARPS_Q6_K;
585
+
586
+ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
587
+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
588
+ const dim3 block_nums(block_num_x, block_num_y, 1);
589
+ const dim3 block_dims(WARP_SIZE, nwarps, 1);
590
+
591
+ if (nrows_x % mmq_y == 0) {
592
+ const bool need_check = false;
593
+ mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>
594
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
595
+ } else {
596
+ const bool need_check = true;
597
+ mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>
598
+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
599
+ }
600
+}
csrc/quantization/gguf/mmvq.cuh
new
+182
@@ -0,0 +1,182 @@
1
+// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
2
+template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>
3
+static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst, const int ncols, const int nrows) {
4
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
5
+
6
+ if (row >= nrows) {
7
+ return;
8
+ }
9
+
10
+ const int blocks_per_row = ncols / qk;
11
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
12
+
13
+// partial sum for each thread
14
+ float tmp = 0.0f;
15
+
16
+ const block_q_t * x = (const block_q_t *) vx;
17
+ const block_q8_1 * y = (const block_q8_1 *) vy;
18
+
19
+ for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) {
20
+ const int ibx = row*blocks_per_row + i; // x block index
21
+
22
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
23
+
24
+ const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int
25
+
26
+ tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
27
+ }
28
+
29
+ // sum up partial sums and write back result
30
+#pragma unroll
31
+ for (int mask = 16; mask > 0; mask >>= 1) {
32
+ tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32);
33
+ }
34
+
35
+ if (threadIdx.x == 0) {
36
+ dst[row] = __float2half(tmp);
37
+ }
38
+}
39
+
40
+static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
41
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
42
+ const dim3 block_nums(block_num_y, 1, 1);
43
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
44
+ mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>
45
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
46
+}
47
+
48
+static void mul_mat_vec_q4_1_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
49
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
50
+ const dim3 block_nums(block_num_y, 1, 1);
51
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
52
+ mul_mat_vec_q<QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>
53
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
54
+}
55
+
56
+static void mul_mat_vec_q5_0_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
57
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
58
+ const dim3 block_nums(block_num_y, 1, 1);
59
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
60
+ mul_mat_vec_q<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>
61
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
62
+}
63
+
64
+static void mul_mat_vec_q5_1_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
65
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
66
+ const dim3 block_nums(block_num_y, 1, 1);
67
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
68
+ mul_mat_vec_q<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>
69
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
70
+}
71
+
72
+static void mul_mat_vec_q8_0_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
73
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
74
+ const dim3 block_nums(block_num_y, 1, 1);
75
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
76
+ mul_mat_vec_q<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>
77
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
78
+}
79
+
80
+static void mul_mat_vec_q2_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
81
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
82
+ const dim3 block_nums(block_num_y, 1, 1);
83
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
84
+ mul_mat_vec_q<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>
85
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
86
+}
87
+
88
+static void mul_mat_vec_q3_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
89
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
90
+ const dim3 block_nums(block_num_y, 1, 1);
91
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
92
+ mul_mat_vec_q<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>
93
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
94
+}
95
+
96
+static void mul_mat_vec_q4_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
97
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
98
+ const dim3 block_nums(block_num_y, 1, 1);
99
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
100
+ mul_mat_vec_q<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>
101
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
102
+}
103
+
104
+static void mul_mat_vec_q5_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
105
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
106
+ const dim3 block_nums(block_num_y, 1, 1);
107
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
108
+ mul_mat_vec_q<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>
109
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
110
+}
111
+
112
+static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
113
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
114
+ const dim3 block_nums(block_num_y, 1, 1);
115
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
116
+ mul_mat_vec_q<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>
117
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
118
+}
119
+
120
+static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
121
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
122
+ const dim3 block_nums(block_num_y, 1, 1);
123
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
124
+ mul_mat_vec_q<QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
125
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
126
+}
127
+
128
+static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
129
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
130
+ const dim3 block_nums(block_num_y, 1, 1);
131
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
132
+ mul_mat_vec_q<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
133
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
134
+}
135
+
136
+static void mul_mat_vec_iq2_s_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
137
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
138
+ const dim3 block_nums(block_num_y, 1, 1);
139
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
140
+ mul_mat_vec_q<QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
141
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
142
+}
143
+
144
+static void mul_mat_vec_iq3_xxs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
145
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
146
+ const dim3 block_nums(block_num_y, 1, 1);
147
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
148
+ mul_mat_vec_q<QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
149
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
150
+}
151
+
152
+static void mul_mat_vec_iq1_s_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
153
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
154
+ const dim3 block_nums(block_num_y, 1, 1);
155
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
156
+ mul_mat_vec_q<QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
157
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
158
+}
159
+
160
+static void mul_mat_vec_iq4_nl_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
161
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
162
+ const dim3 block_nums(block_num_y, 1, 1);
163
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
164
+ mul_mat_vec_q<QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>
165
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
166
+}
167
+
168
+static void mul_mat_vec_iq4_xs_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
169
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
170
+ const dim3 block_nums(block_num_y, 1, 1);
171
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
172
+ mul_mat_vec_q<QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
173
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
174
+}
175
+
176
+static void mul_mat_vec_iq3_s_q8_1_cuda(const void * vx, const void * vy, half * dst, const int ncols, const int nrows, cudaStream_t stream) {
177
+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
178
+ const dim3 block_nums(block_num_y, 1, 1);
179
+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
180
+ mul_mat_vec_q<QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
181
+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);
182
+}
csrc/quantization/gguf/vecdotq.cuh
new
+1745
@@ -0,0 +1,1745 @@
1
+// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh
2
+// and https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
3
+static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {
4
+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment
5
+ int x32 = 0;
6
+ x32 |= x16[0] << 0;
7
+ x32 |= x16[1] << 16;
8
+ return x32;
9
+}
10
+
11
+static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {
12
+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment
13
+ int x32 = 0;
14
+ x32 |= x16[0] << 0;
15
+ x32 |= x16[1] << 16;
16
+ return x32;
17
+}
18
+
19
+static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {
20
+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
21
+}
22
+
23
+static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {
24
+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
25
+}
26
+
27
+
28
+#define VDR_Q4_0_Q8_1_MMVQ 2
29
+#define VDR_Q4_0_Q8_1_MMQ 4
30
+
31
+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
32
+ const int * v, const int * u, const float & d4, const half2 & ds8) {
33
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
34
+ int sumi = 0;
35
+
36
+#pragma unroll
37
+ for (int i = 0; i < vdr; ++i) {
38
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
39
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
40
+
41
+ // SIMD dot product of quantized values
42
+ sumi = __dp4a(vi0, u[2*i+0], sumi);
43
+ sumi = __dp4a(vi1, u[2*i+1], sumi);
44
+ }
45
+
46
+ const float2 ds8f = __half22float2(ds8);
47
+
48
+ // second part effectively subtracts 8 from each quant value
49
+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
50
+#endif
51
+}
52
+
53
+#define VDR_Q4_1_Q8_1_MMVQ 2
54
+#define VDR_Q4_1_Q8_1_MMQ 4
55
+
56
+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
57
+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
58
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
59
+ int sumi = 0;
60
+
61
+#pragma unroll
62
+ for (int i = 0; i < vdr; ++i) {
63
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
64
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
65
+
66
+ // SIMD dot product of quantized values
67
+ sumi = __dp4a(vi0, u[2*i+0], sumi);
68
+ sumi = __dp4a(vi1, u[2*i+1], sumi);
69
+ }
70
+
71
+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));
72
+ const float d4d8 = tmp.x;
73
+ const float m4s8 = tmp.y;
74
+
75
+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
76
+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
77
+#endif
78
+}
79
+
80
+#define VDR_Q5_0_Q8_1_MMVQ 2
81
+#define VDR_Q5_0_Q8_1_MMQ 4
82
+
83
+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
84
+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
85
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
86
+ int sumi = 0;
87
+
88
+#pragma unroll
89
+ for (int i = 0; i < vdr; ++i) {
90
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
91
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
92
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
93
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
94
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
95
+ sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
96
+
97
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
98
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
99
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
100
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
101
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
102
+ sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
103
+ }
104
+
105
+ const float2 ds8f = __half22float2(ds8);
106
+
107
+ // second part effectively subtracts 16 from each quant value
108
+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
109
+#endif
110
+}
111
+
112
+
113
+#define VDR_Q5_1_Q8_1_MMVQ 2
114
+#define VDR_Q5_1_Q8_1_MMQ 4
115
+
116
+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
117
+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
118
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
119
+ int sumi = 0;
120
+
121
+#pragma unroll
122
+ for (int i = 0; i < vdr; ++i) {
123
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
124
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
125
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
126
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
127
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
128
+ sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
129
+
130
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
131
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
132
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
133
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
134
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
135
+ sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
136
+ }
137
+
138
+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));
139
+ const float d5d8 = tmp.x;
140
+ const float m5s8 = tmp.y;
141
+
142
+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
143
+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
144
+#endif
145
+}
146
+
147
+#define VDR_Q8_0_Q8_1_MMVQ 2
148
+#define VDR_Q8_0_Q8_1_MMQ 8
149
+
150
+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(
151
+ const int * v, const int * u, const float & d8_0, const float & d8_1) {
152
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
153
+ int sumi = 0;
154
+
155
+#pragma unroll
156
+ for (int i = 0; i < vdr; ++i) {
157
+ // SIMD dot product of quantized values
158
+ sumi = __dp4a(v[i], u[i], sumi);
159
+ }
160
+ return d8_0*d8_1 * sumi;
161
+#endif
162
+}
163
+
164
+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
165
+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
166
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
167
+
168
+ int sumi = 0;
169
+
170
+#pragma unroll
171
+ for (int i = 0; i < vdr; ++i) {
172
+ // SIMD dot product of quantized values
173
+ sumi = __dp4a(v[i], u[i], sumi);
174
+ }
175
+
176
+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));
177
+ const float d8d8 = tmp.x;
178
+ const float m8s8 = tmp.y;
179
+
180
+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
181
+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
182
+#endif
183
+}
184
+
185
+#define VDR_Q2_K_Q8_1_MMVQ 1
186
+#define VDR_Q2_K_Q8_1_MMQ 2
187
+
188
+// contiguous v/x values
189
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
190
+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
191
+ const half2 & dm2, const float * __restrict__ d8) {
192
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
193
+ float sumf_d = 0.0f;
194
+ float sumf_m = 0.0f;
195
+
196
+#pragma unroll
197
+ for (int i = 0; i < QR2_K; ++i) {
198
+ const int sc = scales[2*i];
199
+
200
+ const int vi = (v >> (2*i)) & 0x03030303;
201
+
202
+ sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
203
+
204
+ // fill int with 4x m
205
+ int m = sc >> 4;
206
+ m |= m << 8;
207
+ m |= m << 16;
208
+ sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
209
+ }
210
+
211
+ const float2 dm2f = __half22float2(dm2);
212
+
213
+ return dm2f.x*sumf_d - dm2f.y*sumf_m;
214
+#endif
215
+}
216
+
217
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
218
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
219
+ const half2 & dm2, const float & d8) {
220
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
221
+ int sumi_d = 0;
222
+ int sumi_m = 0;
223
+
224
+#pragma unroll
225
+ for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {
226
+ int sumi_d_sc = 0;
227
+
228
+ const int sc = scales[i0 / (QI8_1/2)];
229
+
230
+ // fill int with 4x m
231
+ int m = sc >> 4;
232
+ m |= m << 8;
233
+ m |= m << 16;
234
+
235
+#pragma unroll
236
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
237
+ sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product
238
+ sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m
239
+ }
240
+
241
+ sumi_d += sumi_d_sc * (sc & 0xF);
242
+ }
243
+
244
+ const float2 dm2f = __half22float2(dm2);
245
+
246
+ return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);
247
+#endif
248
+}
249
+
250
+#define VDR_Q3_K_Q8_1_MMVQ 1
251
+#define VDR_Q3_K_Q8_1_MMQ 2
252
+
253
+// contiguous v/x values
254
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
255
+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
256
+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {
257
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
258
+
259
+ float sumf = 0.0f;
260
+
261
+#pragma unroll
262
+ for (int i = 0; i < QR3_K; ++i) {
263
+ const int isc = scale_offset + 2*i;
264
+
265
+ const int isc_low = isc % (QK_K/32);
266
+ const int sc_shift_low = 4 * (isc / (QK_K/32));
267
+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
268
+
269
+ const int isc_high = isc % (QK_K/64);
270
+ const int sc_shift_high = 2 * (isc / (QK_K/64));
271
+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
272
+
273
+ const int sc = (sc_low | sc_high) - 32;
274
+
275
+ const int vil = (vl >> (2*i)) & 0x03030303;
276
+
277
+ const int vih = ((vh >> i) << 2) & 0x04040404;
278
+
279
+ const int vi = __vsubss4(vil, vih);
280
+
281
+ sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
282
+ }
283
+
284
+ return d3 * sumf;
285
+#endif
286
+}
287
+
288
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
289
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
290
+ const float & d3, const float & d8) {
291
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
292
+ int sumi = 0;
293
+
294
+#pragma unroll
295
+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
296
+ int sumi_sc = 0;
297
+
298
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
299
+ sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product
300
+ }
301
+
302
+ sumi += sumi_sc * scales[i0 / (QI8_1/2)];
303
+ }
304
+
305
+ return d3*d8 * sumi;
306
+#endif
307
+}
308
+
309
+#define VDR_Q4_K_Q8_1_MMVQ 2
310
+#define VDR_Q4_K_Q8_1_MMQ 8
311
+
312
+// contiguous v/x values
313
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
314
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
315
+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
316
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
317
+
318
+ float sumf_d = 0.0f;
319
+ float sumf_m = 0.0f;
320
+
321
+#pragma unroll
322
+ for (int i = 0; i < QR4_K; ++i) {
323
+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
324
+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
325
+
326
+ const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
327
+ const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u
328
+
329
+ sumf_d += d8[i] * (dot1 * sc[i]);
330
+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
331
+ }
332
+
333
+ const float2 dm4f = __half22float2(dm4);
334
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
335
+#endif
336
+}
337
+
338
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
339
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
340
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
341
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
342
+ float sumf_d = 0.0f;
343
+ float sumf_m = 0.0f;
344
+
345
+#pragma unroll
346
+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
347
+ int sumi_d = 0;
348
+
349
+#pragma unroll
350
+ for (int j = 0; j < QI8_1; ++j) {
351
+ sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
352
+ }
353
+
354
+ const float2 ds8f = __half22float2(ds8[i]);
355
+
356
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
357
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
358
+ }
359
+
360
+ const float2 dm4f = __half22float2(dm4);
361
+
362
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
363
+#endif
364
+}
365
+
366
+#define VDR_Q5_K_Q8_1_MMVQ 2
367
+#define VDR_Q5_K_Q8_1_MMQ 8
368
+
369
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
370
+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
371
+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
372
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
373
+
374
+ float sumf_d = 0.0f;
375
+ float sumf_m = 0.0f;
376
+
377
+#pragma unroll
378
+ for (int i = 0; i < QR5_K; ++i) {
379
+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
380
+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
381
+
382
+ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
383
+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
384
+
385
+ const int v0i = vl0i | vh0i;
386
+ const int v1i = vl1i | vh1i;
387
+
388
+ const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
389
+ const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u
390
+
391
+ sumf_d += d8[i] * (dot1 * sc[i]);
392
+ sumf_m += d8[i] * (dot2 * m[i]);
393
+ }
394
+
395
+ const float2 dm5f = __half22float2(dm5);
396
+ return dm5f.x*sumf_d - dm5f.y*sumf_m;
397
+#endif
398
+}
399
+
400
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
401
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
402
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
403
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
404
+ float sumf_d = 0.0f;
405
+ float sumf_m = 0.0f;
406
+
407
+#pragma unroll
408
+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
409
+ int sumi_d = 0;
410
+
411
+#pragma unroll
412
+ for (int j = 0; j < QI8_1; ++j) {
413
+ sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
414
+ }
415
+
416
+ const float2 ds8f = __half22float2(ds8[i]);
417
+
418
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
419
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
420
+ }
421
+
422
+ const float2 dm4f = __half22float2(dm4);
423
+
424
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
425
+#endif
426
+}
427
+
428
+#define VDR_Q6_K_Q8_1_MMVQ 1
429
+#define VDR_Q6_K_Q8_1_MMQ 8
430
+
431
+// contiguous v/x values
432
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
433
+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
434
+ const float & d, const float * __restrict__ d8) {
435
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
436
+ float sumf = 0.0f;
437
+
438
+#pragma unroll
439
+ for (int i = 0; i < QR6_K; ++i) {
440
+ const int sc = scales[4*i];
441
+ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
442
+ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
443
+ const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
444
+
445
+ sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
446
+ }
447
+
448
+ return d*sumf;
449
+#endif
450
+}
451
+
452
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
453
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
454
+ const float & d6, const float * __restrict__ d8) {
455
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
456
+ float sumf_d = 0.0f;
457
+
458
+#pragma unroll
459
+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
460
+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
461
+
462
+#pragma unroll
463
+ for (int i = i0; i < i0 + 2; ++i) {
464
+ sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
465
+ sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
466
+
467
+ sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
468
+ sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
469
+ }
470
+
471
+ sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);
472
+ }
473
+
474
+ return d6 * sumf_d;
475
+#endif
476
+}
477
+
478
+static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
479
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
480
+
481
+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;
482
+
483
+ int v[VDR_Q4_0_Q8_1_MMVQ];
484
+ int u[2*VDR_Q4_0_Q8_1_MMVQ];
485
+
486
+#pragma unroll
487
+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
488
+ v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);
489
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
490
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);
491
+ }
492
+
493
+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, __half2float(bq4_0->d), bq8_1->ds);
494
+}
495
+
496
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
497
+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
498
+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];
499
+ *x_ql = tile_x_qs;
500
+ *x_dm = (half2 *) tile_x_d;
501
+}
502
+
503
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(
504
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
505
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
506
+ const int kbx = k / QI4_0;
507
+ const int kqsx = k % QI4_0;
508
+
509
+ const block_q4_0 * bx0 = (const block_q4_0 *) vx;
510
+ float * x_dmf = (float *) x_dm;
511
+
512
+#pragma unroll
513
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
514
+ int i = i0 + i_offset;
515
+ if (need_check) {
516
+ i = min(i, i_max);
517
+ }
518
+ const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;
519
+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
520
+ // x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbx] = bxi->d;
521
+ }
522
+
523
+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;
524
+ const int kbxd = k % blocks_per_tile_x_row;
525
+
526
+#pragma unroll
527
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {
528
+ int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;
529
+ if (need_check) {
530
+ i = min(i, i_max);
531
+ }
532
+ const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;
533
+ x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
534
+ }
535
+}
536
+
537
+static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(
538
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
539
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
540
+ (void)x_qh; (void)x_sc;
541
+
542
+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
543
+ const float * x_dmf = (const float *) x_dm;
544
+
545
+ int u[2*VDR_Q4_0_Q8_1_MMQ];
546
+
547
+#pragma unroll
548
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
549
+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
550
+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_0) % WARP_SIZE];
551
+ }
552
+
553
+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
554
+ (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dmf[i * (WARP_SIZE/QI4_0) + i/QI4_0 + k/QI4_0],
555
+ y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
556
+}
557
+
558
+static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
559
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
560
+
561
+ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;
562
+
563
+ int v[VDR_Q4_1_Q8_1_MMVQ];
564
+ int u[2*VDR_Q4_1_Q8_1_MMVQ];
565
+
566
+#pragma unroll
567
+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
568
+ v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);
569
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
570
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);
571
+ }
572
+
573
+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
574
+}
575
+
576
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
577
+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];
578
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];
579
+ *x_ql = tile_x_qs;
580
+ *x_dm = tile_x_dm;
581
+}
582
+
583
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(
584
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
585
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
586
+ const int kbx = k / QI4_1;
587
+ const int kqsx = k % QI4_1;
588
+
589
+ const block_q4_1 * bx0 = (const block_q4_1 *) vx;
590
+
591
+#pragma unroll
592
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
593
+ int i = i0 + i_offset;
594
+ if (need_check) {
595
+ i = min(i, i_max);
596
+ }
597
+ const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;
598
+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
599
+ }
600
+
601
+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;
602
+ const int kbxd = k % blocks_per_tile_x_row;
603
+
604
+#pragma unroll
605
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {
606
+ int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;
607
+ if (need_check) {
608
+ i = min(i, i_max);
609
+ }
610
+ const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;
611
+ x_dm[i * (WARP_SIZE/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
612
+ }
613
+}
614
+
615
+static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(
616
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
617
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
618
+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
619
+
620
+ int u[2*VDR_Q4_1_Q8_1_MMQ];
621
+
622
+#pragma unroll
623
+ for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
624
+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
625
+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_1) % WARP_SIZE];
626
+ }
627
+
628
+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
629
+ (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dm[i * (WARP_SIZE/QI4_1) + i/QI4_1 + k/QI4_1],
630
+ y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
631
+}
632
+
633
+static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
634
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
635
+
636
+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;
637
+
638
+ int vl[VDR_Q5_0_Q8_1_MMVQ];
639
+ int vh[VDR_Q5_0_Q8_1_MMVQ];
640
+ int u[2*VDR_Q5_0_Q8_1_MMVQ];
641
+
642
+#pragma unroll
643
+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
644
+ vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);
645
+ vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));
646
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
647
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);
648
+ }
649
+
650
+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, __half2float(bq5_0->d), bq8_1->ds);
651
+}
652
+
653
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
654
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
655
+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];
656
+
657
+ *x_ql = tile_x_ql;
658
+ *x_dm = (half2 *) tile_x_d;
659
+}
660
+
661
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(
662
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
663
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
664
+ const int kbx = k / QI5_0;
665
+ const int kqsx = k % QI5_0;
666
+
667
+ const block_q5_0 * bx0 = (const block_q5_0 *) vx;
668
+
669
+#pragma unroll
670
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
671
+ int i = i0 + i_offset;
672
+
673
+ if (need_check) {
674
+ i = min(i, i_max);
675
+ }
676
+ const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbx;
677
+ const int ql = get_int_from_uint8(bxi->qs, kqsx);
678
+ const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));
679
+
680
+ int qs0 = (ql >> 0) & 0x0F0F0F0F;
681
+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
682
+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
683
+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
684
+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
685
+ qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
686
+
687
+ x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
688
+
689
+ int qs1 = (ql >> 4) & 0x0F0F0F0F;
690
+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
691
+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
692
+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
693
+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
694
+ qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
695
+
696
+ x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
697
+ }
698
+
699
+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;
700
+ const int kbxd = k % blocks_per_tile_x_row;
701
+ float * x_dmf = (float *) x_dm;
702
+
703
+#pragma unroll
704
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {
705
+ int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;
706
+
707
+ if (need_check) {
708
+ i = min(i, i_max);
709
+ }
710
+
711
+ const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;
712
+ x_dmf[i * (WARP_SIZE/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
713
+ }
714
+}
715
+
716
+static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
717
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
718
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
719
+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
720
+ const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;
721
+ const float * x_dmf = (const float *) x_dm;
722
+ const float * y_df = (const float *) y_ds;
723
+
724
+ int u[2*VDR_Q5_0_Q8_1_MMQ];
725
+
726
+#pragma unroll
727
+ for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {
728
+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
729
+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_0) % WARP_SIZE];
730
+ }
731
+
732
+ return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>
733
+ (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
734
+}
735
+
736
+static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
737
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
738
+
739
+ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;
740
+
741
+ int vl[VDR_Q5_1_Q8_1_MMVQ];
742
+ int vh[VDR_Q5_1_Q8_1_MMVQ];
743
+ int u[2*VDR_Q5_1_Q8_1_MMVQ];
744
+
745
+#pragma unroll
746
+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
747
+ vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);
748
+ vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));
749
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
750
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);
751
+ }
752
+
753
+ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
754
+}
755
+
756
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
757
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
758
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];
759
+
760
+ *x_ql = tile_x_ql;
761
+ *x_dm = tile_x_dm;
762
+}
763
+
764
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(
765
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
766
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
767
+ const int kbx = k / QI5_1;
768
+ const int kqsx = k % QI5_1;
769
+
770
+ const block_q5_1 * bx0 = (const block_q5_1 *) vx;
771
+
772
+#pragma unroll
773
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
774
+ int i = i0 + i_offset;
775
+
776
+ if (need_check) {
777
+ i = min(i, i_max);
778
+ }
779
+
780
+ const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbx;
781
+
782
+ const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
783
+ const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));
784
+
785
+ int qs0 = (ql >> 0) & 0x0F0F0F0F;
786
+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
787
+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
788
+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
789
+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
790
+
791
+ x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
792
+
793
+ int qs1 = (ql >> 4) & 0x0F0F0F0F;
794
+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
795
+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
796
+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
797
+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
798
+
799
+ x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
800
+ }
801
+
802
+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;
803
+ const int kbxd = k % blocks_per_tile_x_row;
804
+
805
+#pragma unroll
806
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {
807
+ int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;
808
+
809
+ if (need_check) {
810
+ i = min(i, i_max);
811
+ }
812
+
813
+ const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;
814
+
815
+ x_dm[i * (WARP_SIZE/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
816
+ }
817
+}
818
+
819
+static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(
820
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
821
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
822
+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
823
+ const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;
824
+
825
+ int u[2*VDR_Q5_1_Q8_1_MMQ];
826
+
827
+#pragma unroll
828
+ for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {
829
+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
830
+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_1) % WARP_SIZE];
831
+ }
832
+
833
+ return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
834
+ (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
835
+}
836
+
837
+static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
838
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
839
+
840
+ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;
841
+
842
+ int v[VDR_Q8_0_Q8_1_MMVQ];
843
+ int u[VDR_Q8_0_Q8_1_MMVQ];
844
+
845
+#pragma unroll
846
+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
847
+ v[i] = get_int_from_int8(bq8_0->qs, iqs + i);
848
+ u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
849
+ }
850
+
851
+ return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, __half2float(bq8_0->d), __low2float(bq8_1->ds));
852
+}
853
+
854
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
855
+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
856
+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];
857
+
858
+ *x_ql = tile_x_qs;
859
+ *x_dm = (half2 *) tile_x_d;
860
+}
861
+
862
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(
863
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
864
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
865
+ const int kbx = k / QI8_0;
866
+ const int kqsx = k % QI8_0;
867
+ float * x_dmf = (float *) x_dm;
868
+
869
+ const block_q8_0 * bx0 = (const block_q8_0 *) vx;
870
+
871
+#pragma unroll
872
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
873
+ int i = i0 + i_offset;
874
+
875
+ if (need_check) {
876
+ i = min(i, i_max);
877
+ }
878
+ const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;
879
+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
880
+ }
881
+
882
+ const int blocks_per_tile_x_row = WARP_SIZE / QI8_0;
883
+ const int kbxd = k % blocks_per_tile_x_row;
884
+
885
+#pragma unroll
886
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {
887
+ int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;
888
+
889
+ if (need_check) {
890
+ i = min(i, i_max);
891
+ }
892
+ const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;
893
+ x_dmf[i * (WARP_SIZE/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
894
+ }
895
+}
896
+
897
+static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(
898
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
899
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
900
+ const float * x_dmf = (const float *) x_dm;
901
+ const float * y_df = (const float *) y_ds;
902
+
903
+ return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>
904
+ (&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[j * WARP_SIZE + k], x_dmf[i * (WARP_SIZE/QI8_0) + i/QI8_0 + k/QI8_0],
905
+ y_df[j * (WARP_SIZE/QI8_1) + k/QI8_1]);
906
+}
907
+
908
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
909
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
910
+
911
+ const block_q2_K * bq2_K = (const block_q2_K *) vbq;
912
+
913
+ const int bq8_offset = QR2_K * (iqs / QI8_1);
914
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
915
+
916
+ const uint8_t * scales = bq2_K->scales + scale_offset;
917
+
918
+ const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);
919
+ int u[QR2_K];
920
+ float d8[QR2_K];
921
+
922
+#pragma unroll
923
+ for (int i = 0; i < QR2_K; ++ i) {
924
+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
925
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
926
+ }
927
+
928
+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
929
+}
930
+
931
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
932
+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
933
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];
934
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
935
+
936
+ *x_ql = tile_x_ql;
937
+ *x_dm = tile_x_dm;
938
+ *x_sc = tile_x_sc;
939
+}
940
+
941
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(
942
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
943
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
944
+ const int kbx = k / QI2_K;
945
+ const int kqsx = k % QI2_K;
946
+
947
+ const block_q2_K * bx0 = (const block_q2_K *) vx;
948
+
949
+#pragma unroll
950
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
951
+ int i = i0 + i_offset;
952
+
953
+ if (need_check) {
954
+ i = min(i, i_max);
955
+ }
956
+ const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;
957
+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
958
+ }
959
+
960
+ const int blocks_per_tile_x_row = WARP_SIZE / QI2_K;
961
+ const int kbxd = k % blocks_per_tile_x_row;
962
+
963
+#pragma unroll
964
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {
965
+ int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;
966
+
967
+ if (need_check) {
968
+ i = min(i, i_max);
969
+ }
970
+ const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;
971
+ x_dm[i * (WARP_SIZE/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
972
+ }
973
+
974
+#pragma unroll
975
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
976
+ int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
977
+
978
+ if (need_check) {
979
+ i = min(i, i_max);
980
+ }
981
+ const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI2_K/4);
982
+ x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
983
+ }
984
+}
985
+
986
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
987
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
988
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
989
+ const int kbx = k / QI2_K;
990
+ const int ky = (k % QI2_K) * QR2_K;
991
+ const float * y_df = (const float *) y_ds;
992
+
993
+ int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];
994
+
995
+ const int kqsx = i * (WARP_SIZE + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
996
+ const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));
997
+
998
+#pragma unroll
999
+ for (int l = 0; l < QR2_K*VDR_Q2_K_Q8_1_MMQ; ++l) {
1000
+ v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;
1001
+ }
1002
+
1003
+ const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE/4) + i/4 + kbx*4]) + ky/4;
1004
+
1005
+ const int index_y = j * WARP_SIZE + (QR2_K*k) % WARP_SIZE;
1006
+ return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1007
+}
1008
+
1009
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
1010
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1011
+
1012
+ const block_q3_K * bq3_K = (const block_q3_K *) vbq;
1013
+
1014
+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
1015
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
1016
+
1017
+ const float d = __half2float(bq3_K->d);
1018
+
1019
+ const int vl = get_int_from_uint8(bq3_K->qs, iqs);
1020
+
1021
+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1022
+ const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
1023
+
1024
+ int u[QR3_K];
1025
+ float d8[QR3_K];
1026
+
1027
+#pragma unroll
1028
+ for (int i = 0; i < QR3_K; ++i) {
1029
+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
1030
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
1031
+ }
1032
+
1033
+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
1034
+}
1035
+
1036
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1037
+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1038
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI3_K) + mmq_y/QI3_K];
1039
+ __shared__ int tile_x_qh[mmq_y * (WARP_SIZE/2) + mmq_y/2];
1040
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
1041
+
1042
+ *x_ql = tile_x_ql;
1043
+ *x_dm = tile_x_dm;
1044
+ *x_qh = tile_x_qh;
1045
+ *x_sc = tile_x_sc;
1046
+}
1047
+
1048
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(
1049
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1050
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1051
+ const int kbx = k / QI3_K;
1052
+ const int kqsx = k % QI3_K;
1053
+
1054
+ const block_q3_K * bx0 = (const block_q3_K *) vx;
1055
+
1056
+#pragma unroll
1057
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1058
+ int i = i0 + i_offset;
1059
+ if (need_check) {
1060
+ i = min(i, i_max);
1061
+ }
1062
+ const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;
1063
+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1064
+ }
1065
+
1066
+ const int blocks_per_tile_x_row = WARP_SIZE / QI3_K;
1067
+ const int kbxd = k % blocks_per_tile_x_row;
1068
+ float * x_dmf = (float *) x_dm;
1069
+
1070
+#pragma unroll
1071
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {
1072
+ int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;
1073
+ if (need_check) {
1074
+ i = min(i, i_max);
1075
+ }
1076
+ const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;
1077
+ x_dmf[i * (WARP_SIZE/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1078
+ }
1079
+
1080
+#pragma unroll
1081
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {
1082
+ int i = i0 + i_offset * 2 + k / (WARP_SIZE/2);
1083
+ if (need_check) {
1084
+ i = min(i, i_max);
1085
+ }
1086
+ const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/2)) / (QI3_K/2);
1087
+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1088
+ x_qh[i * (WARP_SIZE/2) + i / 2 + k % (WARP_SIZE/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1089
+ }
1090
+
1091
+#pragma unroll
1092
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
1093
+ int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
1094
+ if (need_check) {
1095
+ i = min(i, i_max);
1096
+ }
1097
+ const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI3_K/4);
1098
+
1099
+ const int ksc = k % (QI3_K/4);
1100
+
1101
+ const int ksc_low = ksc % (QI3_K/8);
1102
+ const int shift_low = 4 * (ksc / (QI3_K/8));
1103
+ const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;
1104
+
1105
+ const int ksc_high = QI3_K/8;
1106
+ const int shift_high = 2 * ksc;
1107
+ const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;
1108
+
1109
+ const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
1110
+
1111
+ x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = sc;
1112
+ }
1113
+}
1114
+
1115
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(
1116
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1117
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1118
+
1119
+ const int kbx = k / QI3_K;
1120
+ const int ky = (k % QI3_K) * QR3_K;
1121
+ const float * x_dmf = (const float *) x_dm;
1122
+ const float * y_df = (const float *) y_ds;
1123
+
1124
+ const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;
1125
+
1126
+ int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];
1127
+
1128
+#pragma unroll
1129
+ for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {
1130
+ const int kqsx = i * (WARP_SIZE + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1131
+ const int shift = 2 * ((ky % 32) / 8);
1132
+ const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;
1133
+
1134
+ const int vh = x_qh[i * (WARP_SIZE/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1135
+ const int vlh = (vh << 2) & 0x04040404;
1136
+
1137
+ v[l] = __vsubss4(vll, vlh);
1138
+ }
1139
+
1140
+ const int index_y = j * WARP_SIZE + (k*QR3_K) % WARP_SIZE;
1141
+ return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1142
+}
1143
+
1144
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
1145
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1146
+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;
1147
+
1148
+ int v[2];
1149
+ int u[2*QR4_K];
1150
+ float d8[QR4_K];
1151
+
1152
+ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
1153
+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
1154
+
1155
+ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
1156
+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
1157
+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
1158
+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
1159
+
1160
+ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1161
+ v[0] = q4[0];
1162
+ v[1] = q4[4];
1163
+
1164
+ const uint16_t * scales = (const uint16_t *)bq4_K->scales;
1165
+ uint16_t aux[2];
1166
+ const int j = bq8_offset/2;
1167
+ if (j < 2) {
1168
+ aux[0] = scales[j+0] & 0x3f3f;
1169
+ aux[1] = scales[j+2] & 0x3f3f;
1170
+ } else {
1171
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1172
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1173
+ }
1174
+ const uint8_t * sc = (const uint8_t *)aux;
1175
+ const uint8_t * m = sc + 2;
1176
+
1177
+ for (int i = 0; i < QR4_K; ++i) {
1178
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1179
+ d8[i] = __low2float(bq8i->ds);
1180
+
1181
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1182
+ u[2*i+0] = q8[0];
1183
+ u[2*i+1] = q8[4];
1184
+ }
1185
+
1186
+ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
1187
+}
1188
+
1189
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1190
+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1191
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];
1192
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1193
+
1194
+ *x_ql = tile_x_ql;
1195
+ *x_dm = tile_x_dm;
1196
+ *x_sc = tile_x_sc;
1197
+}
1198
+
1199
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(
1200
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1201
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1202
+ const int kbx = k / QI4_K; // == 0 if QK_K == 256
1203
+ const int kqsx = k % QI4_K; // == k if QK_K == 256
1204
+
1205
+ const block_q4_K * bx0 = (const block_q4_K *) vx;
1206
+
1207
+#pragma unroll
1208
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1209
+ int i = i0 + i_offset;
1210
+
1211
+ if (need_check) {
1212
+ i = min(i, i_max);
1213
+ }
1214
+ const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;
1215
+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1216
+ }
1217
+
1218
+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_K; // == 1 if QK_K == 256
1219
+ const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1220
+
1221
+#pragma unroll
1222
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {
1223
+ int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;
1224
+ if (need_check) {
1225
+ i = min(i, i_max);
1226
+ }
1227
+ const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;
1228
+ x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1229
+ }
1230
+
1231
+#pragma unroll
1232
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1233
+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1234
+
1235
+ if (need_check) {
1236
+ i = min(i, i_max);
1237
+ }
1238
+
1239
+ const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);
1240
+
1241
+ const int * scales = (const int *) bxi->scales;
1242
+
1243
+ const int ksc = k % (WARP_SIZE/8);
1244
+ // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1245
+ int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1246
+ scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1247
+
1248
+ x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1249
+ }
1250
+}
1251
+
1252
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(
1253
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1254
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1255
+ (void)x_qh;
1256
+
1257
+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1258
+
1259
+ const int index_y = j * WARP_SIZE + (QR4_K*k) % WARP_SIZE;
1260
+ return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[index_y], sc, sc+8,
1261
+ x_dm[i * (WARP_SIZE/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1262
+}
1263
+
1264
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
1265
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1266
+
1267
+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;
1268
+
1269
+ int vl[2];
1270
+ int vh[2];
1271
+ int u[2*QR5_K];
1272
+ float d8[QR5_K];
1273
+
1274
+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
1275
+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1276
+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
1277
+
1278
+ vl[0] = ql[0];
1279
+ vl[1] = ql[4];
1280
+
1281
+ vh[0] = qh[0] >> bq8_offset;
1282
+ vh[1] = qh[4] >> bq8_offset;
1283
+
1284
+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;
1285
+ uint16_t aux[2];
1286
+ const int j = bq8_offset/2;
1287
+ if (j < 2) {
1288
+ aux[0] = scales[j+0] & 0x3f3f;
1289
+ aux[1] = scales[j+2] & 0x3f3f;
1290
+ } else {
1291
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1292
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1293
+ }
1294
+ const uint8_t * sc = (const uint8_t *)aux;
1295
+ const uint8_t * m = sc + 2;
1296
+
1297
+#pragma unroll
1298
+ for (int i = 0; i < QR5_K; ++i) {
1299
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1300
+ d8[i] = __low2float(bq8i->ds);
1301
+
1302
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1303
+ u[2*i+0] = q8[0];
1304
+ u[2*i+1] = q8[4];
1305
+ }
1306
+
1307
+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
1308
+}
1309
+
1310
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1311
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1312
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];
1313
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1314
+
1315
+ *x_ql = tile_x_ql;
1316
+ *x_dm = tile_x_dm;
1317
+ *x_sc = tile_x_sc;
1318
+}
1319
+
1320
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(
1321
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1322
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1323
+ const int kbx = k / QI5_K; // == 0 if QK_K == 256
1324
+ const int kqsx = k % QI5_K; // == k if QK_K == 256
1325
+
1326
+ const block_q5_K * bx0 = (const block_q5_K *) vx;
1327
+
1328
+#pragma unroll
1329
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1330
+ int i = i0 + i_offset;
1331
+
1332
+ if (need_check) {
1333
+ i = min(i, i_max);
1334
+ }
1335
+
1336
+ const block_q5_K * bxi = bx0 + i*blocks_per_row + kbx;
1337
+ const int ky = QR5_K*kqsx;
1338
+
1339
+ const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
1340
+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;
1341
+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;
1342
+
1343
+ const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K/4));
1344
+ const int qh0 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 0)) << 4) & 0x10101010;
1345
+ const int qh1 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 1)) << 4) & 0x10101010;
1346
+
1347
+ const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;
1348
+ const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);
1349
+
1350
+ x_ql[i * (2*WARP_SIZE + 1) + kq0] = ql0 | qh0;
1351
+ x_ql[i * (2*WARP_SIZE + 1) + kq1] = ql1 | qh1;
1352
+ }
1353
+
1354
+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_K; // == 1 if QK_K == 256
1355
+ const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1356
+
1357
+#pragma unroll
1358
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {
1359
+ int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;
1360
+
1361
+ if (need_check) {
1362
+ i = min(i, i_max);
1363
+ }
1364
+
1365
+ const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;
1366
+ x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1367
+ }
1368
+
1369
+#pragma unroll
1370
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1371
+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1372
+
1373
+ if (need_check) {
1374
+ i = min(i, i_max);
1375
+ }
1376
+
1377
+ const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);
1378
+
1379
+ const int * scales = (const int *) bxi->scales;
1380
+
1381
+ const int ksc = k % (WARP_SIZE/8);
1382
+
1383
+ // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1384
+ int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1385
+ scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1386
+
1387
+ x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1388
+ }
1389
+}
1390
+
1391
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(
1392
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1393
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1394
+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1395
+
1396
+ const int index_x = i * (QR5_K*WARP_SIZE + 1) + QR5_K*k;
1397
+ const int index_y = j * WARP_SIZE + (QR5_K*k) % WARP_SIZE;
1398
+ return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,
1399
+ x_dm[i * (WARP_SIZE/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1400
+}
1401
+
1402
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
1403
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1404
+
1405
+ const block_q6_K * bq6_K = (const block_q6_K *) vbq;
1406
+
1407
+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
1408
+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
1409
+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
1410
+
1411
+ const int vl = get_int_from_uint8(bq6_K->ql, iqs);
1412
+ const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
1413
+
1414
+ const int8_t * scales = bq6_K->scales + scale_offset;
1415
+
1416
+ int u[QR6_K];
1417
+ float d8[QR6_K];
1418
+
1419
+#pragma unroll
1420
+ for (int i = 0; i < QR6_K; ++i) {
1421
+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
1422
+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
1423
+ }
1424
+
1425
+ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, __half2float(bq6_K->d), d8);
1426
+}
1427
+
1428
+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1429
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1430
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];
1431
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1432
+
1433
+ *x_ql = tile_x_ql;
1434
+ *x_dm = tile_x_dm;
1435
+ *x_sc = tile_x_sc;
1436
+}
1437
+
1438
+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(
1439
+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1440
+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1441
+ const int kbx = k / QI6_K; // == 0 if QK_K == 256
1442
+ const int kqsx = k % QI6_K; // == k if QK_K == 256
1443
+
1444
+ const block_q6_K * bx0 = (const block_q6_K *) vx;
1445
+
1446
+#pragma unroll
1447
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1448
+ int i = i0 + i_offset;
1449
+
1450
+ if (need_check) {
1451
+ i = min(i, i_max);
1452
+ }
1453
+
1454
+ const block_q6_K * bxi = bx0 + i*blocks_per_row + kbx;
1455
+ const int ky = QR6_K*kqsx;
1456
+
1457
+ const int ql = get_int_from_uint8(bxi->ql, kqsx);
1458
+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;
1459
+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;
1460
+
1461
+ const int qh = get_int_from_uint8(bxi->qh, (QI6_K/4) * (kqsx / (QI6_K/2)) + kqsx % (QI6_K/4));
1462
+ const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) << 4) & 0x30303030;
1463
+ const int qh1 = (qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) & 0x30303030;
1464
+
1465
+ const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;
1466
+ const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);
1467
+
1468
+ x_ql[i * (2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1469
+ x_ql[i * (2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1470
+ }
1471
+
1472
+ const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256
1473
+ const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1474
+ float * x_dmf = (float *) x_dm;
1475
+
1476
+#pragma unroll
1477
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {
1478
+ int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;
1479
+
1480
+ if (need_check) {
1481
+ i = min(i, i_max);
1482
+ }
1483
+
1484
+ const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;
1485
+
1486
+ x_dmf[i * (WARP_SIZE/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1487
+ }
1488
+
1489
+#pragma unroll
1490
+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1491
+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1492
+
1493
+ if (need_check) {
1494
+ i = min(i, i_max);
1495
+ }
1496
+
1497
+ const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / 4;
1498
+
1499
+ x_sc[i * (WARP_SIZE/8) + i / 8 + k % (WARP_SIZE/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1500
+ }
1501
+}
1502
+
1503
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(
1504
+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1505
+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1506
+ const float * x_dmf = (const float *) x_dm;
1507
+ const float * y_df = (const float *) y_ds;
1508
+
1509
+ const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/8]);
1510
+
1511
+ const int index_x = i * (QR6_K*WARP_SIZE + 1) + QR6_K*k;
1512
+ const int index_y = j * WARP_SIZE + (QR6_K*k) % WARP_SIZE;
1513
+ return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1514
+}
1515
+
1516
+static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
1517
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1518
+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;
1519
+
1520
+ const int ib32 = iqs;
1521
+ const uint16_t * q2 = bq2->qs + 4*ib32;
1522
+ const uint8_t * aux8 = (const uint8_t *)q2;
1523
+ const int8_t * q8 = bq8_1[ib32].qs;
1524
+ uint32_t aux32 = q2[2] | (q2[3] << 16);
1525
+ int sumi = 0;
1526
+ for (int l = 0; l < 4; ++l) {
1527
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
1528
+ const uint8_t signs = ksigns_iq2xs[aux32 & 127];
1529
+ for (int j = 0; j < 8; ++j) {
1530
+ sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1531
+ }
1532
+ q8 += 8;
1533
+ aux32 >>= 7;
1534
+ }
1535
+ const float d = __half2float(bq2->d) * (0.5f + aux32) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
1536
+ return d * sumi;
1537
+}
1538
+
1539
+static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
1540
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1541
+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;
1542
+
1543
+ const int ib32 = iqs;
1544
+ const uint16_t * q2 = bq2->qs + 4*ib32;
1545
+ const int8_t * q8 = bq8_1[ib32].qs;
1546
+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1547
+ const uint8_t ls2 = bq2->scales[ib32] >> 4;
1548
+ int sumi1 = 0;
1549
+ for (int l = 0; l < 2; ++l) {
1550
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1551
+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1552
+ for (int j = 0; j < 8; ++j) {
1553
+ sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1554
+ }
1555
+ q8 += 8;
1556
+ }
1557
+ int sumi2 = 0;
1558
+ for (int l = 2; l < 4; ++l) {
1559
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1560
+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1561
+ for (int j = 0; j < 8; ++j) {
1562
+ sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1563
+ }
1564
+ q8 += 8;
1565
+ }
1566
+ const float d = __half2float(bq2->d) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
1567
+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1568
+}
1569
+
1570
+static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1571
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1572
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1573
+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq;
1574
+
1575
+ const int ib32 = iqs;
1576
+ const int8_t * q8 = bq8_1[ib32].qs;
1577
+ const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;
1578
+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1579
+ const uint8_t ls2 = bq2->scales[ib32] >> 4;
1580
+ int sumi1 = 0;
1581
+ for (int l = 0; l < 2; ++l) {
1582
+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1583
+ const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1584
+ const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1585
+ const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
1586
+ const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
1587
+ sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);
1588
+ sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);
1589
+ q8 += 8;
1590
+ }
1591
+ int sumi2 = 0;
1592
+ for (int l = 2; l < 4; ++l) {
1593
+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1594
+ const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1595
+ const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1596
+ const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
1597
+ const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
1598
+ sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);
1599
+ sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);
1600
+ q8 += 8;
1601
+ }
1602
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[ib32].ds) * 0.25f;
1603
+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1604
+#endif
1605
+}
1606
+
1607
+static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1608
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1609
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1610
+ const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;
1611
+
1612
+ const int ib32 = iqs;
1613
+ const uint8_t * q3 = bq2->qs + 8*ib32;
1614
+ const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;
1615
+ const int8_t * q8 = bq8_1[ib32].qs;
1616
+ uint32_t aux32 = gas[0] | (gas[1] << 16);
1617
+ int sumi = 0;
1618
+ for (int l = 0; l < 4; ++l) {
1619
+ const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];
1620
+ const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];
1621
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));
1622
+ const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);
1623
+ const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);
1624
+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);
1625
+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);
1626
+ q8 += 8;
1627
+ aux32 >>= 7;
1628
+ }
1629
+ const float d = __half2float(bq2->d) * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;
1630
+ return d * sumi;
1631
+#endif
1632
+}
1633
+
1634
+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1635
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1636
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1637
+ const block_iq3_s * bq2 = (const block_iq3_s *) vbq;
1638
+
1639
+ const int ib32 = iqs;
1640
+ const uint8_t * qs = bq2->qs + 8*ib32;
1641
+ const int8_t * q8 = bq8_1[ib32].qs;
1642
+ int sumi = 0;
1643
+ for (int l = 0; l < 4; ++l) {
1644
+ const uint32_t * grid1 = iq3xs_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));
1645
+ const uint32_t * grid2 = iq3xs_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));
1646
+ uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1647
+ uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1648
+ const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);
1649
+ const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);
1650
+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);
1651
+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);
1652
+ q8 += 8;
1653
+ }
1654
+ const float d = __half2float(bq2->d) * (0.5f + ((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds) * 0.5f;
1655
+ return d * sumi;
1656
+#endif
1657
+}
1658
+
1659
+static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1660
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1661
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1662
+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1663
+
1664
+ const int ib32 = iqs;
1665
+ int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;
1666
+ const uint8_t h1 = bq1->scales[2*ib32+0];
1667
+ const uint8_t h2 = bq1->scales[2*ib32+1];
1668
+ const int * q8 = (const int *)bq8_1[ib32].qs;
1669
+ const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5)));
1670
+ const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1)));
1671
+ const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5)));
1672
+ const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1)));
1673
+ for (int j = 0; j < 2; ++j) {
1674
+ sumi1 = __dp4a(q8[j+0], grid1[j], sumi1);
1675
+ sumi2 = __dp4a(q8[j+2], grid2[j], sumi2);
1676
+ sumi3 = __dp4a(q8[j+4], grid3[j], sumi3);
1677
+ sumi4 = __dp4a(q8[j+6], grid4[j], sumi4);
1678
+ }
1679
+ const float d = __half2float(bq1->d) * __low2float(bq8_1[ib32].ds);
1680
+ return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) +
1681
+ sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1));
1682
+#endif
1683
+}
1684
+
1685
+static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,
1686
+ int & val1, int & val2) {
1687
+
1688
+ uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;
1689
+ aux32 = q4 & 0x0f0f0f0f;
1690
+ uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);
1691
+ uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);
1692
+ val1 = v1 | (v2 << 16);
1693
+ aux32 = (q4 >> 4) & 0x0f0f0f0f;
1694
+ v1 = values[q8[0]] | (values[q8[1]] << 8);
1695
+ v2 = values[q8[2]] | (values[q8[3]] << 8);
1696
+ val2 = v1 | (v2 << 16);
1697
+}
1698
+
1699
+static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1700
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1701
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1702
+
1703
+ const block_iq4_nl * bq = (const block_iq4_nl *) vbq;
1704
+
1705
+ const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;
1706
+ const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;
1707
+
1708
+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1709
+
1710
+ int v1, v2;
1711
+ int sumi1 = 0, sumi2 = 0;
1712
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
1713
+ const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);
1714
+ get_int_from_table_16(aux, values, v1, v2);
1715
+ sumi1 = __dp4a(v1, q8[l+0], sumi1);
1716
+ sumi2 = __dp4a(v2, q8[l+4], sumi2);
1717
+ }
1718
+ const float d = __half2float(bq->d) * __low2float(bq8_1->ds);
1719
+ return d * (sumi1 + sumi2);
1720
+#endif
1721
+}
1722
+
1723
+
1724
+static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
1725
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1726
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1727
+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;
1728
+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1729
+
1730
+ // iqs is 0...7
1731
+ const int ib32 = iqs;
1732
+ const int32_t * q8 = (const int *)bq8_1[ib32].qs;
1733
+ const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;
1734
+ const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);
1735
+ const float d = __half2float(bq4->d) * (ls - 32) * __low2float(bq8_1[ib32].ds);
1736
+ int v1, v2;
1737
+ int sumi1 = 0, sumi2 = 0;
1738
+ for (int j = 0; j < 4; ++j) {
1739
+ get_int_from_table_16(q4[j], values, v1, v2);
1740
+ sumi1 = __dp4a(v1, q8[j+0], sumi1);
1741
+ sumi2 = __dp4a(v2, q8[j+4], sumi2);
1742
+ }
1743
+ return d * (sumi1 + sumi2);
1744
+#endif
1745
+}
\ No newline at end of file
csrc/torch_bindings.cpp
+12
@@ -145,6 +145,18 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
145
ops.def("awq_marlin_repack", &awq_marlin_repack);
146
ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
147
148
+ // Dequantization for GGML.
149
+ ops.def("ggml_dequantize", &ggml_dequantize);
150
+ ops.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize);
151
+
152
+ // mmvq kernel for GGML.
153
+ ops.def("ggml_mul_mat_vec_a8", &ggml_mul_mat_vec_a8);
154
+ ops.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8);
155
+
156
+ // mmq kernel for GGML.
157
+ ops.def("ggml_mul_mat_a8", &ggml_mul_mat_a8);
158
+ ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
159
+
160
// fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
161
ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
162
ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
examples/gguf_inference.py
new
+38
@@ -0,0 +1,38 @@
1
+from huggingface_hub import hf_hub_download
2
+
3
+from vllm import LLM, SamplingParams
4
+
5
+
6
+def run_gguf_inference(model_path):
7
+ PROMPT_TEMPLATE = "<|system|>\n{system_message}</s>\n<|user|>\n{prompt}</s>\n<|assistant|>\n" # noqa: E501
8
+ system_message = "You are a friendly chatbot who always responds in the style of a pirate." # noqa: E501
9
+ # Sample prompts.
10
+ prompts = [
11
+ "How many helicopters can a human eat in one sitting?",
12
+ "What's the future of AI?",
13
+ ]
14
+ prompts = [
15
+ PROMPT_TEMPLATE.format(system_message=system_message, prompt=prompt)
16
+ for prompt in prompts
17
+ ]
18
+ # Create a sampling params object.
19
+ sampling_params = SamplingParams(temperature=0, max_tokens=128)
20
+
21
+ # Create an LLM.
22
+ llm = LLM(model=model_path,
23
+ tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
24
+ gpu_memory_utilization=0.95)
25
+
26
+ outputs = llm.generate(prompts, sampling_params)
27
+ # Print the outputs.
28
+ for output in outputs:
29
+ prompt = output.prompt
30
+ generated_text = output.outputs[0].text
31
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
32
+
33
+
34
+if __name__ == "__main__":
35
+ repo_id = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
36
+ filename = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
37
+ model = hf_hub_download(repo_id, filename=filename)
38
+ run_gguf_inference(model)
format.sh
+5
@@ -242,6 +242,11 @@ echo 'vLLM isort: Done'
242
# NOTE: Keep up to date with .github/workflows/clang-format.yml
243
CLANG_FORMAT_EXCLUDES=(
244
'csrc/moe/topk_softmax_kernels.cu'
245
+ 'csrc/quantization/gguf/ggml-common.h'
246
+ 'csrc/quantization/gguf/dequantize.cuh'
247
+ 'csrc/quantization/gguf/vecdotq.cuh'
248
+ 'csrc/quantization/gguf/mmq.cuh'
249
+ 'csrc/quantization/gguf/mmvq.cuh'
250
)
251
252
# Format specified files with clang-format
requirements-common.txt
+1
@@ -22,3 +22,4 @@ outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
22
typing_extensions
23
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
24
pyzmq
25
+gguf == 0.9.1
tests/models/test_gguf.py
new
+76
@@ -0,0 +1,76 @@
1
+"""
2
+Tests gguf models against unquantized models generations
3
+Note: To pass the test, quantization higher than Q4 should be used
4
+"""
5
+
6
+import os
7
+
8
+import pytest
9
+from huggingface_hub import hf_hub_download
10
+
11
+from tests.quantization.utils import is_quant_method_supported
12
+
13
+from .utils import check_logprobs_close
14
+
15
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
+
17
+MAX_MODEL_LEN = 1024
18
+
19
+# FIXME: Move this to confest
20
+MODELS = [
21
+ ("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
22
+ hf_hub_download("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
23
+ filename="tinyllama-1.1b-chat-v1.0.Q4_0.gguf")),
24
+ ("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
25
+ hf_hub_download("duyntnet/TinyLlama-1.1B-Chat-v1.0-imatrix-GGUF",
26
+ filename="TinyLlama-1.1B-Chat-v1.0-IQ4_XS.gguf")),
27
+ ("Qwen/Qwen2-1.5B-Instruct",
28
+ hf_hub_download("Qwen/Qwen2-1.5B-Instruct-GGUF",
29
+ filename="qwen2-1_5b-instruct-q4_k_m.gguf")),
30
+ ("Qwen/Qwen2-1.5B-Instruct",
31
+ hf_hub_download("legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
32
+ filename="Qwen2-1.5B-Instruct.IQ4_XS.gguf")),
33
+]
34
+
35
+
36
+@pytest.mark.skipif(not is_quant_method_supported("gguf"),
37
+ reason="gguf is not supported on this GPU type.")
38
+@pytest.mark.parametrize("model", MODELS)
39
+@pytest.mark.parametrize("dtype", ["half"])
40
+@pytest.mark.parametrize("max_tokens", [32])
41
+@pytest.mark.parametrize("num_logprobs", [5])
42
+def test_models(
43
+ vllm_runner,
44
+ example_prompts,
45
+ model,
46
+ dtype: str,
47
+ max_tokens: int,
48
+ num_logprobs: int,
49
+) -> None:
50
+ original_model, gguf_model = model
51
+
52
+ # Run unquantized model.
53
+ with vllm_runner(model_name=original_model,
54
+ dtype=dtype,
55
+ max_model_len=MAX_MODEL_LEN,
56
+ enforce_eager=True,
57
+ tensor_parallel_size=1) as original_model:
58
+
59
+ original_outputs = original_model.generate_greedy_logprobs(
60
+ example_prompts[:-1], max_tokens, num_logprobs)
61
+
62
+ # Run gguf model.
63
+ with vllm_runner(model_name=gguf_model,
64
+ dtype=dtype,
65
+ max_model_len=MAX_MODEL_LEN,
66
+ enforce_eager=True,
67
+ tensor_parallel_size=1) as gguf_model:
68
+ gguf_outputs = gguf_model.generate_greedy_logprobs(
69
+ example_prompts[:-1], max_tokens, num_logprobs)
70
+
71
+ check_logprobs_close(
72
+ outputs_0_lst=original_outputs,
73
+ outputs_1_lst=gguf_outputs,
74
+ name_0="original",
75
+ name_1="gguf",
76
+ )
tests/quantization/test_lm_head.py
+4
-2
@@ -7,11 +7,12 @@ from typing import Tuple
7
import pytest
8
import torch
9
10
-from vllm.model_executor.layers.linear import UnquantizedLinearMethod
10
from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
11
from vllm.model_executor.layers.quantization.gptq_marlin import (
12
GPTQMarlinLinearMethod)
13
from vllm.model_executor.layers.quantization.marlin import MarlinLinearMethod
14
+from vllm.model_executor.layers.vocab_parallel_embedding import (
15
+ UnquantizedEmbeddingMethod)
16
17
PROMPT = "On the surface of Mars, we found"
18
@@ -37,7 +38,8 @@ def test_lm_head(
38
lm_head_layer.linear_method,
39
(GPTQLinearMethod, GPTQMarlinLinearMethod, MarlinLinearMethod))
40
else:
40
- assert isinstance(lm_head_layer.linear_method, UnquantizedLinearMethod)
41
+ assert isinstance(lm_head_layer.linear_method,
42
+ UnquantizedEmbeddingMethod)
43
44
print(
45
vllm_model.generate_greedy(prompts=["Hello my name is"],
vllm/_custom_ops.py
+32
@@ -404,6 +404,38 @@ def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
404
workspace, size_m, size_n, size_k)
405
406
407
+# gguf
408
+def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int, n: int):
409
+ return torch.ops._C.ggml_dequantize(W, quant_type, m, n)
410
+
411
+
412
+def ggml_mul_mat_vec(
413
+ W: torch.Tensor,
414
+ X: torch.Tensor,
415
+ quant_type: int,
416
+ row: int,
417
+):
418
+ return torch.ops._C.ggml_mul_mat_vec(W, X, quant_type, row)
419
+
420
+
421
+def ggml_mul_mat_vec_a8(
422
+ W: torch.Tensor,
423
+ X: torch.Tensor,
424
+ quant_type: int,
425
+ row: int,
426
+):
427
+ return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
428
+
429
+
430
+def ggml_mul_mat_a8(
431
+ W: torch.Tensor,
432
+ X: torch.Tensor,
433
+ quant_type: int,
434
+ row: int,
435
+):
436
+ return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
437
+
438
+
439
# moe
440
def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
441
block_size: int, sorted_token_ids: torch.Tensor,
vllm/config.py
+1
@@ -582,6 +582,7 @@ class LoadFormat(str, enum.Enum):
582
DUMMY = "dummy"
583
TENSORIZER = "tensorizer"
584
SHARDED_STATE = "sharded_state"
585
+ GGUF = "gguf"
586
BITSANDBYTES = "bitsandbytes"
587
588
vllm/engine/arg_utils.py
+3
@@ -672,6 +672,9 @@ class EngineArgs:
672
return engine_args
673
674
def create_engine_config(self, ) -> EngineConfig:
675
+ # gguf file needs a specific model loader and doesn't use hf_repo
676
+ if self.model.endswith(".gguf"):
677
+ self.quantization = self.load_format = "gguf"
678
679
# bitsandbytes quantization needs a specific model loader
680
# so we make sure the quant method and the load format are consistent
vllm/model_executor/layers/linear.py
+81
-1
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
import torch.nn.functional as F
6
-from torch.nn.parameter import Parameter
6
+from torch.nn.parameter import Parameter, UninitializedParameter
7
8
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9
get_tensor_model_parallel_world_size,
@@ -311,6 +311,17 @@ class ColumnParallelLinear(LinearBase):
311
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
312
tp_rank = get_tensor_model_parallel_rank()
313
output_dim = getattr(param, "output_dim", None)
314
+
315
+ # Special case for GGUF
316
+ is_gguf_weight = getattr(param, "is_gguf_weight", False)
317
+ is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
318
+ if is_gguf_weight_type:
319
+ param.weight_type = loaded_weight.item()
320
+
321
+ # Materialize GGUF UninitializedParameter
322
+ if is_gguf_weight and isinstance(param, UninitializedParameter):
323
+ param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
324
+
325
param_data = param.data
326
if output_dim is not None:
327
shard_size = param_data.shape[output_dim]
@@ -398,6 +409,27 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
409
loaded_weight: torch.Tensor,
410
loaded_shard_id: Optional[int] = None):
411
412
+ # Special case for GGUF
413
+ # initialize GGUF param after we know the quantize type
414
+ is_gguf_weight = getattr(param, "is_gguf_weight", False)
415
+ is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
416
+ if is_gguf_weight_type:
417
+ param.data[loaded_shard_id].copy_(loaded_weight)
418
+ param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
419
+ return
420
+
421
+ if is_gguf_weight and isinstance(param, UninitializedParameter):
422
+ from gguf.constants import GGML_QUANT_SIZES
423
+
424
+ ori_shape = param.tensor_shape
425
+ weight_types = self.qweight_type.shard_weight_type.values()
426
+ row_size = []
427
+ for weight_type in weight_types:
428
+ block_size, type_size = GGML_QUANT_SIZES[weight_type]
429
+ row_size.append(ori_shape[1] // block_size * type_size)
430
+ q_shape = (ori_shape[0], max(row_size))
431
+ param.materialize(q_shape, dtype=loaded_weight.dtype)
432
+
433
param_data = param.data
434
output_dim = getattr(param, "output_dim", None)
435
# Special case for AQLM codebooks.
@@ -460,6 +492,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
492
shard_offset = loaded_weight.shape[output_dim] * \
493
loaded_shard_id
494
495
+ if is_gguf_weight:
496
+ shard_size = loaded_weight.shape[output_dim]
497
+ shard_offset = loaded_weight.shape[output_dim] * \
498
+ loaded_shard_id
499
+ param.shard_id.append(loaded_shard_id)
500
+ param.shard_size[loaded_shard_id] = loaded_weight.shape
501
+
502
param_data = param_data.narrow(output_dim, shard_offset,
503
shard_size)
504
start_idx = tp_rank * shard_size
@@ -563,6 +602,29 @@ class QKVParallelLinear(ColumnParallelLinear):
602
param: Parameter,
603
loaded_weight: torch.Tensor,
604
loaded_shard_id: Optional[str] = None):
605
+
606
+ # Special case for GGUF
607
+ # initialize GGUF param after we know the quantize type
608
+ is_gguf_weight = getattr(param, "is_gguf_weight", False)
609
+ is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
610
+ if is_gguf_weight_type and loaded_shard_id is not None:
611
+ idx_map = {"q": 0, "k": 1, "v": 2}
612
+ param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
613
+ param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
614
+ return
615
+
616
+ if is_gguf_weight and isinstance(param, UninitializedParameter):
617
+ from gguf.constants import GGML_QUANT_SIZES
618
+
619
+ ori_shape = param.tensor_shape
620
+ weight_types = self.qweight_type.shard_weight_type.values()
621
+ row_size = []
622
+ for weight_type in weight_types:
623
+ block_size, type_size = GGML_QUANT_SIZES[weight_type]
624
+ row_size.append(ori_shape[1] // block_size * type_size)
625
+ q_shape = (ori_shape[0], max(row_size))
626
+ param.materialize(q_shape, dtype=loaded_weight.dtype)
627
+
628
param_data = param.data
629
output_dim = getattr(param, "output_dim", None)
630
# Special case for AQLM codebooks.
@@ -650,6 +712,13 @@ class QKVParallelLinear(ColumnParallelLinear):
712
shard_size, shard_offset = adjust_bitsandbytes_shard(
713
param, orig_qkv_offsets, loaded_shard_id)
714
715
+ if is_gguf_weight:
716
+ param.shard_id.append(loaded_shard_id)
717
+ param.shard_size[loaded_shard_id] = loaded_weight.shape
718
+ input_dim = getattr(param, "input_dim", None)
719
+ input_size = loaded_weight.shape[input_dim]
720
+ param_data = param_data.narrow(input_dim, 0, input_size)
721
+
722
param_data = param_data.narrow(output_dim, shard_offset,
723
shard_size)
724
if loaded_shard_id == "q":
@@ -755,6 +824,17 @@ class RowParallelLinear(LinearBase):
824
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
825
tp_rank = get_tensor_model_parallel_rank()
826
input_dim = getattr(param, "input_dim", None)
827
+
828
+ # Special case for GGUF
829
+ is_gguf_weight = getattr(param, "is_gguf_weight", False)
830
+ is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
831
+ if is_gguf_weight_type:
832
+ param.weight_type = loaded_weight.item()
833
+
834
+ # Materialize GGUF UninitializedParameter
835
+ if is_gguf_weight and isinstance(param, UninitializedParameter):
836
+ param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
837
+
838
param_data = param.data
839
if input_dim is not None:
840
shard_size = param_data.shape[input_dim]
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.deepspeedfp import (
13
DeepSpeedFPConfig)
14
from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
15
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
16
+from vllm.model_executor.layers.quantization.gguf import GGUFConfig
17
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
18
from vllm.model_executor.layers.quantization.gptq_marlin import (
19
GPTQMarlinConfig)
@@ -31,6 +32,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
32
# The order of gptq methods is important for config.py iteration over
33
# override_quantization_method(..)
34
"marlin": MarlinConfig,
35
+ "gguf": GGUFConfig,
36
"gptq_marlin_24": GPTQMarlin24Config,
37
"gptq_marlin": GPTQMarlinConfig,
38
"awq_marlin": AWQMarlinConfig,
vllm/model_executor/layers/quantization/base_config.py
+25
-1
@@ -1,5 +1,6 @@
1
+import inspect
2
from abc import ABC, abstractmethod
2
-from typing import Any, Dict, List, Optional
3
+from typing import Any, Dict, List, Optional, Type
4
5
import torch
6
from torch import nn
@@ -23,6 +24,14 @@ class QuantizeMethodBase(ABC):
24
Expects create_weights to have been called before on the layer."""
25
raise NotImplementedError
26
27
+ # Not required functions
28
+ def embedding(self, layer: torch.nn.Module, *args,
29
+ **kwargs) -> torch.Tensor:
30
+ """Gather embeddings in the layer based on indices in the input tensor.
31
+
32
+ Expects create_weights to have been called before on the layer."""
33
+ raise NotImplementedError
34
+
35
def process_weights_after_loading(self, layer: nn.Module) -> None:
36
"""Process the weight after loading.
37
@@ -31,6 +40,21 @@ class QuantizeMethodBase(ABC):
40
return
41
42
43
+def method_has_implemented_embedding(
44
+ method_class: Type[QuantizeMethodBase]) -> bool:
45
+ """
46
+ Not all quant methods have embedding implemented, so we need to check that
47
+ it exists for our given method. We check this by making sure the function
48
+ has been changed from the base implementation.
49
+ """
50
+ base_embedding = inspect.getattr_static(QuantizeMethodBase, "embedding",
51
+ None)
52
+ class_embedding = inspect.getattr_static(method_class, "embedding", None)
53
+
54
+ return (class_embedding is not None
55
+ and class_embedding is not base_embedding)
56
+
57
+
58
class QuantizationConfig(ABC):
59
"""Base class for quantization configs."""
60
vllm/model_executor/layers/quantization/gguf.py
new
+165
@@ -0,0 +1,165 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import gguf
4
+import torch
5
+from torch.nn.parameter import Parameter, UninitializedParameter
6
+
7
+from vllm import _custom_ops as ops
8
+from vllm.distributed import get_tensor_model_parallel_world_size
9
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10
+from vllm.model_executor.layers.quantization.base_config import (
11
+ QuantizationConfig, QuantizeMethodBase)
12
+from vllm.model_executor.layers.vocab_parallel_embedding import (
13
+ VocabParallelEmbedding)
14
+from vllm.model_executor.utils import set_weight_attrs
15
+
16
+
17
+class GGUFConfig(QuantizationConfig):
18
+ """Config class for GGUF."""
19
+
20
+ def __init__(self, ) -> None:
21
+ pass
22
+
23
+ def __repr__(self) -> str:
24
+ return ("GGUFConfig()")
25
+
26
+ def get_name(self) -> str:
27
+ return "gguf"
28
+
29
+ def get_supported_act_dtypes(self) -> List[torch.dtype]:
30
+ return [torch.half, torch.bfloat16]
31
+
32
+ @classmethod
33
+ def get_min_capability(cls) -> int:
34
+ return 60
35
+
36
+ @classmethod
37
+ def get_config_filenames(cls) -> List[str]:
38
+ return [] # no extra configs.
39
+
40
+ @classmethod
41
+ def from_config(cls, config: Dict[str, Any]) -> "GGUFConfig":
42
+ if get_tensor_model_parallel_world_size() > 1:
43
+ raise ValueError(
44
+ "GGUF quantization hasn't supported tensor parallelism yet.")
45
+ return cls()
46
+
47
+ def get_quant_method(self, layer: torch.nn.Module,
48
+ prefix: str) -> Optional["QuantizeMethodBase"]:
49
+ if isinstance(layer, LinearBase):
50
+ return GGUFLinearMethod(self)
51
+ elif isinstance(layer, VocabParallelEmbedding):
52
+ return GGUFEmbeddingMethod(self)
53
+ return None
54
+
55
+ def get_scaled_act_names(self) -> List[str]:
56
+ return []
57
+
58
+
59
+def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
60
+ qweight_type: int) -> torch.Tensor:
61
+ # use dequantize mulmat for IQmatrix, mmq for k-quants
62
+ if qweight_type >= 16:
63
+ block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
64
+ shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
65
+ weight = ops.ggml_dequantize(qweight, qweight_type, *shape)
66
+ y = x @ weight.T
67
+ else:
68
+ y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
69
+ return y
70
+
71
+
72
+class GGUFLinearMethod(LinearMethodBase):
73
+ """Linear method for GGUF.
74
+
75
+ Args:
76
+ quant_config: The GGUF quantization config.
77
+ """
78
+
79
+ def __init__(self, quant_config: GGUFConfig):
80
+ self.quant_config = quant_config
81
+
82
+ def create_weights(self, layer: torch.nn.Module,
83
+ input_size_per_partition: int,
84
+ output_partition_sizes: List[int], input_size: int,
85
+ output_size: int, params_dtype: torch.dtype,
86
+ **extra_weight_attrs):
87
+ output_size_per_partition = sum(output_partition_sizes)
88
+
89
+ tensor_shape = (output_size_per_partition, input_size_per_partition)
90
+ qweight = UninitializedParameter(requires_grad=False)
91
+ set_weight_attrs(
92
+ qweight, {
93
+ "input_dim": 1,
94
+ "output_dim": 0,
95
+ "tensor_shape": tensor_shape,
96
+ "is_gguf_weight": True,
97
+ "shard_size": {},
98
+ "shard_id": [],
99
+ })
100
+ set_weight_attrs(qweight, extra_weight_attrs)
101
+ layer.register_parameter("qweight", qweight)
102
+
103
+ qweight_type = Parameter(torch.empty(len(output_partition_sizes),
104
+ dtype=torch.uint8),
105
+ requires_grad=False)
106
+ set_weight_attrs(
107
+ qweight_type, {
108
+ "is_gguf_weight_type": True,
109
+ "weight_type": 0,
110
+ "shard_weight_type": {},
111
+ "ignore_warning": True
112
+ })
113
+ set_weight_attrs(qweight_type, extra_weight_attrs)
114
+ layer.register_parameter("qweight_type", qweight_type)
115
+
116
+ def apply(self,
117
+ layer: torch.nn.Module,
118
+ x: torch.Tensor,
119
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
120
+ shard_size = getattr(layer.qweight, "shard_size", None)
121
+ shard_id = getattr(layer.qweight, "shard_id", None)
122
+
123
+ if shard_id and shard_size:
124
+ result = []
125
+ offset = 0
126
+ # dequantize shard weights respectively
127
+ shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id
128
+ for id in shard_id:
129
+ shard_weight = layer.qweight[
130
+ offset:offset +
131
+ shard_size[id][0], :shard_size[id][1]].contiguous()
132
+ qweight_type = layer.qweight_type.shard_weight_type[id]
133
+ result.append(_fuse_mul_mat(x, shard_weight, qweight_type))
134
+ offset += shard_size[id][0]
135
+ out = torch.cat(result, axis=1)
136
+ else:
137
+ qweight = layer.qweight
138
+ qweight_type = layer.qweight_type.weight_type
139
+ out = _fuse_mul_mat(x, qweight, qweight_type)
140
+ if bias is not None:
141
+ out.add_(bias)
142
+ return out
143
+
144
+
145
+class GGUFEmbeddingMethod(GGUFLinearMethod):
146
+ """Embedding method for GGUF.
147
+
148
+ Args:
149
+ quant_config: The GGUF quantization config.
150
+ """
151
+
152
+ def embedding(self, layer: torch.nn.Module,
153
+ x: torch.Tensor) -> torch.Tensor:
154
+ qweight = layer.qweight
155
+ qweight_type = layer.qweight_type.weight_type
156
+
157
+ block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
158
+ hidden_size = qweight.shape[1] // type_size * block_size
159
+ if qweight_type < 2:
160
+ return torch.embedding(qweight, x)
161
+ x_flat = x.flatten()
162
+ quant = torch.index_select(qweight, dim=0, index=x_flat)
163
+ dequant = ops.ggml_dequantize(quant, qweight_type, hidden_size,
164
+ x_flat.shape[0])
165
+ return dequant.view(*x.shape, hidden_size)
vllm/model_executor/layers/vocab_parallel_embedding.py
+54
-5
@@ -3,19 +3,46 @@ from typing import List, Optional, Sequence, Tuple
3
4
import torch
5
import torch.nn.functional as F
6
-from torch.nn.parameter import Parameter
6
+from torch.nn.parameter import Parameter, UninitializedParameter
7
8
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9
get_tensor_model_parallel_world_size,
10
tensor_model_parallel_all_reduce)
11
-from vllm.model_executor.layers.linear import UnquantizedLinearMethod
11
from vllm.model_executor.layers.quantization.base_config import (
13
- QuantizationConfig, QuantizeMethodBase)
12
+ QuantizationConfig, QuantizeMethodBase, method_has_implemented_embedding)
13
from vllm.model_executor.utils import set_weight_attrs
14
15
DEFAULT_VOCAB_PADDING_SIZE = 64
16
17
18
+class UnquantizedEmbeddingMethod(QuantizeMethodBase):
19
+ """Unquantized method for embeddings."""
20
+
21
+ def create_weights(self, layer: torch.nn.Module,
22
+ input_size_per_partition: int,
23
+ output_partition_sizes: List[int], input_size: int,
24
+ output_size: int, params_dtype: torch.dtype,
25
+ **extra_weight_attrs):
26
+ """Create weights for embedding layer."""
27
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
28
+ input_size_per_partition,
29
+ dtype=params_dtype),
30
+ requires_grad=False)
31
+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
32
+ layer.register_parameter("weight", weight)
33
+ set_weight_attrs(weight, extra_weight_attrs)
34
+
35
+ def apply(self,
36
+ layer: torch.nn.Module,
37
+ x: torch.Tensor,
38
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
39
+ return F.linear(x, layer.weight, bias)
40
+
41
+ def embedding(self, layer: torch.nn.Module,
42
+ input_: torch.Tensor) -> torch.Tensor:
43
+ return F.embedding(input_, layer.weight)
44
+
45
+
46
def pad_vocab_size(vocab_size: int,
47
pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
48
"""Pad the vocab size to the given value."""
@@ -199,7 +226,19 @@ class VocabParallelEmbedding(torch.nn.Module):
226
if quant_config is not None:
227
linear_method = quant_config.get_quant_method(self, prefix=prefix)
228
if linear_method is None:
202
- linear_method = UnquantizedLinearMethod()
229
+ linear_method = UnquantizedEmbeddingMethod()
230
+
231
+ # If we are making an embedding layer, then our quantization linear
232
+ # method must implement the embedding operation. If we are another
233
+ # layer type like ParallelLMHead, this is not important.
234
+ is_embedding_layer = type(self.__class__) is VocabParallelEmbedding
235
+ linear_method_implements_embedding = method_has_implemented_embedding(
236
+ type(linear_method))
237
+ if is_embedding_layer and not linear_method_implements_embedding:
238
+ raise NotImplementedError(
239
+ f"The class {type(linear_method).__name__} must implement "
240
+ "the 'embedding' method, see UnquantizedEmbeddingMethod.")
241
+
242
self.linear_method: QuantizeMethodBase = linear_method
243
244
if params_dtype is None:
@@ -306,6 +345,14 @@ class VocabParallelEmbedding(torch.nn.Module):
345
output_dim = getattr(param, "output_dim", None)
346
packed_dim = getattr(param, "packed_dim", None)
347
348
+ # If the parameter is a gguf weight, then load it directly.
349
+ if getattr(param, "is_gguf_weight_type", None):
350
+ param.data.copy_(loaded_weight)
351
+ param.weight_type = loaded_weight.item()
352
+ return
353
+ elif isinstance(param, UninitializedParameter):
354
+ param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
355
+
356
# If parameter does not have output dim, then it should
357
# be copied onto all gpus (e.g. g_idx for act_order gptq).
358
if output_dim is None:
@@ -344,7 +391,8 @@ class VocabParallelEmbedding(torch.nn.Module):
391
else:
392
masked_input = input_
393
# Get the embeddings.
347
- output_parallel = F.embedding(masked_input.long(), self.weight)
394
+ output_parallel = self.linear_method.embedding(self,
395
+ masked_input.long())
396
# Mask the output embedding.
397
if self.tp_size > 1:
398
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
@@ -389,6 +437,7 @@ class ParallelLMHead(VocabParallelEmbedding):
437
super().__init__(num_embeddings, embedding_dim, params_dtype,
438
org_num_embeddings, padding_size, quant_config,
439
prefix)
440
+
441
if bias:
442
self.bias = Parameter(
443
torch.empty(self.num_embeddings_per_partition,
vllm/model_executor/model_loader/loader.py
+92
-2
@@ -10,11 +10,13 @@ from abc import ABC, abstractmethod
10
from contextlib import contextmanager
11
from typing import Any, Dict, Generator, List, Optional, Tuple, Type
12
13
+import gguf
14
import huggingface_hub
15
import numpy as np
16
import torch
17
from huggingface_hub import HfApi, hf_hub_download
18
from torch import nn
19
+from transformers import AutoModelForCausalLM
20
21
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
22
LoRAConfig, ModelConfig, MultiModalConfig,
@@ -31,8 +33,9 @@ from vllm.model_executor.model_loader.utils import (get_model_architecture,
33
from vllm.model_executor.model_loader.weight_utils import (
34
download_safetensors_index_file_from_hf, download_weights_from_hf,
35
filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
34
- get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
35
- pt_weights_iterator, safetensors_weights_iterator)
36
+ get_gguf_extra_tensor_names, get_quant_config, gguf_quant_weights_iterator,
37
+ initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
38
+ safetensors_weights_iterator)
39
from vllm.model_executor.models.interfaces import (has_inner_state,
40
supports_lora,
41
supports_vision)
@@ -948,6 +951,90 @@ class BitsAndBytesModelLoader(BaseModelLoader):
951
return model.eval()
952
953
954
+class GGUFModelLoader(BaseModelLoader):
955
+ """
956
+ Model loader that can load GGUF files. This is useful for loading models
957
+ that are quantized with GGUF and saved in the GGUF format. This loader
958
+ supports loading both full models and sharded models.
959
+ """
960
+
961
+ def __init__(self, load_config: LoadConfig):
962
+ super().__init__(load_config)
963
+ if load_config.model_loader_extra_config:
964
+ raise ValueError(f"Model loader extra config is not supported for "
965
+ f"load format {load_config.load_format}")
966
+
967
+ def _prepare_weights(self, model_name_or_path: str):
968
+ if os.path.isfile(model_name_or_path):
969
+ return model_name_or_path
970
+ else:
971
+ raise ValueError(f"{model_name_or_path} is not a file.")
972
+
973
+ def _get_gguf_weights_map(self, model_config: ModelConfig):
974
+ """
975
+ GGUF uses this naming convention for their tensors from HF checkpoint:
976
+ `blk.N.BB.weight` and `blk.N.BB.bias`
977
+ where N signifies the block number of a layer, and BB signifies the
978
+ attention/mlp layer components.
979
+ See "Standardized tensor names" in
980
+ https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details.
981
+ """
982
+ config = model_config.hf_config
983
+ model_type = config.model_type
984
+ # hack: ggufs have a different name than transformers
985
+ if model_type == "cohere":
986
+ model_type = "command-r"
987
+ arch = None
988
+ for key, value in gguf.MODEL_ARCH_NAMES.items():
989
+ if value == model_type:
990
+ arch = key
991
+ break
992
+ if arch is None:
993
+ raise RuntimeError(f"Unknown gguf model_type: {model_type}")
994
+ num_layers = config.num_hidden_layers
995
+ name_map = gguf.get_tensor_name_map(arch, num_layers)
996
+ with torch.device("meta"):
997
+ dummy_model = AutoModelForCausalLM.from_config(config)
998
+ state_dict = dummy_model.state_dict()
999
+
1000
+ gguf_to_hf_name_map = {}
1001
+ for hf_name in state_dict:
1002
+ name, suffix = hf_name.rsplit(".", 1)
1003
+ gguf_name = name_map.get_name(name)
1004
+ gguf_to_hf_name_map[f"{gguf_name}.{suffix}"] = hf_name
1005
+ return gguf_to_hf_name_map
1006
+
1007
+ def _get_weights_iterator(
1008
+ self, model_name_or_path: str, gguf_to_hf_name_map: Dict[str, str]
1009
+ ) -> Generator[Tuple[str, torch.Tensor], None, None]:
1010
+ return gguf_quant_weights_iterator(model_name_or_path,
1011
+ gguf_to_hf_name_map)
1012
+
1013
+ def load_model(self, *, model_config: ModelConfig,
1014
+ device_config: DeviceConfig,
1015
+ lora_config: Optional[LoRAConfig],
1016
+ multimodal_config: Optional[MultiModalConfig],
1017
+ parallel_config: ParallelConfig,
1018
+ scheduler_config: SchedulerConfig,
1019
+ cache_config: CacheConfig) -> nn.Module:
1020
+
1021
+ local_model_path = self._prepare_weights(model_config.model)
1022
+ gguf_weights_map = self._get_gguf_weights_map(model_config)
1023
+ # we can only know if tie word embeddings after mapping weights
1024
+ if "lm_head.weight" in get_gguf_extra_tensor_names(
1025
+ local_model_path, gguf_weights_map):
1026
+ model_config.hf_config.update({"tie_word_embeddings": True})
1027
+
1028
+ with set_default_torch_dtype(model_config.dtype):
1029
+ with torch.device(device_config.device):
1030
+ model = _initialize_model(model_config, self.load_config,
1031
+ lora_config, multimodal_config,
1032
+ cache_config)
1033
+ model.load_weights(
1034
+ self._get_weights_iterator(local_model_path, gguf_weights_map))
1035
+ return model
1036
+
1037
+
1038
def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1039
"""Get a model loader based on the load format."""
1040
@@ -966,4 +1053,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1053
if load_config.load_format == LoadFormat.BITSANDBYTES:
1054
return BitsAndBytesModelLoader(load_config)
1055
1056
+ if load_config.load_format == LoadFormat.GGUF:
1057
+ return GGUFModelLoader(load_config)
1058
+
1059
return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/weight_utils.py
+46
-1
@@ -6,9 +6,10 @@ import json
6
import os
7
import tempfile
8
from collections import defaultdict
9
-from typing import Any, Generator, Iterable, List, Optional, Tuple, Union
9
+from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union
10
11
import filelock
12
+import gguf
13
import huggingface_hub.constants
14
import numpy as np
15
import torch
@@ -121,6 +122,11 @@ def get_quant_config(model_config: ModelConfig,
122
load_config: LoadConfig) -> QuantizationConfig:
123
124
quant_cls = get_quantization_config(model_config.quantization)
125
+
126
+ # GGUF doesn't have config file
127
+ if model_config.quantization == "gguf":
128
+ return quant_cls.from_config({})
129
+
130
# Read the quantization config from the HF model config, if available.
131
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
132
None)
@@ -409,6 +415,45 @@ def pt_weights_iterator(
415
torch.cuda.empty_cache()
416
417
418
+def get_gguf_extra_tensor_names(
419
+ gguf_file: str, gguf_to_hf_name_map: Dict[str, str]) -> List[str]:
420
+ reader = gguf.GGUFReader(gguf_file)
421
+ expected_gguf_keys = set(gguf_to_hf_name_map.keys())
422
+ exact_gguf_keys = set([tensor.name for tensor in reader.tensors])
423
+ extra_keys = expected_gguf_keys - exact_gguf_keys
424
+ return [gguf_to_hf_name_map[key] for key in extra_keys]
425
+
426
+
427
+def gguf_quant_weights_iterator(
428
+ gguf_file: str, gguf_to_hf_name_map: Dict[str, str]
429
+) -> Generator[Tuple[str, torch.Tensor], None, None]:
430
+ """
431
+ Iterate over the quant weights in the model gguf files and convert
432
+ them to torch tensors
433
+ """
434
+
435
+ reader = gguf.GGUFReader(gguf_file)
436
+
437
+ for tensor in reader.tensors:
438
+ weight_type = tensor.tensor_type
439
+ name = gguf_to_hf_name_map[tensor.name]
440
+
441
+ if weight_type.name != "F32":
442
+ weight_type_name = name.replace("weight", "qweight_type")
443
+ weight_type = torch.tensor(weight_type)
444
+ yield weight_type_name, weight_type
445
+
446
+ for tensor in reader.tensors:
447
+ weight = tensor.data
448
+ weight_type = tensor.tensor_type
449
+ name = gguf_to_hf_name_map[tensor.name]
450
+
451
+ if weight_type.name != "F32":
452
+ name = name.replace("weight", "qweight")
453
+ param = torch.tensor(weight)
454
+ yield name, param
455
+
456
+
457
def kv_cache_scales_loader(
458
filename: str, tp_rank: int, tp_size: int, num_hidden_layers: int,
459
model_type: Optional[str]) -> Iterable[Tuple[int, float]]:
vllm/model_executor/models/llama.py
+7
@@ -140,6 +140,7 @@ class LlamaAttention(nn.Module):
140
quant_config=quant_config,
141
prefix=f"{prefix}.qkv_proj",
142
)
143
+
144
self.o_proj = RowParallelLinear(
145
input_size=self.total_num_heads * self.head_dim,
146
output_size=hidden_size,
@@ -148,12 +149,17 @@ class LlamaAttention(nn.Module):
149
prefix=f"{prefix}.o_proj",
150
)
151
152
+ is_neox_style = True
153
+ if quant_config is not None and quant_config.get_name() == "gguf":
154
+ is_neox_style = False
155
+
156
self.rotary_emb = get_rope(
157
self.head_dim,
158
rotary_dim=self.head_dim,
159
max_position=max_position_embeddings,
160
base=rope_theta,
161
rope_scaling=rope_scaling,
162
+ is_neox_style=is_neox_style,
163
)
164
self.attn = Attention(self.num_heads,
165
self.head_dim,
@@ -279,6 +285,7 @@ class LlamaModel(nn.Module):
285
self.vocab_size,
286
config.hidden_size,
287
org_num_embeddings=config.vocab_size,
288
+ quant_config=quant_config,
289
)
290
else:
291
self.embed_tokens = PPMissingLayer()
vllm/model_executor/models/qwen2.py
+1
@@ -238,6 +238,7 @@ class Qwen2Model(nn.Module):
238
self.embed_tokens = VocabParallelEmbedding(
239
config.vocab_size,
240
config.hidden_size,
241
+ quant_config=quant_config,
242
)
243
self.start_layer, self.end_layer, self.layers = make_layers(
244
config.num_hidden_layers,
vllm/transformers_utils/config.py
+32
-8
@@ -1,7 +1,10 @@
1
import contextlib
2
-from typing import Dict, Optional, Type
2
+from pathlib import Path
3
+from typing import Dict, Optional, Type, Union
4
5
from transformers import GenerationConfig, PretrainedConfig
6
+from transformers.models.auto.modeling_auto import (
7
+ MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
8
9
from vllm.envs import VLLM_USE_MODELSCOPE
10
from vllm.logger import init_logger
@@ -36,18 +39,29 @@ for name, cls in _CONFIG_REGISTRY.items():
39
AutoConfig.register(name, cls)
40
41
39
-def get_config(model: str,
40
- trust_remote_code: bool,
41
- revision: Optional[str] = None,
42
- code_revision: Optional[str] = None,
43
- rope_scaling: Optional[dict] = None,
44
- rope_theta: Optional[float] = None) -> PretrainedConfig:
42
+def get_config(
43
+ model: Union[str, Path],
44
+ trust_remote_code: bool,
45
+ revision: Optional[str] = None,
46
+ code_revision: Optional[str] = None,
47
+ rope_scaling: Optional[dict] = None,
48
+ rope_theta: Optional[float] = None,
49
+ **kwargs,
50
+) -> PretrainedConfig:
51
+
52
+ # Separate model folder from file path for GGUF models
53
+ is_gguf = Path(model).is_file() and Path(model).suffix == ".gguf"
54
+ if is_gguf:
55
+ kwargs["gguf_file"] = Path(model).name
56
+ model = Path(model).parent
57
+
58
try:
59
config = AutoConfig.from_pretrained(
60
model,
61
trust_remote_code=trust_remote_code,
62
revision=revision,
50
- code_revision=code_revision)
63
+ code_revision=code_revision,
64
+ **kwargs)
65
except ValueError as e:
66
if (not trust_remote_code and
67
"requires you to execute the configuration file" in str(e)):
@@ -64,12 +78,22 @@ def get_config(model: str,
78
config = config_class.from_pretrained(model,
79
revision=revision,
80
code_revision=code_revision)
81
+
82
+ # Special architecture mapping check for GGUF models
83
+ if is_gguf:
84
+ if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
85
+ raise RuntimeError(
86
+ f"Can't get gguf config for {config.model_type}.")
87
+ model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
88
+ config.update({"architectures": [model_type]})
89
+
90
for key, value in [("rope_scaling", rope_scaling),
91
("rope_theta", rope_theta)]:
92
if value is not None:
93
logger.info("Updating %s from %r to %r", key,
94
getattr(config, key, None), value)
95
config.update({key: value})
96
+
97
return config
98
99
vllm/transformers_utils/tokenizer.py
+9
-1
@@ -1,4 +1,5 @@
1
import os
2
+from pathlib import Path
3
from typing import Optional, Union
4
5
import huggingface_hub
@@ -55,7 +56,7 @@ def get_cached_tokenizer(
56
57
58
def get_tokenizer(
58
- tokenizer_name: str,
59
+ tokenizer_name: Union[str, Path],
60
*args,
61
tokenizer_mode: str = "auto",
62
trust_remote_code: bool = False,
@@ -91,6 +92,13 @@ def get_tokenizer(
92
if "truncation_side" not in kwargs:
93
kwargs["truncation_side"] = "left"
94
95
+ # Separate model folder from file path for GGUF models
96
+ is_gguf = Path(tokenizer_name).is_file() and Path(
97
+ tokenizer_name).suffix == ".gguf"
98
+ if is_gguf:
99
+ kwargs["gguf_file"] = Path(tokenizer_name).name
100
+ tokenizer_name = Path(tokenizer_name).parent
101
+
102
try:
103
tokenizer = AutoTokenizer.from_pretrained(
104
tokenizer_name,