A high-throughput and memory-efficient inference and serving engine for LLMs
[Migration] Migrate GGUF quantization support to plugin (#39612)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jun 13, 2026 at 03:02 UTC
6635279d8a75b9e567080a4c36c74d33b35b0bbd
57 files changed
+71
-9047
.buildkite/test_areas/plugins.yaml
+14
@@ -40,3 +40,17 @@ steps:
40
- pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process
41
- pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process
42
- pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins
43
+
44
+
45
+- label: GGUF Plugin
46
+ key: gguf-plugin
47
+ device: h200_18gb
48
+ timeout_in_minutes: 30
49
+ soft_fail: true
50
+ optional: true
51
+ source_file_dependencies:
52
+ - vllm/model_executor/layers/quantization
53
+ - tests/plugins_tests/test_gguf_plugin.py
54
+ commands:
55
+ - pip install "vllm-gguf-plugin >= 0.0.2"
56
+ - pytest -v -s plugins_tests/gguf
.github/dependabot.yml
-1
@@ -21,7 +21,6 @@ updates:
21
- dependency-name: "torchvision"
22
- dependency-name: "xformers"
23
- dependency-name: "lm-format-enforcer"
24
- - dependency-name: "gguf"
24
- dependency-name: "compressed-tensors"
25
- dependency-name: "ray[cgraph]" # Ray Compiled Graph
26
- dependency-name: "lm-eval"
.pre-commit-config.yaml
+1
-1
@@ -21,7 +21,7 @@ repos:
21
rev: v21.1.2
22
hooks:
23
- id: clang-format
24
- exclude: 'csrc/(libtorch_stable/moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
24
+ exclude: 'csrc/libtorch_stable/moe/topk_softmax_kernels.cu|vllm/third_party/.*'
25
types_or: [c++, cuda]
26
args: [--style=file, --verbose]
27
- repo: https://github.com/DavidAnson/markdownlint-cli2
CMakeLists.txt
-1
@@ -433,7 +433,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
433
"csrc/libtorch_stable/quantization/w8a8/int8/per_token_group_quant.cu"
434
"csrc/libtorch_stable/permute_cols.cu"
435
"csrc/libtorch_stable/quantization/gptq/q_gemm.cu"
436
- "csrc/libtorch_stable/quantization/gguf/gguf_kernel.cu"
436
"csrc/libtorch_stable/pos_encoding_kernels.cu"
437
"csrc/libtorch_stable/fused_qknorm_rope_kernel.cu"
438
"csrc/libtorch_stable/layernorm_kernels.cu"
csrc/libtorch_stable/ops.h
-29
@@ -397,35 +397,6 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
397
void gptq_shuffle(torch::stable::Tensor q_weight, torch::stable::Tensor q_perm,
398
int64_t bit);
399
400
-// GGML kernels (shared CUDA/ROCm)
401
-torch::stable::Tensor ggml_dequantize(
402
- torch::stable::Tensor W, int64_t type, int64_t m, int64_t n,
403
- std::optional<torch::headeronly::ScalarType> const& dtype);
404
-
405
-torch::stable::Tensor ggml_mul_mat_vec_a8(torch::stable::Tensor W,
406
- torch::stable::Tensor X, int64_t type,
407
- int64_t row);
408
-
409
-torch::stable::Tensor ggml_mul_mat_a8(torch::stable::Tensor W,
410
- torch::stable::Tensor X, int64_t type,
411
- int64_t row);
412
-
413
-torch::stable::Tensor ggml_moe_a8(torch::stable::Tensor X,
414
- torch::stable::Tensor W,
415
- torch::stable::Tensor sorted_token_ids,
416
- torch::stable::Tensor expert_ids,
417
- torch::stable::Tensor num_tokens_post_padded,
418
- int64_t type, int64_t row, int64_t top_k,
419
- int64_t tokens);
420
-
421
-torch::stable::Tensor ggml_moe_a8_vec(torch::stable::Tensor X,
422
- torch::stable::Tensor W,
423
- torch::stable::Tensor topk_ids,
424
- int64_t top_k, int64_t type, int64_t row,
425
- int64_t tokens);
426
-
427
-int64_t ggml_moe_get_block_size(int64_t type);
428
-
400
void paged_attention_v1(
401
torch::stable::Tensor& out, torch::stable::Tensor& query,
402
torch::stable::Tensor& key_cache, torch::stable::Tensor& value_cache,
csrc/libtorch_stable/quantization/gguf/dequantize.cuh
deleted
-571
@@ -1,571 +0,0 @@
1
-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu
2
-// Dequant functions
3
-static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
4
- const block_q4_0 * x = (const block_q4_0 *) vx;
5
-
6
- const dfloat d = x[ib].d;
7
-
8
- const int vui = x[ib].qs[iqs];
9
-
10
- v.x = __int2half_rn(vui & 0xF);
11
- v.y = __int2half_rn(vui >> 4);
12
-
13
- v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f));
14
- v = __hmul2(v, {d, d});
15
-}
16
-
17
-static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
18
- const block_q4_1 * x = (const block_q4_1 *) vx;
19
-
20
- const dfloat d = __low2half(x[ib].dm);
21
- const dfloat m = __high2half(x[ib].dm);
22
-
23
- const int vui = x[ib].qs[iqs];
24
-
25
- v.x = __int2half_rn(vui & 0xF);
26
- v.y = __int2half_rn(vui >> 4);
27
-
28
- v = __hmul2(v, {d, d});
29
- v = __hadd2(v, {m, m});
30
-}
31
-
32
-static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
33
- const block_q5_0 * x = (const block_q5_0 *) vx;
34
-
35
- const dfloat d = x[ib].d;
36
-
37
- uint32_t qh;
38
- memcpy(&qh, x[ib].qh, sizeof(qh));
39
-
40
- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
41
- const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
42
-
43
- v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
44
- v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
45
-
46
- v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f));
47
- v = __hmul2(v, {d, d});
48
-}
49
-
50
-static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
51
- const block_q5_1 * x = (const block_q5_1 *) vx;
52
-
53
- const dfloat d = __low2half(x[ib].dm);
54
- const dfloat m = __high2half(x[ib].dm);
55
-
56
- uint32_t qh;
57
- memcpy(&qh, x[ib].qh, sizeof(qh));
58
-
59
- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
60
- const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
61
-
62
- v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
63
- v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
64
-
65
- v = __hmul2(v, {d, d});
66
- v = __hadd2(v, {m, m});
67
-}
68
-
69
-static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
70
- const block_q8_0 * x = (const block_q8_0 *) vx;
71
-
72
- const dfloat d = x[ib].d;
73
-
74
- v.x = __int2half_rn(x[ib].qs[iqs + 0]);
75
- v.y = __int2half_rn(x[ib].qs[iqs + 1]);
76
-
77
- v = __hmul2(v, {d, d});
78
-}
79
-
80
-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
81
-static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {
82
- const int64_t i = 2*((int64_t)blockDim.x*blockIdx.x + threadIdx.x);
83
-
84
- if (i >= k) {
85
- return;
86
- }
87
-
88
- const int ib = i/qk; // block index
89
- const int iqs = (i%qk)/qr; // quant index
90
- const int iybs = i - i%qk; // y block start index
91
- const int y_offset = qr == 1 ? 1 : qk/2;
92
-
93
- // dequantize
94
- dfloat2 v;
95
- dequantize_kernel(vx, ib, iqs, v);
96
-
97
- y[iybs + iqs + 0] = convert_from_half<dst_t>(v.x);
98
- y[iybs + iqs + y_offset] = convert_from_half<dst_t>(v.y);
99
-}
100
-
101
-template<typename dst_t>
102
-static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
103
-
104
- const auto i = blockIdx.x;
105
- const block_q2_K * x = (const block_q2_K *) vx;
106
-
107
- const auto tid = threadIdx.x;
108
- const int n = tid/32;
109
- const int l = tid - 32*n;
110
- const int is = 8*n + l/16;
111
-
112
- const uint8_t q = x[i].qs[32*n + l];
113
- dst_t * y = yy + i*QK_K + 128*n;
114
-
115
- half dall = __low2half(x[i].dm);
116
- half dmin = __high2half(x[i].dm);
117
- y[l+ 0] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+0] & 0xF) * ((q >> 0) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+0] >> 4))));
118
- y[l+32] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+2] & 0xF) * ((q >> 2) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+2] >> 4))));
119
- y[l+64] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+4] & 0xF) * ((q >> 4) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+4] >> 4))));
120
- y[l+96] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+6] & 0xF) * ((q >> 6) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+6] >> 4))));
121
-}
122
-
123
-template<typename dst_t>
124
-static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
125
-
126
- const auto i = blockIdx.x;
127
- const block_q3_K * x = (const block_q3_K *) vx;
128
-
129
- const auto r = threadIdx.x/4;
130
- const int tid = r/2;
131
- const int is0 = r%2;
132
- const int l0 = 16*is0 + 4*(threadIdx.x%4);
133
- const int n = tid / 4;
134
- const int j = tid - 4*n;
135
-
136
- uint8_t m = 1 << (4*n + j);
137
- int is = 8*n + 2*j + is0;
138
- int shift = 2*j;
139
-
140
- int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :
141
- is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :
142
- is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :
143
- (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);
144
- half d_all = x[i].d;
145
- half dl = __hmul(d_all, __int2half_rn(us - 32));
146
-
147
- dst_t * y = yy + i*QK_K + 128*n + 32*j;
148
- const uint8_t * q = x[i].qs + 32*n;
149
- const uint8_t * hm = x[i].hmask;
150
-
151
- for (int l = l0; l < l0+4; ++l) {
152
- y[l] = convert_from_half<dst_t>(__hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4))));
153
- }
154
-}
155
-
156
-static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {
157
- if (j < 4) {
158
- d = q[j] & 63; m = q[j + 4] & 63;
159
- } else {
160
- d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
161
- m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
162
- }
163
-}
164
-
165
-template<typename dst_t>
166
-static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
167
- const block_q4_K * x = (const block_q4_K *) vx;
168
-
169
- const auto i = blockIdx.x;
170
-
171
- // assume 32 threads
172
- const auto tid = threadIdx.x;
173
- const int il = tid/8;
174
- const int ir = tid%8;
175
- const int is = 2*il;
176
- const int n = 4;
177
-
178
- dst_t * y = yy + i*QK_K + 64*il + n*ir;
179
-
180
- const half dall = __low2half(x[i].dm);
181
- const half dmin = __high2half(x[i].dm);
182
-
183
- const uint8_t * q = x[i].qs + 32*il + n*ir;
184
-
185
- uint8_t sc, m;
186
- get_scale_min_k4(is + 0, x[i].scales, sc, m);
187
- const half d1 = __hmul(dall, __int2half_rn(sc));
188
- const half m1 = __hmul(dmin, __int2half_rn(m));
189
- get_scale_min_k4(is + 1, x[i].scales, sc, m);
190
- const half d2 = __hmul(dall, __int2half_rn(sc));
191
- const half m2 = __hmul(dmin, __int2half_rn(m));
192
- for (int l = 0; l < n; ++l) {
193
- y[l + 0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1));
194
- y[l +32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2));
195
- }
196
-}
197
-
198
-template<typename dst_t>
199
-static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
200
- const block_q5_K * x = (const block_q5_K *) vx;
201
-
202
- const auto i = blockIdx.x;
203
-
204
- // assume 64 threads - this is very slightly better than the one below
205
- const auto tid = threadIdx.x;
206
- const int il = tid/16; // il is in 0...3
207
- const int ir = tid%16; // ir is in 0...15
208
- const int is = 2*il; // is is in 0...6
209
-
210
- dst_t * y = yy + i*QK_K + 64*il + 2*ir;
211
-
212
- const half dall = __low2half(x[i].dm);
213
- const half dmin = __high2half(x[i].dm);
214
-
215
- const uint8_t * ql = x[i].qs + 32*il + 2*ir;
216
- const uint8_t * qh = x[i].qh + 2*ir;
217
-
218
- uint8_t sc, m;
219
- get_scale_min_k4(is + 0, x[i].scales, sc, m);
220
- const half d1 = __hmul(dall, __int2half_rn(sc)); const half m1 = __hmul(dmin, __int2half_rn(m));
221
- get_scale_min_k4(is + 1, x[i].scales, sc, m);
222
- const half d2 = __hmul(dall, __int2half_rn(sc)); const half m2 = __hmul(dmin, __int2half_rn(m));
223
-
224
- uint8_t hm = 1 << (2*il);
225
- y[ 0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1));
226
- y[ 1] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1));
227
- hm <<= 1;
228
- y[32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2));
229
- y[33] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2));
230
-}
231
-
232
-template<typename dst_t>
233
-static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
234
- const block_q6_K * x = (const block_q6_K *) vx;
235
-
236
- const auto i = blockIdx.x;
237
-
238
- // assume 64 threads - this is very slightly better than the one below
239
- const auto tid = threadIdx.x;
240
- const int ip = tid/32; // ip is 0 or 1
241
- const int il = tid - 32*ip; // 0...32
242
- const int is = 8*ip + il/16;
243
-
244
- dst_t * y = yy + i*QK_K + 128*ip + il;
245
-
246
- const half d = x[i].d;
247
-
248
- const uint8_t * ql = x[i].ql + 64*ip + il;
249
- const uint8_t qh = x[i].qh[32*ip + il];
250
- const int8_t * sc = x[i].scales + is;
251
-
252
- y[ 0] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32))));
253
- y[32] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32))));
254
- y[64] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32))));
255
- y[96] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32))));
256
-}
257
-
258
-template<typename dst_t>
259
-static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
260
-
261
- const auto i = blockIdx.x;
262
- const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
263
-
264
- const auto tid = threadIdx.x;
265
- const int il = tid/8; // 0...3
266
- const int ib = tid%8; // 0...7
267
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
268
- const uint16_t * q2 = x[i].qs + 4*ib;
269
- const uint8_t * aux8 = (const uint8_t *)q2;
270
- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);
271
- const uint32_t aux32 = q2[2] | (q2[3] << 16);
272
- const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f;
273
- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
274
- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
275
-}
276
-
277
-template<typename dst_t>
278
-static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
279
-
280
- const auto i = blockIdx.x;
281
- const block_iq2_xs * x = (const block_iq2_xs *) vx;
282
-
283
- const auto tid = threadIdx.x;
284
- const int il = tid/8; // 0...3
285
- const int ib = tid%8; // 0...7
286
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
287
- const uint16_t * q2 = x[i].qs + 4*ib;
288
- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
289
- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
290
- const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
291
- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
292
-
293
-}
294
-
295
-template<typename dst_t>
296
-static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
297
-
298
- const auto i = blockIdx.x;
299
- const block_iq2_s * x = (const block_iq2_s *) vx;
300
-
301
- const auto tid = threadIdx.x;
302
- const int il = tid/8; // 0...3
303
- const int ib = tid%8; // 0...7
304
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
305
- const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));
306
- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
307
- const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];
308
- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
309
-}
310
-
311
-template<typename dst_t>
312
-static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
313
-
314
- const auto i = blockIdx.x;
315
- const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
316
-
317
- const auto tid = threadIdx.x;
318
- const int il = tid/8; // 0...3
319
- const int ib = tid%8; // 0...7
320
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
321
- const uint8_t * q3 = x[i].qs + 8*ib;
322
- const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;
323
- const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);
324
- const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);
325
- const uint32_t aux32 = gas[0] | (gas[1] << 16);
326
- const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f;
327
- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
328
- for (int j = 0; j < 4; ++j) {
329
- y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
330
- y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
331
- }
332
-}
333
-
334
-template<typename dst_t>
335
-static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
336
-
337
- const auto i = blockIdx.x;
338
- const block_iq3_s * x = (const block_iq3_s *) vx;
339
-
340
- const auto tid = threadIdx.x;
341
- const int il = tid/8; // 0...3
342
- const int ib = tid%8; // 0...7
343
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
344
- const uint8_t * qs = x[i].qs + 8*ib;
345
- const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));
346
- const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));
347
- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f;
348
- const uint8_t signs = x[i].signs[4*ib + il];
349
- for (int j = 0; j < 4; ++j) {
350
- y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
351
- y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
352
- }
353
-}
354
-
355
-template<typename dst_t>
356
-static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
357
-
358
- const int64_t i = blockIdx.x;
359
- const block_iq1_s * x = (const block_iq1_s *) vx;
360
-
361
- const int64_t tid = threadIdx.x;
362
- const int64_t il = tid/8; // 0...3
363
- const int64_t ib = tid%8; // 0...7
364
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
365
- const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;
366
- const float d = __half2float(x[i].d) * (2*((x[i].qh[ib] >> 12) & 7) + 1);
367
- uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
368
- grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];
369
- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
370
- grid32[0] &= 0x0f0f0f0f;
371
- for (int j = 0; j < 8; ++j) {
372
- y[j] = d * (q[j] + delta);
373
- }
374
-}
375
-
376
-template<typename dst_t>
377
-static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {
378
-
379
- const int64_t i = blockIdx.x;
380
- const block_iq1_m * x = (const block_iq1_m *) vx;
381
-
382
- const int64_t tid = threadIdx.x;
383
- const int64_t il = tid/8; // 0...3
384
- const int64_t ib = tid%8; // 0...7
385
- dst_t * y = yy + i*QK_K + 32*ib + 8*il;
386
- const uint16_t * sc = (const uint16_t *)x[i].scales;
387
- iq1m_scale_t scale;
388
- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
389
- const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
390
- const float d = __half2float(scale.f16) * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);
391
- const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;
392
- uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
393
- grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];
394
- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
395
- grid32[0] &= 0x0f0f0f0f;
396
- for (int j = 0; j < 8; ++j) {
397
- y[j] = d * (q[j] + delta);
398
- }
399
-}
400
-
401
-template<typename dst_t>
402
-static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {
403
-
404
- const auto i = blockIdx.x;
405
- const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);
406
-
407
- const auto tid = threadIdx.x;
408
- const int il = tid/8; // 0...3
409
- const int ib = tid%8; // 0...7
410
- dst_t * y = yy + i*QK_K + 32*ib + 4*il;
411
- const uint8_t * q4 = x[ib].qs + 4*il;
412
- const float d = __half2float(x[ib].d);
413
- for (int j = 0; j < 4; ++j) {
414
- y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];
415
- y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];
416
- }
417
-
418
-}
419
-
420
-template<typename dst_t>
421
-static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
422
- const auto i = blockIdx.x;
423
- const block_iq4_xs * x = (const block_iq4_xs *)vx;
424
-
425
- const auto tid = threadIdx.x;
426
- const int il = tid/8; // 0...3
427
- const int ib = tid%8; // 0...7
428
- dst_t * y = yy + i*QK_K + 32*ib + 4*il;
429
- const uint8_t * q4 = x[i].qs + 16*ib + 4*il;
430
- const float d = __half2float(x[i].d) * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);
431
- for (int j = 0; j < 4; ++j) {
432
- y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];
433
- y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];
434
- }
435
-}
436
-
437
-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
438
-static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {
439
- const int64_t num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);
440
- dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
441
-}
442
-
443
-template<typename dst_t>
444
-static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
445
- const int nb = k / QK_K;
446
- dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
447
-}
448
-
449
-template<typename dst_t>
450
-static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
451
- const int nb = k / QK_K;
452
- dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
453
-}
454
-
455
-template<typename dst_t>
456
-static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
457
- const int nb = k / QK_K;
458
- dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
459
-}
460
-
461
-template<typename dst_t>
462
-static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
463
- const int nb = k / QK_K;
464
- dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
465
-}
466
-
467
-template<typename dst_t>
468
-static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
469
- const int nb = k / QK_K;
470
- dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
471
-}
472
-
473
-template<typename dst_t>
474
-static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
475
- const int nb = k / QK_K;
476
- dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
477
-}
478
-
479
-template<typename dst_t>
480
-static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
481
- const int nb = k / QK_K;
482
- dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
483
-}
484
-
485
-template<typename dst_t>
486
-static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
487
- const int nb = k / QK_K;
488
- dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
489
-}
490
-
491
-template<typename dst_t>
492
-static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
493
- const int nb = k / QK_K;
494
- dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
495
-}
496
-
497
-template<typename dst_t>
498
-static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
499
- const int nb = k / QK_K;
500
- dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
501
-}
502
-
503
-template<typename dst_t>
504
-static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
505
- const int nb = k / QK_K;
506
- dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
507
-}
508
-
509
-template<typename dst_t>
510
-static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
511
- const int nb = k / QK_K;
512
- dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);
513
-}
514
-
515
-template<typename dst_t>
516
-static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
517
- const int nb = (k + QK_K - 1) / QK_K;
518
- dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);
519
-}
520
-
521
-template<typename dst_t>
522
-static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
523
- const int nb = (k + QK_K - 1) / QK_K;
524
- dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
525
-}
526
-
527
-template<typename dst_t>
528
-static to_cuda_ggml_t<dst_t> ggml_get_to_cuda(int64_t type) {
529
- switch (type) {
530
- case 2:
531
- return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
532
- case 3:
533
- return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
534
- case 6:
535
- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
536
- case 7:
537
- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
538
- case 8:
539
- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
540
- case 10:
541
- return dequantize_row_q2_K_cuda;
542
- case 11:
543
- return dequantize_row_q3_K_cuda;
544
- case 12:
545
- return dequantize_row_q4_K_cuda;
546
- case 13:
547
- return dequantize_row_q5_K_cuda;
548
- case 14:
549
- return dequantize_row_q6_K_cuda;
550
- case 16:
551
- return dequantize_row_iq2_xxs_cuda;
552
- case 17:
553
- return dequantize_row_iq2_xs_cuda;
554
- case 18:
555
- return dequantize_row_iq3_xxs_cuda;
556
- case 19:
557
- return dequantize_row_iq1_s_cuda;
558
- case 20:
559
- return dequantize_row_iq4_nl_cuda;
560
- case 21:
561
- return dequantize_row_iq3_s_cuda;
562
- case 22:
563
- return dequantize_row_iq2_s_cuda;
564
- case 23:
565
- return dequantize_row_iq4_xs_cuda;
566
- case 29:
567
- return dequantize_row_iq1_m_cuda;
568
- default:
569
- return nullptr;
570
- }
571
-}
csrc/libtorch_stable/quantization/gguf/ggml-common.h
deleted
-1150
@@ -1,1150 +0,0 @@
1
-// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
2
-#define QK_K 256
3
-#define K_QUANTS_PER_ITERATION 2
4
-#define WARP_SIZE_GGUF 32
5
-#define K_SCALE_SIZE 12
6
-#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
7
-#define CUDA_QUANTIZE_BLOCK_SIZE 256
8
-#define GGML_CUDA_DMMV_X 32
9
-#define GGML_CUDA_MMV_Y 1
10
-
11
-
12
-// Data Structures
13
-// QK = number of values after dequantization
14
-// QR = QK / number of values before dequantization
15
-// QI = number of 32 bit integers before dequantization
16
-
17
-#define QK4_0 32
18
-#define QR4_0 2
19
-#define QI4_0 (QK4_0 / (4 * QR4_0))
20
-typedef struct {
21
- half d; // delta
22
- uint8_t qs[QK4_0 / 2]; // nibbles / quants
23
-} block_q4_0;
24
-
25
-#define QK4_1 32
26
-#define QR4_1 2
27
-#define QI4_1 (QK4_1 / (4 * QR4_1))
28
-typedef struct {
29
- half2 dm; // dm.x = delta, dm.y = min
30
- uint8_t qs[QK4_1 / 2]; // nibbles / quants
31
-} block_q4_1;
32
-
33
-#define QK5_0 32
34
-#define QR5_0 2
35
-#define QI5_0 (QK5_0 / (4 * QR5_0))
36
-typedef struct {
37
- half d; // delta
38
- uint8_t qh[4]; // 5-th bit of quants
39
- uint8_t qs[QK5_0 / 2]; // nibbles / quants
40
-} block_q5_0;
41
-
42
-#define QK5_1 32
43
-#define QR5_1 2
44
-#define QI5_1 (QK5_1 / (4 * QR5_1))
45
-typedef struct {
46
- half2 dm; // dm.x = delta, dm.y = min
47
- uint8_t qh[4]; // 5-th bit of quants
48
- uint8_t qs[QK5_1 / 2]; // nibbles / quants
49
-} block_q5_1;
50
-
51
-#define QK8_0 32
52
-#define QR8_0 1
53
-#define QI8_0 (QK8_0 / (4 * QR8_0))
54
-typedef struct {
55
- half d; // delta
56
- int8_t qs[QK8_0]; // quants
57
-} block_q8_0;
58
-
59
-#define QK8_1 32
60
-#define QR8_1 1
61
-#define QI8_1 (QK8_1 / (4 * QR8_1))
62
-typedef struct {
63
- half2 ds; // ds.x = delta, ds.y = sum
64
- int8_t qs[QK8_0]; // quants
65
-} block_q8_1;
66
-
67
-#define QR2_K 4
68
-#define QI2_K (QK_K / (4*QR2_K))
69
-typedef struct {
70
- uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
71
- uint8_t qs[QK_K/4]; // quants
72
- half2 dm; // super-block scale for quantized scales/mins
73
-} block_q2_K;
74
-
75
-#define QR3_K 4
76
-#define QI3_K (QK_K / (4*QR3_K))
77
-typedef struct {
78
- uint8_t hmask[QK_K/8]; // quants - high bit
79
- uint8_t qs[QK_K/4]; // quants - low 2 bits
80
- uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits
81
- half d; // super-block scale
82
-} block_q3_K;
83
-
84
-#define QR4_K 2
85
-#define QI4_K (QK_K / (4*QR4_K))
86
-typedef struct {
87
- half2 dm; // super-block scale for quantized scales/mins
88
- uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits
89
- uint8_t qs[QK_K/2]; // 4--bit quants
90
-} block_q4_K;
91
-
92
-#define QR5_K 2
93
-#define QI5_K (QK_K / (4*QR5_K))
94
-typedef struct {
95
- half2 dm; // super-block scale for quantized scales/mins
96
- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
97
- uint8_t qh[QK_K/8]; // quants, high bit
98
- uint8_t qs[QK_K/2]; // quants, low 4 bits
99
-} block_q5_K;
100
-
101
-#define QR6_K 2
102
-#define QI6_K (QK_K / (4*QR6_K))
103
-typedef struct {
104
- uint8_t ql[QK_K/2]; // quants, lower 4 bits
105
- uint8_t qh[QK_K/4]; // quants, upper 2 bits
106
- int8_t scales[QK_K/16]; // scales
107
- half d; // delta
108
-} block_q6_K;
109
-
110
-#define QR2_XXS 8
111
-#define QI2_XXS (QK_K / (4*QR2_XXS))
112
-typedef struct {
113
- half d;
114
- uint16_t qs[QK_K/8];
115
-} block_iq2_xxs;
116
-
117
-#define QR2_XS 8
118
-#define QI2_XS (QK_K / (4*QR2_XS))
119
-typedef struct {
120
- half d;
121
- uint16_t qs[QK_K/8];
122
- uint8_t scales[QK_K/32];
123
-} block_iq2_xs;
124
-
125
-#define QR2_S 8
126
-#define QI2_S (QK_K / (4*QR2_S))
127
-typedef struct {
128
- half d;
129
- uint8_t qs[QK_K/4];
130
- uint8_t qh[QK_K/32];
131
- uint8_t scales[QK_K/32];
132
-} block_iq2_s;
133
-
134
-#define QR3_XXS 8
135
-#define QI3_XXS (QK_K / (4*QR3_XXS))
136
-typedef struct {
137
- half d;
138
- uint8_t qs[3*(QK_K/8)];
139
-} block_iq3_xxs;
140
-
141
-#define QR3_XS 8
142
-#define QI3_XS (QK_K / (4*QR3_XS))
143
-#define IQ3S_N_SCALE QK_K/64
144
-typedef struct {
145
- half d;
146
- uint8_t qs[QK_K/4];
147
- uint8_t qh[QK_K/32];
148
- uint8_t signs[QK_K/8];
149
- uint8_t scales[IQ3S_N_SCALE];
150
-} block_iq3_s;
151
-
152
-// 1.5625 bpw
153
-#define QR1_S 8
154
-#define QI1_S (QK_K / (4*QR1_S))
155
-typedef struct {
156
- half d;
157
- uint8_t qs[QK_K/8];
158
- uint16_t qh[QK_K/32];
159
-} block_iq1_s;
160
-
161
-// 1.75 bpw
162
-#define QR1_M 8
163
-#define QI1_M (QK_K / (4*QR1_M))
164
-typedef struct {
165
- uint8_t qs[QK_K/8]; // grid index, low 8 bits
166
- uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)
167
- uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64)
168
-} block_iq1_m;
169
-
170
-// Used by IQ1_M quants
171
-typedef union {
172
- half f16;
173
- uint16_t u16;
174
-} iq1m_scale_t;
175
-
176
-#define QK4_NL 32
177
-#define QR4_NL 2
178
-#define QI4_NL (QK4_NL / (4*QR4_NL))
179
-typedef struct {
180
- half d;
181
- uint8_t qs[QK4_NL/2];
182
-} block_iq4_nl;
183
-
184
-#define QR4_XS 8
185
-#define QI4_XS (QK_K / (4*QR4_XS))
186
-typedef struct {
187
- half d;
188
- uint16_t scales_h;
189
- uint8_t scales_l[QK_K/64];
190
- uint8_t qs[QK_K/2];
191
-} block_iq4_xs;
192
-
193
-static const __device__ uint64_t iq2xxs_grid[256] = {
194
- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
195
- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,
196
- 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,
197
- 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,
198
- 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,
199
- 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,
200
- 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,
201
- 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,
202
- 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,
203
- 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,
204
- 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,
205
- 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,
206
- 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,
207
- 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,
208
- 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,
209
- 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,
210
- 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,
211
- 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,
212
- 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,
213
- 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,
214
- 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,
215
- 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,
216
- 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,
217
- 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,
218
- 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,
219
- 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,
220
- 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,
221
- 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,
222
- 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,
223
- 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,
224
- 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,
225
- 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,
226
- 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,
227
- 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,
228
- 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,
229
- 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,
230
- 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,
231
- 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,
232
- 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,
233
- 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,
234
- 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,
235
- 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,
236
- 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,
237
- 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,
238
- 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,
239
- 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,
240
- 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,
241
- 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,
242
- 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,
243
- 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,
244
- 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,
245
- 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,
246
- 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,
247
- 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,
248
- 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,
249
- 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,
250
- 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,
251
- 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,
252
- 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,
253
- 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,
254
- 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,
255
- 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,
256
- 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,
257
- 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,
258
-};
259
-
260
-static const __device__ uint64_t iq2xs_grid[512] = {
261
- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
262
- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
263
- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
264
- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
265
- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
266
- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,
267
- 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,
268
- 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,
269
- 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,
270
- 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,
271
- 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,
272
- 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,
273
- 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,
274
- 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,
275
- 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,
276
- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,
277
- 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,
278
- 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,
279
- 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,
280
- 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,
281
- 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,
282
- 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,
283
- 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,
284
- 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,
285
- 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,
286
- 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,
287
- 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,
288
- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,
289
- 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,
290
- 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,
291
- 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,
292
- 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,
293
- 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,
294
- 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,
295
- 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,
296
- 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,
297
- 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,
298
- 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,
299
- 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,
300
- 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,
301
- 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,
302
- 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,
303
- 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,
304
- 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,
305
- 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,
306
- 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,
307
- 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,
308
- 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,
309
- 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,
310
- 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,
311
- 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,
312
- 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,
313
- 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,
314
- 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,
315
- 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,
316
- 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,
317
- 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,
318
- 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,
319
- 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,
320
- 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,
321
- 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,
322
- 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,
323
- 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,
324
- 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,
325
- 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,
326
- 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,
327
- 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,
328
- 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,
329
- 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,
330
- 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,
331
- 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,
332
- 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,
333
- 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,
334
- 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,
335
- 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,
336
- 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,
337
- 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,
338
- 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,
339
- 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,
340
- 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,
341
- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,
342
- 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,
343
- 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,
344
- 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,
345
- 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,
346
- 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,
347
- 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,
348
- 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,
349
- 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,
350
- 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,
351
- 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,
352
- 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,
353
- 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,
354
- 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,
355
- 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,
356
- 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,
357
- 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,
358
- 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,
359
- 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,
360
- 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,
361
- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,
362
- 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,
363
- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,
364
- 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,
365
- 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,
366
- 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,
367
- 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,
368
- 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,
369
- 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,
370
- 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,
371
- 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,
372
- 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,
373
- 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,
374
- 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,
375
- 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,
376
- 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,
377
- 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,
378
- 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,
379
- 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,
380
- 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,
381
- 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,
382
- 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,
383
- 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,
384
- 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,
385
- 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,
386
- 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,
387
- 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,
388
- 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,
389
-};
390
-
391
-static const __device__ uint64_t iq2s_grid[1024] = {
392
- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
393
- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
394
- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
395
- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
396
- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
397
- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,
398
- 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,
399
- 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,
400
- 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,
401
- 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
402
- 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,
403
- 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,
404
- 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,
405
- 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,
406
- 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
407
- 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,
408
- 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,
409
- 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,
410
- 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,
411
- 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
412
- 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,
413
- 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,
414
- 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,
415
- 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,
416
- 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
417
- 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,
418
- 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,
419
- 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,
420
- 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,
421
- 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
422
- 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,
423
- 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,
424
- 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,
425
- 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,
426
- 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
427
- 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,
428
- 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,
429
- 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,
430
- 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,
431
- 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
432
- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,
433
- 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,
434
- 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,
435
- 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,
436
- 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
437
- 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,
438
- 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,
439
- 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,
440
- 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,
441
- 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
442
- 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,
443
- 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,
444
- 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,
445
- 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,
446
- 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
447
- 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,
448
- 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,
449
- 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,
450
- 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,
451
- 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
452
- 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,
453
- 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,
454
- 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,
455
- 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,
456
- 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
457
- 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,
458
- 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,
459
- 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,
460
- 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,
461
- 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
462
- 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,
463
- 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,
464
- 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,
465
- 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,
466
- 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
467
- 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,
468
- 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,
469
- 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,
470
- 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,
471
- 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
472
- 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,
473
- 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,
474
- 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,
475
- 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,
476
- 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
477
- 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,
478
- 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,
479
- 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,
480
- 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,
481
- 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
482
- 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,
483
- 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,
484
- 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,
485
- 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,
486
- 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
487
- 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,
488
- 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,
489
- 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,
490
- 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,
491
- 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
492
- 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,
493
- 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,
494
- 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,
495
- 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,
496
- 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
497
- 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,
498
- 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,
499
- 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,
500
- 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,
501
- 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
502
- 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,
503
- 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,
504
- 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,
505
- 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,
506
- 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
507
- 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,
508
- 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,
509
- 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
510
- 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,
511
- 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
512
- 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,
513
- 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,
514
- 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,
515
- 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,
516
- 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
517
- 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,
518
- 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,
519
- 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,
520
- 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,
521
- 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
522
- 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,
523
- 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,
524
- 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,
525
- 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,
526
- 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
527
- 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,
528
- 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,
529
- 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,
530
- 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,
531
- 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
532
- 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,
533
- 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,
534
- 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,
535
- 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,
536
- 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
537
- 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,
538
- 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,
539
- 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,
540
- 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,
541
- 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
542
- 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,
543
- 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,
544
- 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,
545
- 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,
546
- 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
547
- 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,
548
- 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,
549
- 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,
550
- 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,
551
- 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
552
- 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,
553
- 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,
554
- 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,
555
- 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,
556
- 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
557
- 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,
558
- 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,
559
- 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,
560
- 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,
561
- 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
562
- 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,
563
- 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,
564
- 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,
565
- 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,
566
- 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
567
- 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,
568
- 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,
569
- 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,
570
- 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,
571
- 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
572
- 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,
573
- 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,
574
- 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,
575
- 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,
576
- 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
577
- 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,
578
- 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,
579
- 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,
580
- 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,
581
- 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
582
- 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,
583
- 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,
584
- 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,
585
- 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,
586
- 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
587
- 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,
588
- 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,
589
- 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,
590
- 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,
591
- 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
592
- 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,
593
- 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,
594
- 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,
595
- 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,
596
- 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
597
- 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
598
- 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,
599
- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,
600
- 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,
601
- 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
602
- 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,
603
- 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,
604
- 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,
605
- 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,
606
- 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
607
- 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,
608
- 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,
609
- 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,
610
- 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,
611
- 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
612
- 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,
613
- 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,
614
- 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,
615
- 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,
616
- 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
617
- 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,
618
- 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,
619
- 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,
620
- 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,
621
- 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
622
- 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,
623
- 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,
624
- 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,
625
- 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,
626
- 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
627
- 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,
628
- 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,
629
- 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,
630
- 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,
631
- 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
632
- 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,
633
- 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,
634
- 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,
635
- 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,
636
- 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
637
- 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,
638
- 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,
639
- 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,
640
- 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,
641
- 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
642
- 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,
643
- 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,
644
- 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,
645
- 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,
646
- 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
647
- 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
648
-};
649
-
650
-static const __device__ uint32_t iq3xxs_grid[256] = {
651
- 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
652
- 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
653
- 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
654
- 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
655
- 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
656
- 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
657
- 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
658
- 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
659
- 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
660
- 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
661
- 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
662
- 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
663
- 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
664
- 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
665
- 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
666
- 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
667
- 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
668
- 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
669
- 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
670
- 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
671
- 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
672
- 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
673
- 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
674
- 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
675
- 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
676
- 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
677
- 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
678
- 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
679
- 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
680
- 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
681
- 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
682
- 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,
683
-};
684
-
685
-static const __device__ uint32_t iq3xs_grid[512] = {
686
- 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14,
687
- 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414,
688
- 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24,
689
- 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c,
690
- 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c,
691
- 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34,
692
- 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c,
693
- 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414,
694
- 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,
695
- 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404,
696
- 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434,
697
- 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c,
698
- 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404,
699
- 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414,
700
- 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414,
701
- 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404,
702
- 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c,
703
- 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,
704
- 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404,
705
- 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e,
706
- 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14,
707
- 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c,
708
- 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424,
709
- 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c,
710
- 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c,
711
- 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e,
712
- 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,
713
- 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e,
714
- 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424,
715
- 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e,
716
- 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424,
717
- 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404,
718
- 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c,
719
- 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e,
720
- 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c,
721
- 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,
722
- 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c,
723
- 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404,
724
- 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04,
725
- 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c,
726
- 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414,
727
- 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c,
728
- 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c,
729
- 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424,
730
- 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,
731
- 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c,
732
- 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414,
733
- 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c,
734
- 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e,
735
- 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04,
736
- 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424,
737
- 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14,
738
- 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34,
739
- 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,
740
- 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434,
741
- 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c,
742
- 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424,
743
- 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24,
744
- 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24,
745
- 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e,
746
- 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c,
747
- 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c,
748
- 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,
749
- 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,
750
-};
751
-
752
-#define IQ1S_DELTA 0.125f
753
-#define IQ1M_DELTA 0.125f
754
-static const __device__ uint64_t iq1s_grid_gpu[2048] = {
755
- 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,
756
- 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,
757
- 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,
758
- 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,
759
- 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,
760
- 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,
761
- 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,
762
- 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,
763
- 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,
764
- 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,
765
- 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,
766
- 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,
767
- 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,
768
- 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,
769
- 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,
770
- 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,
771
- 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,
772
- 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,
773
- 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,
774
- 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,
775
- 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,
776
- 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,
777
- 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,
778
- 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,
779
- 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,
780
- 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,
781
- 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,
782
- 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,
783
- 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,
784
- 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,
785
- 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,
786
- 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,
787
- 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,
788
- 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,
789
- 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,
790
- 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,
791
- 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,
792
- 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,
793
- 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,
794
- 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,
795
- 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,
796
- 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,
797
- 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,
798
- 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,
799
- 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,
800
- 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,
801
- 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,
802
- 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,
803
- 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,
804
- 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,
805
- 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,
806
- 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,
807
- 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,
808
- 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,
809
- 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,
810
- 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,
811
- 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,
812
- 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,
813
- 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,
814
- 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,
815
- 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,
816
- 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,
817
- 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,
818
- 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,
819
- 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,
820
- 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,
821
- 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,
822
- 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,
823
- 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,
824
- 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,
825
- 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,
826
- 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,
827
- 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,
828
- 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,
829
- 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,
830
- 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,
831
- 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,
832
- 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,
833
- 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,
834
- 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,
835
- 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,
836
- 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,
837
- 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,
838
- 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,
839
- 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,
840
- 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,
841
- 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,
842
- 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,
843
- 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,
844
- 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,
845
- 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,
846
- 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,
847
- 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,
848
- 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,
849
- 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,
850
- 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,
851
- 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,
852
- 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,
853
- 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,
854
- 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,
855
- 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,
856
- 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,
857
- 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,
858
- 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,
859
- 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,
860
- 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,
861
- 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,
862
- 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,
863
- 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,
864
- 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,
865
- 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,
866
- 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,
867
- 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,
868
- 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,
869
- 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,
870
- 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,
871
- 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,
872
- 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,
873
- 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,
874
- 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,
875
- 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,
876
- 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,
877
- 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,
878
- 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,
879
- 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,
880
- 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,
881
- 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,
882
- 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,
883
- 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,
884
- 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,
885
- 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,
886
- 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,
887
- 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,
888
- 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,
889
- 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,
890
- 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,
891
- 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,
892
- 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,
893
- 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,
894
- 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,
895
- 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,
896
- 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,
897
- 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,
898
- 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,
899
- 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,
900
- 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,
901
- 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,
902
- 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,
903
- 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,
904
- 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,
905
- 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,
906
- 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,
907
- 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,
908
- 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,
909
- 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,
910
- 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,
911
- 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,
912
- 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,
913
- 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,
914
- 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,
915
- 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,
916
- 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,
917
- 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,
918
- 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,
919
- 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,
920
- 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,
921
- 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,
922
- 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,
923
- 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,
924
- 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,
925
- 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,
926
- 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,
927
- 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,
928
- 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,
929
- 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,
930
- 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,
931
- 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,
932
- 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,
933
- 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,
934
- 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,
935
- 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,
936
- 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,
937
- 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,
938
- 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,
939
- 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,
940
- 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,
941
- 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,
942
- 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,
943
- 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,
944
- 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,
945
- 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,
946
- 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,
947
- 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,
948
- 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,
949
- 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,
950
- 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,
951
- 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,
952
- 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,
953
- 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,
954
- 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,
955
- 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,
956
- 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,
957
- 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,
958
- 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,
959
- 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,
960
- 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,
961
- 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,
962
- 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,
963
- 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,
964
- 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,
965
- 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,
966
- 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,
967
- 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,
968
- 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,
969
- 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,
970
- 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,
971
- 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,
972
- 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,
973
- 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,
974
- 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,
975
- 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,
976
- 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,
977
- 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,
978
- 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,
979
- 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,
980
- 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,
981
- 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,
982
- 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,
983
- 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,
984
- 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,
985
- 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,
986
- 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,
987
- 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,
988
- 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,
989
- 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,
990
- 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,
991
- 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,
992
- 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,
993
- 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,
994
- 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,
995
- 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,
996
- 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,
997
- 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,
998
- 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,
999
- 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,
1000
- 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,
1001
- 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,
1002
- 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,
1003
- 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,
1004
- 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,
1005
- 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,
1006
- 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,
1007
- 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,
1008
- 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,
1009
- 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,
1010
- 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,
1011
-};
1012
-
1013
-static const __device__ uint8_t ksigns_iq2xs[128] = {
1014
- 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,
1015
- 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,
1016
- 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,
1017
- 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,
1018
- 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,
1019
- 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,
1020
- 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,
1021
- 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
1022
-};
1023
-
1024
-static const __device__ uint64_t ksigns64[128] = {
1025
- 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
1026
- 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,
1027
- 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,
1028
- 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,
1029
- 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,
1030
- 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,
1031
- 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,
1032
- 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,
1033
- 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,
1034
- 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,
1035
- 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,
1036
- 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,
1037
- 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,
1038
- 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,
1039
- 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,
1040
- 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,
1041
- 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,
1042
- 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,
1043
- 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,
1044
- 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,
1045
- 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,
1046
- 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,
1047
- 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,
1048
- 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,
1049
- 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,
1050
- 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,
1051
- 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,
1052
- 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,
1053
- 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,
1054
- 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,
1055
- 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,
1056
- 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
1057
-};
1058
-
1059
-static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};
1060
-static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
1061
-
1062
-
1063
-typedef half dfloat; // dequantize float
1064
-typedef half2 dfloat2;
1065
-typedef void (*dequantize_kernel_t)(const void * vx, const int ib, const int iqs, dfloat2 & v);
1066
-template<typename dst_t>
1067
-using to_cuda_ggml_t = void (*)(const void * __restrict__ x, dst_t * __restrict__ y, int64_t k, cudaStream_t stream);
1068
-typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs);
1069
-typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc);
1070
-typedef void (*load_tiles_cuda_t)(
1071
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1072
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row);
1073
-typedef float (*vec_dot_q_mul_mat_cuda_t)(
1074
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1075
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k);
1076
-
1077
-// Utility function
1078
-
1079
-template<typename dst_t>
1080
-static __device__ __forceinline__ dst_t convert_from_half(half val) {
1081
- return val;
1082
-}
1083
-
1084
-template<>
1085
-__device__ __forceinline__ c10::BFloat16 convert_from_half<c10::BFloat16>(half val) {
1086
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1087
- return __float2bfloat16(__half2float(val));
1088
-#else
1089
- return __half2float(val);
1090
-#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1091
-}
1092
-
1093
-template<>
1094
-__device__ __forceinline__ float convert_from_half<float>(half val) {
1095
- return __half2float(val);
1096
-}
1097
-
1098
-#if defined(USE_ROCM)
1099
-
1100
-#ifndef __has_builtin
1101
- #define __has_builtin(x) 0
1102
-#endif
1103
-
1104
-typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
1105
-static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
1106
- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
1107
- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
1108
-#if __has_builtin(__builtin_elementwise_sub_sat)
1109
- const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);
1110
- return reinterpret_cast<const int &>(c);
1111
-#else
1112
- int8x4_t c;
1113
- int16_t tmp;
1114
-#pragma unroll
1115
- for (int i = 0; i < 4; i++) {
1116
- tmp = va[i] - vb[i];
1117
- if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();
1118
- if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();
1119
- c[i] = tmp;
1120
- }
1121
- return reinterpret_cast<int &>(c);
1122
-#endif // __has_builtin(__builtin_elementwise_sub_sat)
1123
-}
1124
-
1125
-static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
1126
-#if __has_builtin(__builtin_amdgcn_sdot4)
1127
- c = __builtin_amdgcn_sdot4(a, b, c, false);
1128
-#else
1129
- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
1130
- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
1131
- c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];
1132
-#endif
1133
- return c;
1134
-}
1135
-
1136
-static __device__ __forceinline__ uint32_t __vcmpeq4(const uint32_t a, const uint32_t b) {
1137
- uint32_t neq = a^b;
1138
- return !(neq & 0xff000000) * 0xff000000 |
1139
- !(neq & 0x00ff0000) * 0x00ff0000 |
1140
- !(neq & 0x0000ff00) * 0x0000ff00 |
1141
- !(neq & 0x000000ff) * 0x000000ff;
1142
-}
1143
-
1144
-static __device__ __forceinline__ uint32_t __vsub4(const uint32_t a, const uint32_t b) {
1145
- return (static_cast<uint8_t>(((a & 0xff000000) >> 24) - ((b & 0xff000000) >> 24)) << 24) +
1146
- (static_cast<uint8_t>(((a & 0x00ff0000) >> 16) - ((b & 0x00ff0000) >> 16)) << 16) +
1147
- (static_cast<uint8_t>(((a & 0x0000ff00) >> 8) - ((b & 0x0000ff00) >> 8)) << 8) +
1148
- (static_cast<uint8_t>(((a & 0x000000ff) >> 0) - ((b & 0x000000ff) >> 0)) << 0);
1149
-}
1150
-#endif // defined(USE_ROCM)
csrc/libtorch_stable/quantization/gguf/gguf_kernel.cu
deleted
-561
@@ -1,561 +0,0 @@
1
-#include <cuda_fp16.h>
2
-#include <cuda_runtime.h>
3
-
4
-#include "../../../cuda_compat.h"
5
-#include "../../dispatch_utils.h"
6
-#include "../../torch_utils.h"
7
-
8
-#include <torch/csrc/stable/ops.h>
9
-
10
-#include "ggml-common.h"
11
-#include "vecdotq.cuh"
12
-#include "dequantize.cuh"
13
-#include "mmvq.cuh"
14
-#include "mmq.cuh"
15
-#include "moe.cuh"
16
-#include "moe_vec.cuh"
17
-
18
-// Q8 gemv
19
-template <typename scalar_t>
20
-static __global__ void quantize_q8_1(const scalar_t* __restrict__ x,
21
- void* __restrict__ vy, const int kx,
22
- const int kx_padded) {
23
- const auto ix = blockDim.x * blockIdx.x + threadIdx.x;
24
- if (ix >= kx_padded) {
25
- return;
26
- }
27
- const auto iy = blockDim.y * blockIdx.y + threadIdx.y;
28
- const int i_padded = iy * kx_padded + ix;
29
-
30
- block_q8_1* y = (block_q8_1*)vy;
31
-
32
- const int ib = i_padded / QK8_1; // block index
33
- const int iqs = i_padded % QK8_1; // quant index
34
-
35
- const float xi = ix < kx ? static_cast<float>(x[iy * kx + ix]) : 0.0f;
36
- float amax = fabsf(xi);
37
- float sum = xi;
38
-
39
-#pragma unroll
40
- for (int mask = 16; mask > 0; mask >>= 1) {
41
- amax = fmaxf(amax, VLLM_SHFL_XOR_SYNC_WIDTH(amax, mask, 32));
42
- sum += VLLM_SHFL_XOR_SYNC_WIDTH(sum, mask, 32);
43
- }
44
-
45
- const float d = amax / 127;
46
- const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
47
-
48
- y[ib].qs[iqs] = q;
49
-
50
- if (iqs > 0) {
51
- return;
52
- }
53
-
54
- y[ib].ds.x = __float2half(d);
55
- y[ib].ds.y = __float2half(sum);
56
-}
57
-
58
-template <typename scalar_t>
59
-static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx,
60
- const int ky, cudaStream_t stream) {
61
- const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
62
- const int block_num_x =
63
- (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
64
- constexpr int MAX_BLOCK_SIZE = 65535;
65
- for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) {
66
- const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off;
67
- const dim3 num_blocks(block_num_x, num_blocks_y, 1);
68
- const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
69
- quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(
70
- &x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded);
71
- }
72
-}
73
-
74
-torch::stable::Tensor ggml_dequantize(
75
- torch::stable::Tensor W, // quant weight
76
- int64_t type, int64_t m, int64_t n,
77
- std::optional<torch::headeronly::ScalarType> const& dtype) {
78
- const torch::stable::accelerator::DeviceGuard device_guard(
79
- W.get_device_index());
80
- auto dtype_ = dtype.value_or(torch::headeronly::ScalarType::Half);
81
- auto DW = torch::stable::empty({m, n}, dtype_, std::nullopt, W.device());
82
- torch::stable::fill_(DW, 0.0);
83
- cudaStream_t stream = get_current_cuda_stream();
84
-
85
- VLLM_STABLE_DISPATCH_FLOATING_TYPES(DW.scalar_type(), "ggml_dequantize", [&] {
86
- auto to_cuda = ggml_get_to_cuda<scalar_t>(type);
87
- to_cuda((void*)W.data_ptr(), (scalar_t*)DW.data_ptr(), m * n, stream);
88
- });
89
-
90
- return DW;
91
-}
92
-
93
-torch::stable::Tensor ggml_mul_mat_vec_a8(
94
- torch::stable::Tensor W, // quant weight
95
- torch::stable::Tensor X, // input
96
- int64_t type, int64_t row) {
97
- int64_t col = X.sizes()[1];
98
- int64_t vecs = X.sizes()[0];
99
- const int64_t padded = (col + 512 - 1) / 512 * 512;
100
- const torch::stable::accelerator::DeviceGuard device_guard(
101
- X.get_device_index());
102
- auto Y = torch::stable::empty({vecs, row}, X.scalar_type(), std::nullopt,
103
- W.device());
104
- torch::stable::fill_(Y, 0.0);
105
- cudaStream_t stream = get_current_cuda_stream();
106
- auto quant_X = torch::stable::empty({vecs, padded / 32 * 9},
107
- torch::headeronly::ScalarType::Int,
108
- std::nullopt, W.device());
109
- VLLM_STABLE_DISPATCH_FLOATING_TYPES(
110
- X.scalar_type(), "ggml_mul_mat_vec_a8", [&] {
111
- quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(),
112
- (void*)quant_X.data_ptr(), col, vecs,
113
- stream);
114
- switch (type) {
115
- case 2:
116
- mul_mat_vec_q4_0_q8_1_cuda<scalar_t>(
117
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
118
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
119
- break;
120
- case 3:
121
- mul_mat_vec_q4_1_q8_1_cuda<scalar_t>(
122
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
123
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
124
- break;
125
- case 6:
126
- mul_mat_vec_q5_0_q8_1_cuda<scalar_t>(
127
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
128
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
129
- break;
130
- case 7:
131
- mul_mat_vec_q5_1_q8_1_cuda<scalar_t>(
132
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
133
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
134
- break;
135
- case 8:
136
- mul_mat_vec_q8_0_q8_1_cuda<scalar_t>(
137
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
138
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
139
- break;
140
- case 10:
141
- mul_mat_vec_q2_K_q8_1_cuda<scalar_t>(
142
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
143
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
144
- break;
145
- case 11:
146
- mul_mat_vec_q3_K_q8_1_cuda<scalar_t>(
147
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
148
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
149
- break;
150
- case 12:
151
- mul_mat_vec_q4_K_q8_1_cuda<scalar_t>(
152
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
153
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
154
- break;
155
- case 13:
156
- mul_mat_vec_q5_K_q8_1_cuda<scalar_t>(
157
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
158
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
159
- break;
160
- case 14:
161
- mul_mat_vec_q6_K_q8_1_cuda<scalar_t>(
162
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
163
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
164
- break;
165
- case 16:
166
- mul_mat_vec_iq2_xxs_q8_1_cuda<scalar_t>(
167
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
168
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
169
- break;
170
- case 17:
171
- mul_mat_vec_iq2_xs_q8_1_cuda<scalar_t>(
172
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
173
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
174
- break;
175
- case 18:
176
- mul_mat_vec_iq3_xxs_q8_1_cuda<scalar_t>(
177
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
178
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
179
- break;
180
- case 19:
181
- mul_mat_vec_iq1_s_q8_1_cuda<scalar_t>(
182
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
183
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
184
- break;
185
- case 20:
186
- mul_mat_vec_iq4_nl_q8_1_cuda<scalar_t>(
187
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
188
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
189
- break;
190
- case 21:
191
- mul_mat_vec_iq3_s_q8_1_cuda<scalar_t>(
192
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
193
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
194
- break;
195
- case 22:
196
- mul_mat_vec_iq2_s_q8_1_cuda<scalar_t>(
197
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
198
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
199
- break;
200
- case 23:
201
- mul_mat_vec_iq4_xs_q8_1_cuda<scalar_t>(
202
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
203
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
204
- break;
205
- case 29:
206
- mul_mat_vec_iq1_m_q8_1_cuda<scalar_t>(
207
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
208
- (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
209
- break;
210
- }
211
- });
212
- return Y;
213
-}
214
-
215
-torch::stable::Tensor ggml_mul_mat_a8(torch::stable::Tensor W, // quant weight
216
- torch::stable::Tensor X, // input
217
- int64_t type, int64_t row) {
218
- int64_t col = X.sizes()[1];
219
- int64_t padded = (col + 512 - 1) / 512 * 512;
220
- int64_t batch = X.sizes()[0];
221
- const torch::stable::accelerator::DeviceGuard device_guard(
222
- X.get_device_index());
223
- auto Y = torch::stable::empty({batch, row}, X.scalar_type(), std::nullopt,
224
- W.device());
225
- torch::stable::fill_(Y, 0.0);
226
- cudaStream_t stream = get_current_cuda_stream();
227
- auto quant_X = torch::stable::empty({batch, padded / 32 * 9},
228
- torch::headeronly::ScalarType::Int,
229
- std::nullopt, W.device());
230
- VLLM_STABLE_DISPATCH_FLOATING_TYPES(X.scalar_type(), "ggml_mul_mat_a8", [&] {
231
- quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(),
232
- col, batch, stream);
233
-
234
- switch (type) {
235
- case 2:
236
- ggml_mul_mat_q4_0_q8_1_cuda(
237
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
238
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
239
- break;
240
- case 3:
241
- ggml_mul_mat_q4_1_q8_1_cuda(
242
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
243
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
244
- break;
245
- case 6:
246
- ggml_mul_mat_q5_0_q8_1_cuda(
247
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
248
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
249
- break;
250
- case 7:
251
- ggml_mul_mat_q5_1_q8_1_cuda(
252
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
253
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
254
- break;
255
- case 8:
256
- ggml_mul_mat_q8_0_q8_1_cuda(
257
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
258
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
259
- break;
260
- case 10:
261
- ggml_mul_mat_q2_K_q8_1_cuda(
262
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
263
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
264
- break;
265
- case 11:
266
- ggml_mul_mat_q3_K_q8_1_cuda(
267
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
268
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
269
- break;
270
- case 12:
271
- ggml_mul_mat_q4_K_q8_1_cuda(
272
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
273
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
274
- break;
275
- case 13:
276
- ggml_mul_mat_q5_K_q8_1_cuda(
277
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
278
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
279
- break;
280
- case 14:
281
- ggml_mul_mat_q6_K_q8_1_cuda(
282
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
283
- (scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
284
- break;
285
- }
286
- });
287
- return Y;
288
-}
289
-
290
-torch::stable::Tensor ggml_moe_a8(torch::stable::Tensor X, // input
291
- torch::stable::Tensor W, // expert weights
292
- torch::stable::Tensor sorted_token_ids,
293
- torch::stable::Tensor expert_ids,
294
- torch::stable::Tensor num_tokens_post_padded,
295
- int64_t type, int64_t row, int64_t top_k,
296
- int64_t tokens) {
297
- int64_t col = X.sizes()[1];
298
- int64_t padded = (col + 512 - 1) / 512 * 512;
299
- const torch::stable::accelerator::DeviceGuard device_guard(
300
- X.get_device_index());
301
- auto Y = torch::stable::empty({tokens * top_k, row}, X.scalar_type(),
302
- std::nullopt, W.device());
303
- torch::stable::fill_(Y, 0.0);
304
- cudaStream_t stream = get_current_cuda_stream();
305
- auto quant_X = torch::stable::empty({tokens, padded / 32 * 9},
306
- torch::headeronly::ScalarType::Int,
307
- std::nullopt, W.device());
308
- VLLM_STABLE_DISPATCH_FLOATING_TYPES(X.scalar_type(), "ggml_moe_a8", [&] {
309
- quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(),
310
- col, tokens, stream);
311
- switch (type) {
312
- case 2:
313
- ggml_moe_q4_0_q8_1_cuda(
314
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
315
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
316
- (int*)expert_ids.data_ptr(),
317
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
318
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
319
- break;
320
- case 3:
321
- ggml_moe_q4_1_q8_1_cuda(
322
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
323
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
324
- (int*)expert_ids.data_ptr(),
325
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
326
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
327
- break;
328
- case 6:
329
- ggml_moe_q5_0_q8_1_cuda(
330
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
331
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
332
- (int*)expert_ids.data_ptr(),
333
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
334
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
335
- break;
336
- case 7:
337
- ggml_moe_q5_1_q8_1_cuda(
338
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
339
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
340
- (int*)expert_ids.data_ptr(),
341
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
342
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
343
- break;
344
- case 8:
345
- ggml_moe_q8_0_q8_1_cuda(
346
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
347
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
348
- (int*)expert_ids.data_ptr(),
349
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
350
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
351
- break;
352
- case 10:
353
- ggml_moe_q2_K_q8_1_cuda(
354
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
355
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
356
- (int*)expert_ids.data_ptr(),
357
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
358
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
359
- break;
360
- case 11:
361
- ggml_moe_q3_K_q8_1_cuda(
362
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
363
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
364
- (int*)expert_ids.data_ptr(),
365
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
366
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
367
- break;
368
- case 12:
369
- ggml_moe_q4_K_q8_1_cuda(
370
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
371
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
372
- (int*)expert_ids.data_ptr(),
373
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
374
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
375
- break;
376
- case 13:
377
- ggml_moe_q5_K_q8_1_cuda(
378
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
379
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
380
- (int*)expert_ids.data_ptr(),
381
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
382
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
383
- break;
384
- case 14:
385
- ggml_moe_q6_K_q8_1_cuda(
386
- (void*)quant_X.data_ptr(), (void*)W.data_ptr(),
387
- (scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
388
- (int*)expert_ids.data_ptr(),
389
- (int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
390
- tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
391
- break;
392
- }
393
- });
394
- return Y;
395
-}
396
-
397
-torch::stable::Tensor ggml_moe_a8_vec(
398
- torch::stable::Tensor X, // input
399
- torch::stable::Tensor W, // expert weights
400
- torch::stable::Tensor topk_ids, int64_t top_k, int64_t type, int64_t row,
401
- int64_t tokens) {
402
- int64_t col = X.sizes()[1];
403
- const int64_t padded = (col + 512 - 1) / 512 * 512;
404
- const torch::stable::accelerator::DeviceGuard device_guard(
405
- X.get_device_index());
406
- auto Y = torch::stable::empty({tokens * top_k, row}, X.scalar_type(),
407
- std::nullopt, W.device());
408
- torch::stable::fill_(Y, 0.0);
409
- cudaStream_t stream = get_current_cuda_stream();
410
- auto quant_X = torch::stable::empty({tokens, padded / 32 * 9},
411
- torch::headeronly::ScalarType::Int,
412
- std::nullopt, W.device());
413
- VLLM_STABLE_DISPATCH_FLOATING_TYPES(X.scalar_type(), "ggml_moe_vec_a8", [&] {
414
- quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(),
415
- (void*)quant_X.data_ptr(), col, tokens,
416
- stream);
417
- switch (type) {
418
- case 2:
419
- moe_vec_q4_0_q8_1_cuda<scalar_t>(
420
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
421
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
422
- col, row, quant_X.stride(0), stream);
423
- break;
424
- case 3:
425
- moe_vec_q4_1_q8_1_cuda<scalar_t>(
426
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
427
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
428
- col, row, quant_X.stride(0), stream);
429
- break;
430
- case 6:
431
- moe_vec_q5_0_q8_1_cuda<scalar_t>(
432
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
433
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
434
- col, row, quant_X.stride(0), stream);
435
- break;
436
- case 7:
437
- moe_vec_q5_1_q8_1_cuda<scalar_t>(
438
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
439
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
440
- col, row, quant_X.stride(0), stream);
441
- break;
442
- case 8:
443
- moe_vec_q8_0_q8_1_cuda<scalar_t>(
444
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
445
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
446
- col, row, quant_X.stride(0), stream);
447
- break;
448
- case 10:
449
- moe_vec_q2_K_q8_1_cuda<scalar_t>(
450
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
451
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
452
- col, row, quant_X.stride(0), stream);
453
- break;
454
- case 11:
455
- moe_vec_q3_K_q8_1_cuda<scalar_t>(
456
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
457
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
458
- col, row, quant_X.stride(0), stream);
459
- break;
460
- case 12:
461
- moe_vec_q4_K_q8_1_cuda<scalar_t>(
462
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
463
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
464
- col, row, quant_X.stride(0), stream);
465
- break;
466
- case 13:
467
- moe_vec_q5_K_q8_1_cuda<scalar_t>(
468
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
469
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
470
- col, row, quant_X.stride(0), stream);
471
- break;
472
- case 14:
473
- moe_vec_q6_K_q8_1_cuda<scalar_t>(
474
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
475
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
476
- col, row, quant_X.stride(0), stream);
477
- break;
478
- case 16:
479
- moe_vec_iq2_xxs_q8_1_cuda<scalar_t>(
480
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
481
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
482
- col, row, quant_X.stride(0), stream);
483
- break;
484
- case 17:
485
- moe_vec_iq2_xs_q8_1_cuda<scalar_t>(
486
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
487
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
488
- col, row, quant_X.stride(0), stream);
489
- break;
490
- case 18:
491
- moe_vec_iq3_xxs_q8_1_cuda<scalar_t>(
492
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
493
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
494
- col, row, quant_X.stride(0), stream);
495
- break;
496
- case 19:
497
- moe_vec_iq1_s_q8_1_cuda<scalar_t>(
498
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
499
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
500
- col, row, quant_X.stride(0), stream);
501
- break;
502
- case 20:
503
- moe_vec_iq4_nl_q8_1_cuda<scalar_t>(
504
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
505
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
506
- col, row, quant_X.stride(0), stream);
507
- break;
508
- case 21:
509
- moe_vec_iq3_s_q8_1_cuda<scalar_t>(
510
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
511
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
512
- col, row, quant_X.stride(0), stream);
513
- break;
514
- case 22:
515
- moe_vec_iq2_s_q8_1_cuda<scalar_t>(
516
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
517
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
518
- col, row, quant_X.stride(0), stream);
519
- break;
520
- case 23:
521
- moe_vec_iq4_xs_q8_1_cuda<scalar_t>(
522
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
523
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
524
- col, row, quant_X.stride(0), stream);
525
- break;
526
- case 29:
527
- moe_vec_iq1_m_q8_1_cuda<scalar_t>(
528
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(),
529
- (scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
530
- col, row, quant_X.stride(0), stream);
531
- break;
532
- }
533
- });
534
- return Y;
535
-}
536
-
537
-int64_t ggml_moe_get_block_size(int64_t type) {
538
- switch (type) {
539
- case 2:
540
- return MOE_X_Q4_0;
541
- case 3:
542
- return MOE_X_Q4_1;
543
- case 6:
544
- return MOE_X_Q5_0;
545
- case 7:
546
- return MOE_X_Q5_1;
547
- case 8:
548
- return MOE_X_Q8_0;
549
- case 10:
550
- return MOE_X_Q2_K;
551
- case 11:
552
- return MOE_X_Q3_K;
553
- case 12:
554
- return MOE_X_Q4_K;
555
- case 13:
556
- return MOE_X_Q5_K;
557
- case 14:
558
- return MOE_X_Q6_K;
559
- }
560
- return 0;
561
-}
csrc/libtorch_stable/quantization/gguf/mmq.cuh
deleted
-610
@@ -1,610 +0,0 @@
1
-// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
2
-template <typename scalar_t, int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,
3
- allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>
4
-static __device__ __forceinline__ void mul_mat_q(
5
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
6
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
7
-
8
- const block_q_t * x = (const block_q_t *) vx;
9
- const block_q8_1 * y = (const block_q8_1 *) vy;
10
-
11
- const int blocks_per_row_x = ncols_x / qk;
12
- const int blocks_per_col_y = nrows_y / QK8_1;
13
- const int blocks_per_warp = WARP_SIZE_GGUF / qi;
14
-
15
- const int & ncols_dst = ncols_y;
16
-
17
- const auto row_dst_0 = blockIdx.x*mmq_y;
18
- const int & row_x_0 = row_dst_0;
19
-
20
- const auto col_dst_0 = blockIdx.y*mmq_x;
21
- const int & col_y_0 = col_dst_0;
22
-
23
- int * tile_x_ql = nullptr;
24
- half2 * tile_x_dm = nullptr;
25
- int * tile_x_qh = nullptr;
26
- int * tile_x_sc = nullptr;
27
-
28
- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
29
-
30
- __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
31
- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF/QI8_1];
32
-
33
- float sum[mmq_y/WARP_SIZE_GGUF][mmq_x/nwarps] = {{0.0f}};
34
-
35
- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
36
-
37
- load_tiles(x + row_x_0*blocks_per_row_x + ib0, tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc,
38
- threadIdx.y, nrows_x-row_x_0-1, threadIdx.x, blocks_per_row_x);
39
-
40
-#pragma unroll
41
- for (int ir = 0; ir < qr && ib0 + ir * blocks_per_warp/qr < blocks_per_row_x; ++ir) {
42
- const auto kqs = ir*WARP_SIZE_GGUF + threadIdx.x;
43
- const int kbxd = kqs / QI8_1;
44
-
45
-#pragma unroll
46
- for (int i = 0; i < mmq_x; i += nwarps) {
47
- const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses
48
- const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];
49
- const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
50
- tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
51
- }
52
-
53
-#pragma unroll
54
- for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
55
- const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF/QI8_1)) % mmq_x;
56
- const auto kby = threadIdx.x % (WARP_SIZE_GGUF/QI8_1);
57
- const int col_y_eff = min(col_y_0 + ids, ncols_y-1);
58
-
59
- // if the sum is not needed it's faster to transform the scale to f32 ahead of time
60
- const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE_GGUF/QI8_1) + kby].ds;
61
- half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF/QI8_1) + kby];
62
- if (need_sum) {
63
- *dsi_dst = *dsi_src;
64
- } else {
65
- float * dfi_dst = (float *) dsi_dst;
66
- *dfi_dst = __low2float(*dsi_src);
67
- }
68
- }
69
-
70
- __syncthreads();
71
-
72
-// #pragma unroll // unrolling this loop causes too much register pressure
73
- for (int k = ir*WARP_SIZE_GGUF/qr; k < (ir+1)*WARP_SIZE_GGUF/qr; k += vdr) {
74
-#pragma unroll
75
- for (int j = 0; j < mmq_x; j += nwarps) {
76
-#pragma unroll
77
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
78
- sum[i/WARP_SIZE_GGUF][j/nwarps] += vec_dot(
79
- tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,
80
- threadIdx.x + i, threadIdx.y + j, k);
81
- }
82
- }
83
- }
84
- __syncthreads();
85
- }
86
- }
87
-
88
-#pragma unroll
89
- for (int j = 0; j < mmq_x; j += nwarps) {
90
- const auto col_dst = col_dst_0 + j + threadIdx.y;
91
- if (col_dst >= ncols_dst) {
92
- return;
93
- }
94
-
95
-#pragma unroll
96
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
97
- const auto row_dst = row_dst_0 + threadIdx.x + i;
98
- if (row_dst >= nrows_dst) {
99
- continue;
100
- }
101
- dst[col_dst*nrows_dst + row_dst] = sum[i/WARP_SIZE_GGUF][j/nwarps];
102
- }
103
- }
104
-}
105
-
106
-#if defined(USE_ROCM)
107
-#define MMQ_X_Q4_0 64
108
-#define MMQ_Y_Q4_0 128
109
-#define NWARPS_Q4_0 8
110
-#else
111
-#define MMQ_X_Q4_0 4
112
-#define MMQ_Y_Q4_0 32
113
-#define NWARPS_Q4_0 4
114
-#endif
115
-
116
-template<typename scalar_t, bool need_check> static __global__ void
117
-#if defined(USE_ROCM)
118
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_0, 2)
119
-#endif
120
-mul_mat_q4_0(
121
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
122
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
123
- const int mmq_x = MMQ_X_Q4_0;
124
- const int mmq_y = MMQ_Y_Q4_0;
125
- const int nwarps = NWARPS_Q4_0;
126
-
127
- mul_mat_q<scalar_t, QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,
128
- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>
129
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
130
-}
131
-
132
-template<typename scalar_t>
133
-static void ggml_mul_mat_q4_0_q8_1_cuda(
134
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
135
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
136
-
137
- int mmq_x = MMQ_X_Q4_0;
138
- int mmq_y = MMQ_Y_Q4_0;
139
- int nwarps = NWARPS_Q4_0;
140
-
141
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
142
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
143
- const dim3 block_nums(block_num_x, block_num_y, 1);
144
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
145
-
146
- if (nrows_x % mmq_y == 0) {
147
- const bool need_check = false;
148
- mul_mat_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
149
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
150
- } else {
151
- const bool need_check = true;
152
- mul_mat_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
153
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
154
- }
155
-}
156
-
157
-#if defined(USE_ROCM)
158
-#define MMQ_X_Q4_1 64
159
-#define MMQ_Y_Q4_1 128
160
-#define NWARPS_Q4_1 8
161
-#else
162
-#define MMQ_X_Q4_1 4
163
-#define MMQ_Y_Q4_1 32
164
-#define NWARPS_Q4_1 4
165
-#endif
166
-
167
-template<typename scalar_t, bool need_check> static __global__ void
168
-#if defined(USE_ROCM)
169
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_1, 2)
170
-#endif
171
-mul_mat_q4_1(
172
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
173
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
174
- const int mmq_x = MMQ_X_Q4_1;
175
- const int mmq_y = MMQ_Y_Q4_1;
176
- const int nwarps = NWARPS_Q4_1;
177
-
178
- mul_mat_q<scalar_t, QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,
179
- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>
180
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
181
-}
182
-
183
-template<typename scalar_t>
184
-static void ggml_mul_mat_q4_1_q8_1_cuda(
185
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
186
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
187
-
188
- int mmq_x = MMQ_X_Q4_1;
189
- int mmq_y = MMQ_Y_Q4_1;
190
- int nwarps = NWARPS_Q4_1;
191
-
192
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
193
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
194
- const dim3 block_nums(block_num_x, block_num_y, 1);
195
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
196
-
197
- if (nrows_x % mmq_y == 0) {
198
- const bool need_check = false;
199
- mul_mat_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
200
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
201
- } else {
202
- const bool need_check = true;
203
- mul_mat_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
204
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
205
- }
206
-}
207
-
208
-#if defined(USE_ROCM)
209
-#define MMQ_X_Q5_0 64
210
-#define MMQ_Y_Q5_0 128
211
-#define NWARPS_Q5_0 8
212
-#else
213
-#define MMQ_X_Q5_0 4
214
-#define MMQ_Y_Q5_0 32
215
-#define NWARPS_Q5_0 4
216
-#endif
217
-
218
-template<typename scalar_t, bool need_check> static __global__ void
219
-#if defined(USE_ROCM)
220
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_0, 2)
221
-#endif
222
-mul_mat_q5_0(
223
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
224
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
225
- const int mmq_x = MMQ_X_Q5_0;
226
- const int mmq_y = MMQ_Y_Q5_0;
227
- const int nwarps = NWARPS_Q5_0;
228
-
229
- mul_mat_q<scalar_t, QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,
230
- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>
231
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
232
-}
233
-
234
-template<typename scalar_t>
235
-static void ggml_mul_mat_q5_0_q8_1_cuda(
236
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
237
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
238
-
239
- const int mmq_x = MMQ_X_Q5_0;
240
- const int mmq_y = MMQ_Y_Q5_0;
241
- const int nwarps = NWARPS_Q5_0;
242
-
243
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
244
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
245
- const dim3 block_nums(block_num_x, block_num_y, 1);
246
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
247
-
248
- if (nrows_x % mmq_y == 0) {
249
- const bool need_check = false;
250
- mul_mat_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
251
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
252
- } else {
253
- const bool need_check = true;
254
- mul_mat_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
255
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
256
- }
257
-}
258
-
259
-#if defined(USE_ROCM)
260
-#define MMQ_X_Q5_1 64
261
-#define MMQ_Y_Q5_1 128
262
-#define NWARPS_Q5_1 8
263
-#else
264
-#define MMQ_X_Q5_1 4
265
-#define MMQ_Y_Q5_1 32
266
-#define NWARPS_Q5_1 4
267
-#endif
268
-
269
-template<typename scalar_t, bool need_check> static __global__ void
270
-#if defined(USE_ROCM)
271
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_1, 2)
272
-#endif
273
-mul_mat_q5_1(
274
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
275
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
276
- const int mmq_x = MMQ_X_Q5_1;
277
- const int mmq_y = MMQ_Y_Q5_1;
278
- const int nwarps = NWARPS_Q5_1;
279
-
280
- mul_mat_q<scalar_t, QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,
281
- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>
282
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
283
-}
284
-
285
-template<typename scalar_t>
286
-static void ggml_mul_mat_q5_1_q8_1_cuda(
287
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
288
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
289
- const int mmq_x = MMQ_X_Q5_1;
290
- const int mmq_y = MMQ_Y_Q5_1;
291
- const int nwarps = NWARPS_Q5_1;
292
-
293
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
294
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
295
- const dim3 block_nums(block_num_x, block_num_y, 1);
296
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
297
-
298
- if (nrows_x % mmq_y == 0) {
299
- const bool need_check = false;
300
- mul_mat_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
301
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
302
- } else {
303
- const bool need_check = true;
304
- mul_mat_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
305
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
306
- }
307
-}
308
-
309
-#if defined(USE_ROCM)
310
-#define MMQ_X_Q8_0 64
311
-#define MMQ_Y_Q8_0 128
312
-#define NWARPS_Q8_0 8
313
-#else
314
-#define MMQ_X_Q8_0 4
315
-#define MMQ_Y_Q8_0 32
316
-#define NWARPS_Q8_0 4
317
-#endif
318
-
319
-template<typename scalar_t, bool need_check> static __global__ void
320
-#if defined(USE_ROCM)
321
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q8_0, 2)
322
-#endif
323
-mul_mat_q8_0(
324
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
325
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
326
- const int mmq_x = MMQ_X_Q8_0;
327
- const int mmq_y = MMQ_Y_Q8_0;
328
- const int nwarps = NWARPS_Q8_0;
329
-
330
- mul_mat_q<scalar_t, QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,
331
- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>
332
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
333
-}
334
-
335
-template<typename scalar_t>
336
-static void ggml_mul_mat_q8_0_q8_1_cuda(
337
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
338
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
339
- const int mmq_x = MMQ_X_Q8_0;
340
- const int mmq_y = MMQ_Y_Q8_0;
341
- const int nwarps = NWARPS_Q8_0;
342
-
343
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
344
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
345
- const dim3 block_nums(block_num_x, block_num_y, 1);
346
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
347
-
348
- if (nrows_x % mmq_y == 0) {
349
- const bool need_check = false;
350
- mul_mat_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
351
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
352
- } else {
353
- const bool need_check = true;
354
- mul_mat_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
355
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
356
- }
357
-}
358
-
359
-#if defined(USE_ROCM)
360
-#define MMQ_X_Q2_K 64
361
-#define MMQ_Y_Q2_K 128
362
-#define NWARPS_Q2_K 8
363
-#else
364
-#define MMQ_X_Q2_K 4
365
-#define MMQ_Y_Q2_K 32
366
-#define NWARPS_Q2_K 4
367
-#endif
368
-
369
-template<typename scalar_t, bool need_check> static __global__ void
370
-#if defined(USE_ROCM)
371
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q2_K, 2)
372
-#endif
373
-mul_mat_q2_K(
374
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
375
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
376
- const int mmq_x = MMQ_X_Q2_K;
377
- const int mmq_y = MMQ_Y_Q2_K;
378
- const int nwarps = NWARPS_Q2_K;
379
-
380
- mul_mat_q<scalar_t, QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,
381
- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>
382
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
383
-}
384
-
385
-template<typename scalar_t>
386
-static void ggml_mul_mat_q2_K_q8_1_cuda(
387
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
388
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
389
- const int mmq_x = MMQ_X_Q2_K;
390
- const int mmq_y = MMQ_Y_Q2_K;
391
- const int nwarps = NWARPS_Q2_K;
392
-
393
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
394
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
395
- const dim3 block_nums(block_num_x, block_num_y, 1);
396
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
397
-
398
- if (nrows_x % mmq_y == 0) {
399
- const bool need_check = false;
400
- mul_mat_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
401
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
402
- } else {
403
- const bool need_check = true;
404
- mul_mat_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
405
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
406
- }
407
-}
408
-
409
-#if defined(USE_ROCM)
410
-#define MMQ_X_Q3_K 64
411
-#define MMQ_Y_Q3_K 128
412
-#define NWARPS_Q3_K 8
413
-#else
414
-#define MMQ_X_Q3_K 4
415
-#define MMQ_Y_Q3_K 32
416
-#define NWARPS_Q3_K 4
417
-#endif
418
-
419
-template<typename scalar_t, bool need_check> static __global__ void
420
-#if defined(USE_ROCM)
421
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q3_K, 2)
422
-#endif
423
-mul_mat_q3_K(
424
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
425
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
426
-
427
- const int mmq_x = MMQ_X_Q3_K;
428
- const int mmq_y = MMQ_Y_Q3_K;
429
- const int nwarps = NWARPS_Q3_K;
430
-
431
- mul_mat_q<scalar_t, QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,
432
- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>
433
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
434
-}
435
-
436
-template<typename scalar_t>
437
-static void ggml_mul_mat_q3_K_q8_1_cuda(
438
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
439
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
440
-
441
- const int mmq_x = MMQ_X_Q3_K;
442
- const int mmq_y = MMQ_Y_Q3_K;
443
- const int nwarps = NWARPS_Q3_K;
444
-
445
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
446
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
447
- const dim3 block_nums(block_num_x, block_num_y, 1);
448
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
449
-
450
- if (nrows_x % mmq_y == 0) {
451
- const bool need_check = false;
452
- mul_mat_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
453
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
454
- } else {
455
- const bool need_check = true;
456
- mul_mat_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
457
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
458
- }
459
-}
460
-
461
-#if defined(USE_ROCM)
462
-#define MMQ_X_Q4_K 64
463
-#define MMQ_Y_Q4_K 128
464
-#define NWARPS_Q4_K 8
465
-#else
466
-#define MMQ_X_Q4_K 4
467
-#define MMQ_Y_Q4_K 32
468
-#define NWARPS_Q4_K 4
469
-#endif
470
-
471
-template<typename scalar_t, bool need_check> static __global__ void
472
-#if defined(USE_ROCM)
473
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_K, 2)
474
-#endif
475
-mul_mat_q4_K(
476
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
477
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
478
- const int mmq_x = MMQ_X_Q4_K;
479
- const int mmq_y = MMQ_Y_Q4_K;
480
- const int nwarps = NWARPS_Q4_K;
481
-
482
- mul_mat_q<scalar_t, QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,
483
- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>
484
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
485
-}
486
-
487
-template<typename scalar_t>
488
-static void ggml_mul_mat_q4_K_q8_1_cuda(
489
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
490
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
491
- const int mmq_x = MMQ_X_Q4_K;
492
- const int mmq_y = MMQ_Y_Q4_K;
493
- const int nwarps = NWARPS_Q4_K;
494
-
495
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
496
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
497
- const dim3 block_nums(block_num_x, block_num_y, 1);
498
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
499
-
500
- if (nrows_x % mmq_y == 0) {
501
- const bool need_check = false;
502
- mul_mat_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
503
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
504
- } else {
505
- const bool need_check = true;
506
- mul_mat_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
507
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
508
- }
509
-}
510
-
511
-#if defined(USE_ROCM)
512
-#define MMQ_X_Q5_K 64
513
-#define MMQ_Y_Q5_K 128
514
-#define NWARPS_Q5_K 8
515
-#else
516
-#define MMQ_X_Q5_K 4
517
-#define MMQ_Y_Q5_K 32
518
-#define NWARPS_Q5_K 4
519
-#endif
520
-
521
-template<typename scalar_t, bool need_check> static __global__ void
522
-#if defined(USE_ROCM)
523
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_K, 2)
524
-#endif
525
-mul_mat_q5_K(
526
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
527
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
528
- const int mmq_x = MMQ_X_Q5_K;
529
- const int mmq_y = MMQ_Y_Q5_K;
530
- const int nwarps = NWARPS_Q5_K;
531
-
532
- mul_mat_q<scalar_t, QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,
533
- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>
534
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
535
-}
536
-
537
-template<typename scalar_t>
538
-static void ggml_mul_mat_q5_K_q8_1_cuda(
539
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
540
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
541
-
542
- const int mmq_x = MMQ_X_Q5_K;
543
- const int mmq_y = MMQ_Y_Q5_K;
544
- const int nwarps = NWARPS_Q5_K;
545
-
546
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
547
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
548
- const dim3 block_nums(block_num_x, block_num_y, 1);
549
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
550
-
551
- if (nrows_x % mmq_y == 0) {
552
- const bool need_check = false;
553
- mul_mat_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
554
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
555
- } else {
556
- const bool need_check = true;
557
- mul_mat_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
558
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
559
- }
560
-}
561
-
562
-#if defined(USE_ROCM)
563
-#define MMQ_X_Q6_K 64
564
-#define MMQ_Y_Q6_K 128
565
-#define NWARPS_Q6_K 8
566
-#else
567
-#define MMQ_X_Q6_K 4
568
-#define MMQ_Y_Q6_K 32
569
-#define NWARPS_Q6_K 4
570
-#endif
571
-
572
-template<typename scalar_t, bool need_check> static __global__ void
573
-#if defined(USE_ROCM)
574
-__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q6_K, 2)
575
-#endif
576
-mul_mat_q6_K(
577
- const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
578
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
579
- const int mmq_x = MMQ_X_Q6_K;
580
- const int mmq_y = MMQ_Y_Q6_K;
581
- const int nwarps = NWARPS_Q6_K;
582
-
583
- mul_mat_q<scalar_t, QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,
584
- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>
585
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
586
-}
587
-
588
-template<typename scalar_t>
589
-static void ggml_mul_mat_q6_K_q8_1_cuda(
590
- const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
591
- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
592
- const int mmq_x = MMQ_X_Q6_K;
593
- const int mmq_y = MMQ_Y_Q6_K;
594
- const int nwarps = NWARPS_Q6_K;
595
-
596
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
597
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
598
- const dim3 block_nums(block_num_x, block_num_y, 1);
599
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
600
-
601
- if (nrows_x % mmq_y == 0) {
602
- const bool need_check = false;
603
- mul_mat_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
604
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
605
- } else {
606
- const bool need_check = true;
607
- mul_mat_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
608
- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
609
- }
610
-}
csrc/libtorch_stable/quantization/gguf/mmvq.cuh
deleted
-212
@@ -1,212 +0,0 @@
1
-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
2
-template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>
3
-static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst, const int ncols, const int nrows, const int nvecs) {
4
- const auto row = blockIdx.x*blockDim.y + threadIdx.y;
5
- const auto vec = blockIdx.y;
6
-
7
- if (row >= nrows || vec >= nvecs) {
8
- return;
9
- }
10
-
11
- const int blocks_per_row = ncols / qk;
12
- const int blocks_per_warp = vdr * WARP_SIZE / qi;
13
- const int nrows_y = (ncols + 512 - 1) / 512 * 512;
14
-
15
-
16
- // partial sum for each thread
17
- float tmp = 0.0f;
18
-
19
- const block_q_t * x = (const block_q_t *) vx;
20
- const block_q8_1 * y = (const block_q8_1 *) vy;
21
-
22
- for (auto i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) {
23
- const int ibx = row*blocks_per_row + i; // x block index
24
-
25
- const int iby = vec*(nrows_y/QK8_1) + i * (qk/QK8_1); // y block index that aligns with ibx
26
-
27
- const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int
28
-
29
- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
30
- }
31
-
32
- // sum up partial sums and write back result
33
-#pragma unroll
34
- for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1) {
35
- tmp += VLLM_SHFL_XOR_SYNC(tmp, mask);
36
- }
37
-
38
- if (threadIdx.x == 0) {
39
- dst[vec*nrows + row] = tmp;
40
- }
41
-}
42
-
43
-template<typename scalar_t>
44
-static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
45
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
46
- const dim3 block_nums(block_num_y, nvecs, 1);
47
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
48
- mul_mat_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>
49
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
50
-}
51
-
52
-template<typename scalar_t>
53
-static void mul_mat_vec_q4_1_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
54
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
55
- const dim3 block_nums(block_num_y, nvecs, 1);
56
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
57
- mul_mat_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>
58
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
59
-}
60
-
61
-template<typename scalar_t>
62
-static void mul_mat_vec_q5_0_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
63
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
64
- const dim3 block_nums(block_num_y, nvecs, 1);
65
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
66
- mul_mat_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>
67
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
68
-}
69
-
70
-template<typename scalar_t>
71
-static void mul_mat_vec_q5_1_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
72
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
73
- const dim3 block_nums(block_num_y, nvecs, 1);
74
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
75
- mul_mat_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>
76
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
77
-}
78
-
79
-template<typename scalar_t>
80
-static void mul_mat_vec_q8_0_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
81
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
82
- const dim3 block_nums(block_num_y, nvecs, 1);
83
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
84
- mul_mat_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>
85
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
86
-}
87
-
88
-template<typename scalar_t>
89
-static void mul_mat_vec_q2_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
90
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
91
- const dim3 block_nums(block_num_y, nvecs, 1);
92
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
93
- mul_mat_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>
94
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
95
-}
96
-
97
-template<typename scalar_t>
98
-static void mul_mat_vec_q3_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
99
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
100
- const dim3 block_nums(block_num_y, nvecs, 1);
101
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
102
- mul_mat_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>
103
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
104
-}
105
-
106
-template<typename scalar_t>
107
-static void mul_mat_vec_q4_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
108
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
109
- const dim3 block_nums(block_num_y, nvecs, 1);
110
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
111
- mul_mat_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>
112
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
113
-}
114
-
115
-template<typename scalar_t>
116
-static void mul_mat_vec_q5_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
117
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
118
- const dim3 block_nums(block_num_y, nvecs, 1);
119
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
120
- mul_mat_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>
121
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
122
-}
123
-
124
-template<typename scalar_t>
125
-static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
126
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
127
- const dim3 block_nums(block_num_y, nvecs, 1);
128
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
129
- mul_mat_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>
130
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
131
-}
132
-
133
-template<typename scalar_t>
134
-static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
135
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
136
- const dim3 block_nums(block_num_y, nvecs, 1);
137
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
138
- mul_mat_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
139
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
140
-}
141
-
142
-template<typename scalar_t>
143
-static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
144
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
145
- const dim3 block_nums(block_num_y, nvecs, 1);
146
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
147
- mul_mat_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
148
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
149
-}
150
-
151
-template<typename scalar_t>
152
-static void mul_mat_vec_iq2_s_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
153
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
154
- const dim3 block_nums(block_num_y, nvecs, 1);
155
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
156
- mul_mat_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
157
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
158
-}
159
-
160
-template<typename scalar_t>
161
-static void mul_mat_vec_iq3_xxs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
162
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
163
- const dim3 block_nums(block_num_y, nvecs, 1);
164
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
165
- mul_mat_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
166
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
167
-}
168
-
169
-template<typename scalar_t>
170
-static void mul_mat_vec_iq1_s_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
171
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
172
- const dim3 block_nums(block_num_y, nvecs, 1);
173
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
174
- mul_mat_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
175
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
176
-}
177
-
178
-template<typename scalar_t>
179
-static void mul_mat_vec_iq1_m_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
180
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
181
- const dim3 block_nums(block_num_y, nvecs, 1);
182
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
183
- mul_mat_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>
184
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
185
-}
186
-
187
-template<typename scalar_t>
188
-static void mul_mat_vec_iq4_nl_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
189
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
190
- const dim3 block_nums(block_num_y, nvecs, 1);
191
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
192
- mul_mat_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>
193
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
194
-}
195
-
196
-template<typename scalar_t>
197
-static void mul_mat_vec_iq4_xs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
198
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
199
- const dim3 block_nums(block_num_y, nvecs, 1);
200
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
201
- mul_mat_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
202
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
203
-}
204
-
205
-template<typename scalar_t>
206
-static void mul_mat_vec_iq3_s_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
207
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
208
- const dim3 block_nums(block_num_y, nvecs, 1);
209
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
210
- mul_mat_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
211
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
212
-}
csrc/libtorch_stable/quantization/gguf/moe.cuh
deleted
-739
@@ -1,739 +0,0 @@
1
-#include <cstdint>
2
-
3
-/* Adapted from ./csrc/quantization/gguf/mmq.cuh
4
- based on ./vllm/model_executor/layers/fused_moe/experts/triton_moe.py */
5
-template <typename scalar_t, int qk, int qr, int qi, bool need_sum,
6
- typename block_q_t, int mmq_x, int mmq_y, int nwarps,
7
- allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles,
8
- int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>
9
-static __device__ __forceinline__ void moe_q(
10
- const void* __restrict__ vx, const void* __restrict__ vy,
11
- scalar_t* __restrict__ dst, const int* __restrict__ sorted_token_ids,
12
- const int* __restrict__ expert_ids,
13
- const int* __restrict__ num_tokens_post_padded, const int exp_stride,
14
- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y,
15
- const int nrows_dst, const int top_k) {
16
- const int blocks_per_row_x = ncols_x / qk;
17
- const int blocks_per_col_y = nrows_y / QK8_1;
18
- const int blocks_per_warp = WARP_SIZE_GGUF / qi;
19
-
20
- const int ncols_dst = ncols_y * top_k;
21
-
22
- const auto row_dst_0 = blockIdx.x * mmq_y;
23
- const int& row_x_0 = row_dst_0;
24
-
25
- const auto col_dst_0 = blockIdx.y * mmq_x;
26
-
27
- int token_offs[mmq_x / nwarps];
28
- for (int i = 0; i < mmq_x; i += nwarps) {
29
- token_offs[i / nwarps] = sorted_token_ids[col_dst_0 + threadIdx.y + i];
30
- }
31
-
32
- const int exp_idx = expert_ids[blockIdx.y];
33
- if (exp_idx > 255 || exp_idx < 0) return;
34
- if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
35
-
36
- const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
37
- const block_q8_1* y = (const block_q8_1*)(vy);
38
-
39
- int* tile_x_ql = nullptr;
40
- half2* tile_x_dm = nullptr;
41
- int* tile_x_qh = nullptr;
42
- int* tile_x_sc = nullptr;
43
-
44
- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
45
-
46
- __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
47
- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1];
48
-
49
- float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}};
50
-
51
- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
52
- load_tiles(x + row_x_0 * blocks_per_row_x + ib0, tile_x_ql, tile_x_dm,
53
- tile_x_qh, tile_x_sc, threadIdx.y, nrows_x - row_x_0 - 1,
54
- threadIdx.x, blocks_per_row_x);
55
-
56
- const int n_per_r = ((qk * blocks_per_warp) / qr);
57
-#pragma unroll
58
- for (int ir = 0; ir < qr && ib0 * qk + ir * n_per_r < ncols_x; ++ir) {
59
- const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x;
60
- const int kbxd = kqs / QI8_1;
61
-
62
-#pragma unroll
63
- for (int i = 0; i < mmq_x; i += nwarps) {
64
- const int col_y_eff = token_offs[i / nwarps] / top_k;
65
- const int block_x = ib0 * (qk / QK8_1) + kbxd;
66
- if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
67
- const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + block_x];
68
- const int index_y =
69
- (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
70
- tile_y_qs[index_y] =
71
- get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
72
- }
73
- }
74
-
75
- if (threadIdx.x < n_per_r / QK8_1) {
76
- const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);
77
- const int col_y_eff = token_offs[threadIdx.y] / top_k;
78
- const int block_x =
79
- ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby;
80
-
81
- if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
82
- const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds;
83
- half2* dsi_dst =
84
- &tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby];
85
-
86
- if (need_sum) {
87
- *dsi_dst = *dsi_src;
88
- } else {
89
- float* dfi_dst = (float*)dsi_dst;
90
- *dfi_dst = __low2float(*dsi_src);
91
- }
92
- }
93
- }
94
- __syncthreads();
95
-
96
- // #pragma unroll // unrolling this loop causes too much register pressure
97
- for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr;
98
- k += vdr) {
99
-#pragma unroll
100
- for (int j = 0; j < mmq_x; j += nwarps) {
101
-#pragma unroll
102
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
103
- sum[i / WARP_SIZE_GGUF][j / nwarps] +=
104
- vec_dot(tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs,
105
- tile_y_ds, threadIdx.x + i, threadIdx.y + j, k);
106
- }
107
- }
108
- }
109
- __syncthreads();
110
- }
111
- }
112
-
113
-#pragma unroll
114
- for (int j = 0; j < mmq_x; j += nwarps) {
115
- const int col_dst = token_offs[j / nwarps];
116
- if (col_dst >= ncols_dst) {
117
- return;
118
- }
119
-
120
-#pragma unroll
121
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
122
- const auto row_dst = row_dst_0 + threadIdx.x + i;
123
- if (row_dst >= nrows_dst) {
124
- continue;
125
- }
126
- dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps];
127
- }
128
- }
129
-}
130
-
131
-#if defined(USE_ROCM)
132
- #define MOE_X_Q4_0 8
133
- #define MOE_Y_Q4_0 128
134
- #define NWARPS_Q4_0 8
135
-#else
136
- #define MOE_X_Q4_0 4
137
- #define MOE_Y_Q4_0 32
138
- #define NWARPS_Q4_0 4
139
-#endif
140
-
141
-template <typename scalar_t, bool need_check>
142
-static __global__ void
143
-#if defined(USE_ROCM)
144
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2)
145
-#endif
146
- moe_q4_0(const void* __restrict__ vx, const void* __restrict__ vy,
147
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
148
- const int* expert_ids, const int* num_tokens_post_padded,
149
- const int exp_stride, const int ncols_x, const int nrows_x,
150
- const int ncols_y, const int nrows_y, const int nrows_dst,
151
- const int top_k) {
152
- const int mmq_x = MOE_X_Q4_0;
153
- const int mmq_y = MOE_Y_Q4_0;
154
- const int nwarps = NWARPS_Q4_0;
155
-
156
- moe_q<scalar_t, QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps,
157
- allocate_tiles_q4_0<mmq_y>, load_tiles_q4_0<mmq_y, nwarps, need_check>,
158
- VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>(
159
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
160
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
161
-}
162
-
163
-template <typename scalar_t>
164
-static void ggml_moe_q4_0_q8_1_cuda(
165
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
166
- const int* expert_ids, const int* num_tokens_post_padded,
167
- const int exp_stride, const int ncols_x, const int nrows_x,
168
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
169
- const int tokens_post_padded, cudaStream_t stream) {
170
- int mmq_x = MOE_X_Q4_0;
171
- int mmq_y = MOE_Y_Q4_0;
172
- int nwarps = NWARPS_Q4_0;
173
-
174
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
175
- const int block_num_y = (tokens_post_padded) / mmq_x;
176
- const dim3 block_nums(block_num_x, block_num_y, 1);
177
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
178
-
179
- if (nrows_x % mmq_y == 0) {
180
- constexpr bool need_check = false;
181
- moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
182
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
183
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
184
- } else {
185
- constexpr bool need_check = true;
186
- moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
187
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
188
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
189
- }
190
-}
191
-
192
-#if defined(USE_ROCM)
193
- #define MOE_X_Q4_1 8
194
- #define MOE_Y_Q4_1 128
195
- #define NWARPS_Q4_1 8
196
-#else
197
- #define MOE_X_Q4_1 4
198
- #define MOE_Y_Q4_1 32
199
- #define NWARPS_Q4_1 4
200
-#endif
201
-
202
-template <typename scalar_t, bool need_check>
203
-static __global__ void
204
-#if defined(USE_ROCM)
205
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2)
206
-#endif
207
- moe_q4_1(const void* __restrict__ vx, const void* __restrict__ vy,
208
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
209
- const int* expert_ids, const int* num_tokens_post_padded,
210
- const int exp_stride, const int ncols_x, const int nrows_x,
211
- const int ncols_y, const int nrows_y, const int nrows_dst,
212
- const int top_k) {
213
- const int mmq_x = MOE_X_Q4_1;
214
- const int mmq_y = MOE_Y_Q4_1;
215
- const int nwarps = NWARPS_Q4_1;
216
-
217
- moe_q<scalar_t, QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps,
218
- allocate_tiles_q4_1<mmq_y>, load_tiles_q4_1<mmq_y, nwarps, need_check>,
219
- VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>(
220
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
221
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
222
-}
223
-
224
-template <typename scalar_t>
225
-static void ggml_moe_q4_1_q8_1_cuda(
226
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
227
- const int* expert_ids, const int* num_tokens_post_padded,
228
- const int exp_stride, const int ncols_x, const int nrows_x,
229
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
230
- const int tokens_post_padded, cudaStream_t stream) {
231
- int mmq_x = MOE_X_Q4_1;
232
- int mmq_y = MOE_Y_Q4_1;
233
- int nwarps = NWARPS_Q4_1;
234
-
235
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
236
- const int block_num_y = (tokens_post_padded) / mmq_x;
237
- const dim3 block_nums(block_num_x, block_num_y, 1);
238
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
239
-
240
- if (nrows_x % mmq_y == 0) {
241
- constexpr bool need_check = false;
242
- moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
243
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
244
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
245
- } else {
246
- constexpr bool need_check = true;
247
- moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
248
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
249
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
250
- }
251
-}
252
-
253
-#if defined(USE_ROCM)
254
- #define MOE_X_Q5_0 8
255
- #define MOE_Y_Q5_0 128
256
- #define NWARPS_Q5_0 8
257
-#else
258
- #define MOE_X_Q5_0 4
259
- #define MOE_Y_Q5_0 32
260
- #define NWARPS_Q5_0 4
261
-#endif
262
-
263
-template <typename scalar_t, bool need_check>
264
-static __global__ void
265
-#if defined(USE_ROCM)
266
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2)
267
-#endif
268
- moe_q5_0(const void* __restrict__ vx, const void* __restrict__ vy,
269
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
270
- const int* expert_ids, const int* num_tokens_post_padded,
271
- const int exp_stride, const int ncols_x, const int nrows_x,
272
- const int ncols_y, const int nrows_y, const int nrows_dst,
273
- const int top_k) {
274
- const int mmq_x = MOE_X_Q5_0;
275
- const int mmq_y = MOE_Y_Q5_0;
276
- const int nwarps = NWARPS_Q5_0;
277
-
278
- moe_q<scalar_t, QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps,
279
- allocate_tiles_q5_0<mmq_y>, load_tiles_q5_0<mmq_y, nwarps, need_check>,
280
- VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>(
281
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
282
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
283
-}
284
-
285
-template <typename scalar_t>
286
-static void ggml_moe_q5_0_q8_1_cuda(
287
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
288
- const int* expert_ids, const int* num_tokens_post_padded,
289
- const int exp_stride, const int ncols_x, const int nrows_x,
290
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
291
- const int tokens_post_padded, cudaStream_t stream) {
292
- const int mmq_x = MOE_X_Q5_0;
293
- const int mmq_y = MOE_Y_Q5_0;
294
- const int nwarps = NWARPS_Q5_0;
295
-
296
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
297
- const int block_num_y = (tokens_post_padded) / mmq_x;
298
- const dim3 block_nums(block_num_x, block_num_y, 1);
299
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
300
-
301
- if (nrows_x % mmq_y == 0) {
302
- constexpr bool need_check = false;
303
- moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
304
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
305
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
306
- } else {
307
- constexpr bool need_check = true;
308
- moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
309
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
310
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
311
- }
312
-}
313
-
314
-#if defined(USE_ROCM)
315
- #define MOE_X_Q5_1 8
316
- #define MOE_Y_Q5_1 128
317
- #define NWARPS_Q5_1 8
318
-#else
319
- #define MOE_X_Q5_1 4
320
- #define MOE_Y_Q5_1 32
321
- #define NWARPS_Q5_1 4
322
-#endif
323
-
324
-template <typename scalar_t, bool need_check>
325
-static __global__ void
326
-#if defined(USE_ROCM)
327
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2)
328
-#endif
329
- moe_q5_1(const void* __restrict__ vx, const void* __restrict__ vy,
330
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
331
- const int* expert_ids, const int* num_tokens_post_padded,
332
- const int exp_stride, const int ncols_x, const int nrows_x,
333
- const int ncols_y, const int nrows_y, const int nrows_dst,
334
- const int top_k) {
335
- const int mmq_x = MOE_X_Q5_1;
336
- const int mmq_y = MOE_Y_Q5_1;
337
- const int nwarps = NWARPS_Q5_1;
338
-
339
- moe_q<scalar_t, QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps,
340
- allocate_tiles_q5_1<mmq_y>, load_tiles_q5_1<mmq_y, nwarps, need_check>,
341
- VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>(
342
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
343
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
344
-}
345
-
346
-template <typename scalar_t>
347
-static void ggml_moe_q5_1_q8_1_cuda(
348
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
349
- const int* expert_ids, const int* num_tokens_post_padded,
350
- const int exp_stride, const int ncols_x, const int nrows_x,
351
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
352
- const int tokens_post_padded, cudaStream_t stream) {
353
- const int mmq_x = MOE_X_Q5_1;
354
- const int mmq_y = MOE_Y_Q5_1;
355
- const int nwarps = NWARPS_Q5_1;
356
-
357
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
358
- const int block_num_y = (tokens_post_padded) / mmq_x;
359
- const dim3 block_nums(block_num_x, block_num_y, 1);
360
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
361
-
362
- if (nrows_x % mmq_y == 0) {
363
- constexpr bool need_check = false;
364
- moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
365
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
366
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
367
- } else {
368
- constexpr bool need_check = true;
369
- moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
370
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
371
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
372
- }
373
-}
374
-
375
-#if defined(USE_ROCM)
376
- #define MOE_X_Q8_0 8
377
- #define MOE_Y_Q8_0 128
378
- #define NWARPS_Q8_0 8
379
-#else
380
- #define MOE_X_Q8_0 4
381
- #define MOE_Y_Q8_0 32
382
- #define NWARPS_Q8_0 4
383
-#endif
384
-
385
-template <typename scalar_t, bool need_check>
386
-static __global__ void
387
-#if defined(USE_ROCM)
388
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2)
389
-#endif
390
- moe_q8_0(const void* __restrict__ vx, const void* __restrict__ vy,
391
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
392
- const int* expert_ids, const int* num_tokens_post_padded,
393
- const int exp_stride, const int ncols_x, const int nrows_x,
394
- const int ncols_y, const int nrows_y, const int nrows_dst,
395
- const int top_k) {
396
- const int mmq_x = MOE_X_Q8_0;
397
- const int mmq_y = MOE_Y_Q8_0;
398
- const int nwarps = NWARPS_Q8_0;
399
-
400
- moe_q<scalar_t, QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps,
401
- allocate_tiles_q8_0<mmq_y>, load_tiles_q8_0<mmq_y, nwarps, need_check>,
402
- VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>(
403
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
404
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
405
-}
406
-
407
-template <typename scalar_t>
408
-static void ggml_moe_q8_0_q8_1_cuda(
409
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
410
- const int* expert_ids, const int* num_tokens_post_padded,
411
- const int exp_stride, const int ncols_x, const int nrows_x,
412
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
413
- const int tokens_post_padded, cudaStream_t stream) {
414
- const int mmq_x = MOE_X_Q8_0;
415
- const int mmq_y = MOE_Y_Q8_0;
416
- const int nwarps = NWARPS_Q8_0;
417
-
418
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
419
- const int block_num_y = (tokens_post_padded) / mmq_x;
420
- const dim3 block_nums(block_num_x, block_num_y, 1);
421
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
422
-
423
- if (nrows_x % mmq_y == 0) {
424
- constexpr bool need_check = false;
425
- moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
426
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
427
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
428
- } else {
429
- constexpr bool need_check = true;
430
- moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
431
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
432
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
433
- }
434
-}
435
-
436
-#if defined(USE_ROCM)
437
- #define MOE_X_Q2_K 8
438
- #define MOE_Y_Q2_K 128
439
- #define NWARPS_Q2_K 8
440
-#else
441
- #define MOE_X_Q2_K 4
442
- #define MOE_Y_Q2_K 32
443
- #define NWARPS_Q2_K 4
444
-#endif
445
-
446
-template <typename scalar_t, bool need_check>
447
-static __global__ void
448
-#if defined(USE_ROCM)
449
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2)
450
-#endif
451
- moe_q2_K(const void* __restrict__ vx, const void* __restrict__ vy,
452
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
453
- const int* expert_ids, const int* num_tokens_post_padded,
454
- const int exp_stride, const int ncols_x, const int nrows_x,
455
- const int ncols_y, const int nrows_y, const int nrows_dst,
456
- const int top_k) {
457
- const int mmq_x = MOE_X_Q2_K;
458
- const int mmq_y = MOE_Y_Q2_K;
459
- const int nwarps = NWARPS_Q2_K;
460
-
461
- moe_q<scalar_t, QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps,
462
- allocate_tiles_q2_K<mmq_y>, load_tiles_q2_K<mmq_y, nwarps, need_check>,
463
- VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>(
464
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
465
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
466
-}
467
-
468
-template <typename scalar_t>
469
-static void ggml_moe_q2_K_q8_1_cuda(
470
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
471
- const int* expert_ids, const int* num_tokens_post_padded,
472
- const int exp_stride, const int ncols_x, const int nrows_x,
473
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
474
- const int tokens_post_padded, cudaStream_t stream) {
475
- const int mmq_x = MOE_X_Q2_K;
476
- const int mmq_y = MOE_Y_Q2_K;
477
- const int nwarps = NWARPS_Q2_K;
478
-
479
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
480
- const int block_num_y = (tokens_post_padded) / mmq_x;
481
- const dim3 block_nums(block_num_x, block_num_y, 1);
482
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
483
-
484
- if (nrows_x % mmq_y == 0) {
485
- constexpr bool need_check = false;
486
- moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
487
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
488
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
489
- } else {
490
- constexpr bool need_check = true;
491
- moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
492
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
493
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
494
- }
495
-}
496
-
497
-#if defined(USE_ROCM)
498
- #define MOE_X_Q3_K 8
499
- #define MOE_Y_Q3_K 128
500
- #define NWARPS_Q3_K 8
501
-#else
502
- #define MOE_X_Q3_K 4
503
- #define MOE_Y_Q3_K 32
504
- #define NWARPS_Q3_K 4
505
-#endif
506
-
507
-template <typename scalar_t, bool need_check>
508
-static __global__ void
509
-#if defined(USE_ROCM)
510
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2)
511
-#endif
512
- moe_q3_K(const void* __restrict__ vx, const void* __restrict__ vy,
513
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
514
- const int* expert_ids, const int* num_tokens_post_padded,
515
- const int exp_stride, const int ncols_x, const int nrows_x,
516
- const int ncols_y, const int nrows_y, const int nrows_dst,
517
- const int top_k) {
518
-
519
- const int mmq_x = MOE_X_Q3_K;
520
- const int mmq_y = MOE_Y_Q3_K;
521
- const int nwarps = NWARPS_Q3_K;
522
-
523
- moe_q<scalar_t, QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps,
524
- allocate_tiles_q3_K<mmq_y>, load_tiles_q3_K<mmq_y, nwarps, need_check>,
525
- VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>(
526
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
527
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
528
-}
529
-template <typename scalar_t>
530
-static void ggml_moe_q3_K_q8_1_cuda(
531
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
532
- const int* expert_ids, const int* num_tokens_post_padded,
533
- const int exp_stride, const int ncols_x, const int nrows_x,
534
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
535
- const int tokens_post_padded, cudaStream_t stream) {
536
- const int mmq_x = MOE_X_Q3_K;
537
- const int mmq_y = MOE_Y_Q3_K;
538
- const int nwarps = NWARPS_Q3_K;
539
-
540
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
541
- const int block_num_y = (tokens_post_padded) / mmq_x;
542
- const dim3 block_nums(block_num_x, block_num_y, 1);
543
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
544
-
545
- if (nrows_x % mmq_y == 0) {
546
- constexpr bool need_check = false;
547
- moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
548
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
549
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
550
- } else {
551
- constexpr bool need_check = true;
552
- moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
553
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
554
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
555
- }
556
-}
557
-
558
-#if defined(USE_ROCM)
559
- #define MOE_X_Q4_K 8
560
- #define MOE_Y_Q4_K 128
561
- #define NWARPS_Q4_K 8
562
-#else
563
- #define MOE_X_Q4_K 4
564
- #define MOE_Y_Q4_K 32
565
- #define NWARPS_Q4_K 4
566
-#endif
567
-
568
-template <typename scalar_t, bool need_check>
569
-static __global__ void
570
-#if defined(USE_ROCM)
571
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2)
572
-#endif
573
- moe_q4_K(const void* __restrict__ vx, const void* __restrict__ vy,
574
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
575
- const int* expert_ids, const int* num_tokens_post_padded,
576
- const int exp_stride, const int ncols_x, const int nrows_x,
577
- const int ncols_y, const int nrows_y, const int nrows_dst,
578
- const int top_k) {
579
- const int mmq_x = MOE_X_Q4_K;
580
- const int mmq_y = MOE_Y_Q4_K;
581
- const int nwarps = NWARPS_Q4_K;
582
-
583
- moe_q<scalar_t, QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps,
584
- allocate_tiles_q4_K<mmq_y>, load_tiles_q4_K<mmq_y, nwarps, need_check>,
585
- VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>(
586
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
587
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
588
-}
589
-
590
-template <typename scalar_t>
591
-static void ggml_moe_q4_K_q8_1_cuda(
592
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
593
- const int* expert_ids, const int* num_tokens_post_padded,
594
- const int exp_stride, const int ncols_x, const int nrows_x,
595
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
596
- const int tokens_post_padded, cudaStream_t stream) {
597
- const int mmq_x = MOE_X_Q4_K;
598
- const int mmq_y = MOE_Y_Q4_K;
599
- const int nwarps = NWARPS_Q4_K;
600
-
601
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
602
- const int block_num_y = (tokens_post_padded) / mmq_x;
603
- const dim3 block_nums(block_num_x, block_num_y, 1);
604
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
605
-
606
- if (nrows_x % mmq_y == 0) {
607
- constexpr bool need_check = false;
608
- moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
609
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
610
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
611
- } else {
612
- constexpr bool need_check = true;
613
- moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
614
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
615
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
616
- }
617
-}
618
-
619
-#if defined(USE_ROCM)
620
- #define MOE_X_Q5_K 8
621
- #define MOE_Y_Q5_K 128
622
- #define NWARPS_Q5_K 8
623
-#else
624
- #define MOE_X_Q5_K 4
625
- #define MOE_Y_Q5_K 32
626
- #define NWARPS_Q5_K 4
627
-#endif
628
-
629
-template <typename scalar_t, bool need_check>
630
-static __global__ void
631
-#if defined(USE_ROCM)
632
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2)
633
-#endif
634
- moe_q5_K(const void* __restrict__ vx, const void* __restrict__ vy,
635
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
636
- const int* expert_ids, const int* num_tokens_post_padded,
637
- const int exp_stride, const int ncols_x, const int nrows_x,
638
- const int ncols_y, const int nrows_y, const int nrows_dst,
639
- const int top_k) {
640
- const int mmq_x = MOE_X_Q5_K;
641
- const int mmq_y = MOE_Y_Q5_K;
642
- const int nwarps = NWARPS_Q5_K;
643
-
644
- moe_q<scalar_t, QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps,
645
- allocate_tiles_q5_K<mmq_y>, load_tiles_q5_K<mmq_y, nwarps, need_check>,
646
- VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>(
647
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
648
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
649
-}
650
-
651
-template <typename scalar_t>
652
-static void ggml_moe_q5_K_q8_1_cuda(
653
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
654
- const int* expert_ids, const int* num_tokens_post_padded,
655
- const int exp_stride, const int ncols_x, const int nrows_x,
656
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
657
- const int tokens_post_padded, cudaStream_t stream) {
658
- const int mmq_x = MOE_X_Q5_K;
659
- const int mmq_y = MOE_Y_Q5_K;
660
- const int nwarps = NWARPS_Q5_K;
661
-
662
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
663
- const int block_num_y = (tokens_post_padded) / mmq_x;
664
- const dim3 block_nums(block_num_x, block_num_y, 1);
665
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
666
-
667
- if (nrows_x % mmq_y == 0) {
668
- constexpr bool need_check = false;
669
- moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
670
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
671
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
672
- } else {
673
- constexpr bool need_check = true;
674
- moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
675
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
676
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
677
- }
678
-}
679
-
680
-#if defined(USE_ROCM)
681
- #define MOE_X_Q6_K 8
682
- #define MOE_Y_Q6_K 128
683
- #define NWARPS_Q6_K 8
684
-#else
685
- #define MOE_X_Q6_K 4
686
- #define MOE_Y_Q6_K 32
687
- #define NWARPS_Q6_K 4
688
-#endif
689
-
690
-template <typename scalar_t, bool need_check>
691
-static __global__ void
692
-#if defined(USE_ROCM)
693
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2)
694
-#endif
695
- moe_q6_K(const void* __restrict__ vx, const void* __restrict__ vy,
696
- scalar_t* __restrict__ dst, const int* sorted_token_ids,
697
- const int* expert_ids, const int* num_tokens_post_padded,
698
- const int exp_stride, const int ncols_x, const int nrows_x,
699
- const int ncols_y, const int nrows_y, const int nrows_dst,
700
- const int top_k) {
701
- const int mmq_x = MOE_X_Q6_K;
702
- const int mmq_y = MOE_Y_Q6_K;
703
- const int nwarps = NWARPS_Q6_K;
704
-
705
- moe_q<scalar_t, QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps,
706
- allocate_tiles_q6_K<mmq_y>, load_tiles_q6_K<mmq_y, nwarps, need_check>,
707
- VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>(
708
- vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
709
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
710
-}
711
-
712
-template <typename scalar_t>
713
-static void ggml_moe_q6_K_q8_1_cuda(
714
- const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
715
- const int* expert_ids, const int* num_tokens_post_padded,
716
- const int exp_stride, const int ncols_x, const int nrows_x,
717
- const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
718
- const int tokens_post_padded, cudaStream_t stream) {
719
- const int mmq_x = MOE_X_Q6_K;
720
- const int mmq_y = MOE_Y_Q6_K;
721
- const int nwarps = NWARPS_Q6_K;
722
-
723
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
724
- const int block_num_y = (tokens_post_padded) / mmq_x;
725
- const dim3 block_nums(block_num_x, block_num_y, 1);
726
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
727
-
728
- if (nrows_x % mmq_y == 0) {
729
- constexpr bool need_check = false;
730
- moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
731
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
732
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
733
- } else {
734
- constexpr bool need_check = true;
735
- moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
736
- w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
737
- exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
738
- }
739
-}
csrc/libtorch_stable/quantization/gguf/moe_vec.cuh
deleted
-338
@@ -1,338 +0,0 @@
1
-// copied and adapted from
2
-// https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
3
-template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr,
4
- vec_dot_q_cuda_t vec_dot_q_cuda>
5
-static __global__ void moe_vec_q(const void* __restrict__ vx,
6
- const void* __restrict__ vy,
7
- scalar_t* __restrict__ dst,
8
- const int* topk_ids, const int topk,
9
- const int ncols, const int nrows,
10
- const int token_stride) {
11
- const auto row = blockIdx.x * blockDim.y + threadIdx.y;
12
-
13
- const auto token = blockIdx.z / topk;
14
- const auto expert = (topk_ids)[blockIdx.z];
15
-
16
- if (row >= nrows) {
17
- return;
18
- }
19
-
20
- const int blocks_per_row = ncols / qk;
21
- const int blocks_per_warp = vdr * WARP_SIZE / qi;
22
-
23
- // partial sum for each thread
24
- float tmp = 0.0f;
25
-
26
- const block_q_t* x = ((const block_q_t*)vx) + expert * nrows * blocks_per_row;
27
- const block_q8_1* y =
28
- (const block_q8_1*)(((const int*)vy) + token * token_stride);
29
-
30
- for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row;
31
- i += blocks_per_warp) {
32
- const int ibx = row * blocks_per_row + i; // x block index
33
-
34
- const int iby = i * (qk / QK8_1); // y block index that aligns with ibx
35
-
36
- const int iqs =
37
- vdr *
38
- (threadIdx.x %
39
- (qi / vdr)); // x block quant index when casting the quants to int
40
-
41
- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
42
- }
43
-
44
- // sum up partial sums and write back result
45
-#pragma unroll
46
- for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
47
- tmp += VLLM_SHFL_XOR_SYNC(tmp, mask);
48
- }
49
-
50
- if (threadIdx.x == 0) {
51
- dst[blockIdx.z * nrows + row] = tmp;
52
- }
53
-}
54
-
55
-template <typename scalar_t>
56
-static void moe_vec_q4_0_q8_1_cuda(const void* vx, const void* vy,
57
- scalar_t* dst, const int* topk_ids,
58
- const int top_k, const int tokens,
59
- const int ncols, const int nrows,
60
- const int token_stride,
61
- cudaStream_t stream) {
62
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
63
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
64
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
65
- moe_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ,
66
- vec_dot_q4_0_q8_1><<<block_nums, block_dims, 0, stream>>>(
67
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
68
-}
69
-
70
-template <typename scalar_t>
71
-static void moe_vec_q4_1_q8_1_cuda(const void* vx, const void* vy,
72
- scalar_t* dst, const int* topk_ids,
73
- const int top_k, const int tokens,
74
- const int ncols, const int nrows,
75
- const int token_stride,
76
- cudaStream_t stream) {
77
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
78
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
79
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
80
- moe_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ,
81
- vec_dot_q4_1_q8_1><<<block_nums, block_dims, 0, stream>>>(
82
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
83
-}
84
-
85
-template <typename scalar_t>
86
-static void moe_vec_q5_0_q8_1_cuda(const void* vx, const void* vy,
87
- scalar_t* dst, const int* topk_ids,
88
- const int top_k, const int tokens,
89
- const int ncols, const int nrows,
90
- const int token_stride,
91
- cudaStream_t stream) {
92
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
93
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
94
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
95
- moe_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ,
96
- vec_dot_q5_0_q8_1><<<block_nums, block_dims, 0, stream>>>(
97
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
98
-}
99
-
100
-template <typename scalar_t>
101
-static void moe_vec_q5_1_q8_1_cuda(const void* vx, const void* vy,
102
- scalar_t* dst, const int* topk_ids,
103
- const int top_k, const int tokens,
104
- const int ncols, const int nrows,
105
- const int token_stride,
106
- cudaStream_t stream) {
107
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
108
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
109
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
110
- moe_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ,
111
- vec_dot_q5_1_q8_1><<<block_nums, block_dims, 0, stream>>>(
112
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
113
-}
114
-
115
-template <typename scalar_t>
116
-static void moe_vec_q8_0_q8_1_cuda(const void* vx, const void* vy,
117
- scalar_t* dst, const int* topk_ids,
118
- const int top_k, const int tokens,
119
- const int ncols, const int nrows,
120
- const int token_stride,
121
- cudaStream_t stream) {
122
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
123
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
124
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
125
- moe_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ,
126
- vec_dot_q8_0_q8_1><<<block_nums, block_dims, 0, stream>>>(
127
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
128
-}
129
-
130
-template <typename scalar_t>
131
-static void moe_vec_q2_K_q8_1_cuda(const void* vx, const void* vy,
132
- scalar_t* dst, const int* topk_ids,
133
- const int top_k, const int tokens,
134
- const int ncols, const int nrows,
135
- const int token_stride,
136
- cudaStream_t stream) {
137
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
138
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
139
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
140
- moe_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ,
141
- vec_dot_q2_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
142
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
143
-}
144
-
145
-template <typename scalar_t>
146
-static void moe_vec_q3_K_q8_1_cuda(const void* vx, const void* vy,
147
- scalar_t* dst, const int* topk_ids,
148
- const int top_k, const int tokens,
149
- const int ncols, const int nrows,
150
- const int token_stride,
151
- cudaStream_t stream) {
152
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
153
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
154
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
155
- moe_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ,
156
- vec_dot_q3_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
157
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
158
-}
159
-
160
-template <typename scalar_t>
161
-static void moe_vec_q4_K_q8_1_cuda(const void* vx, const void* vy,
162
- scalar_t* dst, const int* topk_ids,
163
- const int top_k, const int tokens,
164
- const int ncols, const int nrows,
165
- const int token_stride,
166
- cudaStream_t stream) {
167
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
168
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
169
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
170
- moe_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ,
171
- vec_dot_q4_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
172
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
173
-}
174
-
175
-template <typename scalar_t>
176
-static void moe_vec_q5_K_q8_1_cuda(const void* vx, const void* vy,
177
- scalar_t* dst, const int* topk_ids,
178
- const int top_k, const int tokens,
179
- const int ncols, const int nrows,
180
- const int token_stride,
181
- cudaStream_t stream) {
182
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
183
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
184
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
185
- moe_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ,
186
- vec_dot_q5_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
187
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
188
-}
189
-
190
-template <typename scalar_t>
191
-static void moe_vec_q6_K_q8_1_cuda(const void* vx, const void* vy,
192
- scalar_t* dst, const int* topk_ids,
193
- const int top_k, const int tokens,
194
- const int ncols, const int nrows,
195
- const int token_stride,
196
- cudaStream_t stream) {
197
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
198
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
199
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
200
- moe_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ,
201
- vec_dot_q6_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
202
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
203
-}
204
-
205
-template <typename scalar_t>
206
-static void moe_vec_iq2_xxs_q8_1_cuda(const void* vx, const void* vy,
207
- scalar_t* dst, const int* topk_ids,
208
- const int top_k, const int tokens,
209
- const int ncols, const int nrows,
210
- const int token_stride,
211
- cudaStream_t stream) {
212
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
213
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
214
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
215
- moe_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
216
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
217
- ncols, nrows, token_stride);
218
-}
219
-
220
-template <typename scalar_t>
221
-static void moe_vec_iq2_xs_q8_1_cuda(const void* vx, const void* vy,
222
- scalar_t* dst, const int* topk_ids,
223
- const int top_k, const int tokens,
224
- const int ncols, const int nrows,
225
- const int token_stride,
226
- cudaStream_t stream) {
227
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
228
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
229
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
230
- moe_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
231
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
232
- ncols, nrows, token_stride);
233
-}
234
-
235
-template <typename scalar_t>
236
-static void moe_vec_iq2_s_q8_1_cuda(const void* vx, const void* vy,
237
- scalar_t* dst, const int* topk_ids,
238
- const int top_k, const int tokens,
239
- const int ncols, const int nrows,
240
- const int token_stride,
241
- cudaStream_t stream) {
242
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
243
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
244
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
245
- moe_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
246
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
247
- ncols, nrows, token_stride);
248
-}
249
-
250
-template <typename scalar_t>
251
-static void moe_vec_iq3_xxs_q8_1_cuda(const void* vx, const void* vy,
252
- scalar_t* dst, const int* topk_ids,
253
- const int top_k, const int tokens,
254
- const int ncols, const int nrows,
255
- const int token_stride,
256
- cudaStream_t stream) {
257
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
258
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
259
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
260
- moe_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
261
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
262
- ncols, nrows, token_stride);
263
-}
264
-
265
-template <typename scalar_t>
266
-static void moe_vec_iq1_s_q8_1_cuda(const void* vx, const void* vy,
267
- scalar_t* dst, const int* topk_ids,
268
- const int top_k, const int tokens,
269
- const int ncols, const int nrows,
270
- const int token_stride,
271
- cudaStream_t stream) {
272
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
273
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
274
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
275
- moe_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
276
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
277
- ncols, nrows, token_stride);
278
-}
279
-
280
-template <typename scalar_t>
281
-static void moe_vec_iq1_m_q8_1_cuda(const void* vx, const void* vy,
282
- scalar_t* dst, const int* topk_ids,
283
- const int top_k, const int tokens,
284
- const int ncols, const int nrows,
285
- const int token_stride,
286
- cudaStream_t stream) {
287
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
288
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
289
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
290
- moe_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>
291
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
292
- ncols, nrows, token_stride);
293
-}
294
-
295
-template <typename scalar_t>
296
-static void moe_vec_iq4_nl_q8_1_cuda(const void* vx, const void* vy,
297
- scalar_t* dst, const int* topk_ids,
298
- const int top_k, const int tokens,
299
- const int ncols, const int nrows,
300
- const int token_stride,
301
- cudaStream_t stream) {
302
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
303
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
304
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
305
- moe_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ,
306
- vec_dot_iq4_nl_q8_1><<<block_nums, block_dims, 0, stream>>>(
307
- vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
308
-}
309
-
310
-template <typename scalar_t>
311
-static void moe_vec_iq4_xs_q8_1_cuda(const void* vx, const void* vy,
312
- scalar_t* dst, const int* topk_ids,
313
- const int top_k, const int tokens,
314
- const int ncols, const int nrows,
315
- const int token_stride,
316
- cudaStream_t stream) {
317
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
318
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
319
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
320
- moe_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
321
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
322
- ncols, nrows, token_stride);
323
-}
324
-
325
-template <typename scalar_t>
326
-static void moe_vec_iq3_s_q8_1_cuda(const void* vx, const void* vy,
327
- scalar_t* dst, const int* topk_ids,
328
- const int top_k, const int tokens,
329
- const int ncols, const int nrows,
330
- const int token_stride,
331
- cudaStream_t stream) {
332
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
333
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
334
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
335
- moe_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
336
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
337
- ncols, nrows, token_stride);
338
-}
csrc/libtorch_stable/quantization/gguf/vecdotq.cuh
deleted
-1812
@@ -1,1812 +0,0 @@
1
-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh
2
-// and https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
3
-static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {
4
- const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment
5
-
6
- int x32 = x16[2*i32 + 0] << 0;
7
- x32 |= x16[2*i32 + 1] << 16;
8
-
9
- return x32;
10
-}
11
-
12
-static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {
13
- return ((const int *) x)[i32]; // assume at least 4 byte alignment
14
-}
15
-
16
-static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {
17
- const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment
18
- int x32 = 0;
19
- x32 |= x16[0] << 0;
20
- x32 |= x16[1] << 16;
21
- return x32;
22
-}
23
-
24
-static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {
25
- const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment
26
- int x32 = 0;
27
- x32 |= x16[0] << 0;
28
- x32 |= x16[1] << 16;
29
- return x32;
30
-}
31
-
32
-static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {
33
- return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
34
-}
35
-
36
-static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {
37
- return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
38
-}
39
-
40
-// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
41
-// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
42
-
43
-#define VDR_Q4_0_Q8_1_MMVQ 2
44
-#define VDR_Q4_0_Q8_1_MMQ 4
45
-
46
-template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
47
- const int * v, const int * u, const float & d4, const half2 & ds8) {
48
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
49
- int sumi = 0;
50
-
51
-#pragma unroll
52
- for (int i = 0; i < vdr; ++i) {
53
- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
54
- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
55
-
56
- // SIMD dot product of quantized values
57
- sumi = __dp4a(vi0, u[2*i+0], sumi);
58
- sumi = __dp4a(vi1, u[2*i+1], sumi);
59
- }
60
-
61
- const float2 ds8f = __half22float2(ds8);
62
-
63
- // second part effectively subtracts 8 from each quant value
64
- return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
65
-#endif
66
-}
67
-
68
-#define VDR_Q4_1_Q8_1_MMVQ 2
69
-#define VDR_Q4_1_Q8_1_MMQ 4
70
-
71
-template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
72
- const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
73
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
74
- int sumi = 0;
75
-
76
-#pragma unroll
77
- for (int i = 0; i < vdr; ++i) {
78
- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
79
- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
80
-
81
- // SIMD dot product of quantized values
82
- sumi = __dp4a(vi0, u[2*i+0], sumi);
83
- sumi = __dp4a(vi1, u[2*i+1], sumi);
84
- }
85
-
86
- const float2 tmp = __half22float2(__hmul2(dm4, ds8));
87
- const float d4d8 = tmp.x;
88
- const float m4s8 = tmp.y;
89
-
90
- // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
91
- return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
92
-#endif
93
-}
94
-
95
-#define VDR_Q5_0_Q8_1_MMVQ 2
96
-#define VDR_Q5_0_Q8_1_MMQ 4
97
-
98
-template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
99
- const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
100
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
101
- int sumi = 0;
102
-
103
-#pragma unroll
104
- for (int i = 0; i < vdr; ++i) {
105
- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
106
- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
107
- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
108
- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
109
- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
110
- sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
111
-
112
- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
113
- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
114
- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
115
- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
116
- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
117
- sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
118
- }
119
-
120
- const float2 ds8f = __half22float2(ds8);
121
-
122
- // second part effectively subtracts 16 from each quant value
123
- return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
124
-#endif
125
-}
126
-
127
-
128
-#define VDR_Q5_1_Q8_1_MMVQ 2
129
-#define VDR_Q5_1_Q8_1_MMQ 4
130
-
131
-template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
132
- const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
133
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
134
- int sumi = 0;
135
-
136
-#pragma unroll
137
- for (int i = 0; i < vdr; ++i) {
138
- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
139
- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
140
- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
141
- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
142
- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
143
- sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
144
-
145
- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
146
- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
147
- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
148
- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
149
- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
150
- sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
151
- }
152
-
153
- const float2 tmp = __half22float2(__hmul2(dm5, ds8));
154
- const float d5d8 = tmp.x;
155
- const float m5s8 = tmp.y;
156
-
157
- // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
158
- return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
159
-#endif
160
-}
161
-
162
-#define VDR_Q8_0_Q8_1_MMVQ 2
163
-#define VDR_Q8_0_Q8_1_MMQ 8
164
-
165
-template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(
166
- const int * v, const int * u, const float & d8_0, const float & d8_1) {
167
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
168
- int sumi = 0;
169
-
170
-#pragma unroll
171
- for (int i = 0; i < vdr; ++i) {
172
- // SIMD dot product of quantized values
173
- sumi = __dp4a(v[i], u[i], sumi);
174
- }
175
- return d8_0*d8_1 * sumi;
176
-#endif
177
-}
178
-
179
-template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
180
- const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
181
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
182
-
183
- int sumi = 0;
184
-
185
-#pragma unroll
186
- for (int i = 0; i < vdr; ++i) {
187
- // SIMD dot product of quantized values
188
- sumi = __dp4a(v[i], u[i], sumi);
189
- }
190
-
191
- const float2 tmp = __half22float2(__hmul2(dm8, ds8));
192
- const float d8d8 = tmp.x;
193
- const float m8s8 = tmp.y;
194
-
195
- // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
196
- return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
197
-#endif
198
-}
199
-
200
-#define VDR_Q2_K_Q8_1_MMVQ 1
201
-#define VDR_Q2_K_Q8_1_MMQ 2
202
-
203
-// contiguous v/x values
204
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
205
- const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
206
- const half2 & dm2, const float * __restrict__ d8) {
207
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
208
- float sumf_d = 0.0f;
209
- float sumf_m = 0.0f;
210
-
211
-#pragma unroll
212
- for (int i = 0; i < QR2_K; ++i) {
213
- const int sc = scales[2*i];
214
-
215
- const int vi = (v >> (2*i)) & 0x03030303;
216
-
217
- sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
218
-
219
- // fill int with 4x m
220
- int m = sc >> 4;
221
- m |= m << 8;
222
- m |= m << 16;
223
- sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
224
- }
225
-
226
- const float2 dm2f = __half22float2(dm2);
227
-
228
- return dm2f.x*sumf_d - dm2f.y*sumf_m;
229
-#endif
230
-}
231
-
232
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
233
- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
234
- const half2 & dm2, const float & d8) {
235
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
236
- int sumi_d = 0;
237
- int sumi_m = 0;
238
-
239
-#pragma unroll
240
- for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {
241
- int sumi_d_sc = 0;
242
-
243
- const int sc = scales[i0 / (QI8_1/2)];
244
-
245
- // fill int with 4x m
246
- int m = sc >> 4;
247
- m |= m << 8;
248
- m |= m << 16;
249
-
250
-#pragma unroll
251
- for (int i = i0; i < i0 + QI8_1/2; ++i) {
252
- sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product
253
- sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m
254
- }
255
-
256
- sumi_d += sumi_d_sc * (sc & 0xF);
257
- }
258
-
259
- const float2 dm2f = __half22float2(dm2);
260
-
261
- return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);
262
-#endif
263
-}
264
-
265
-#define VDR_Q3_K_Q8_1_MMVQ 1
266
-#define VDR_Q3_K_Q8_1_MMQ 2
267
-
268
-// contiguous v/x values
269
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
270
- const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
271
- const int & scale_offset, const float & d3, const float * __restrict__ d8) {
272
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
273
-
274
- float sumf = 0.0f;
275
-
276
-#pragma unroll
277
- for (int i = 0; i < QR3_K; ++i) {
278
- const int isc = scale_offset + 2*i;
279
-
280
- const int isc_low = isc % (QK_K/32);
281
- const int sc_shift_low = 4 * (isc / (QK_K/32));
282
- const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
283
-
284
- const int isc_high = isc % (QK_K/64);
285
- const int sc_shift_high = 2 * (isc / (QK_K/64));
286
- const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
287
-
288
- const int sc = (sc_low | sc_high) - 32;
289
-
290
- const int vil = (vl >> (2*i)) & 0x03030303;
291
-
292
- const int vih = ((vh >> i) << 2) & 0x04040404;
293
-
294
- const int vi = __vsubss4(vil, vih);
295
-
296
- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
297
- }
298
-
299
- return d3 * sumf;
300
-#endif
301
-}
302
-
303
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
304
- const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
305
- const float & d3, const float & d8) {
306
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
307
- int sumi = 0;
308
-
309
-#pragma unroll
310
- for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
311
- int sumi_sc = 0;
312
-
313
- for (int i = i0; i < i0 + QI8_1/2; ++i) {
314
- sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product
315
- }
316
-
317
- sumi += sumi_sc * scales[i0 / (QI8_1/2)];
318
- }
319
-
320
- return d3*d8 * sumi;
321
-#endif
322
-}
323
-
324
-#define VDR_Q4_K_Q8_1_MMVQ 2
325
-#define VDR_Q4_K_Q8_1_MMQ 8
326
-
327
-// contiguous v/x values
328
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
329
- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
330
- const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
331
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
332
-
333
- float sumf_d = 0.0f;
334
- float sumf_m = 0.0f;
335
-
336
-#pragma unroll
337
- for (int i = 0; i < QR4_K; ++i) {
338
- const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
339
- const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
340
-
341
- const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
342
- const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u
343
-
344
- sumf_d += d8[i] * (dot1 * sc[i]);
345
- sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
346
- }
347
-
348
- const float2 dm4f = __half22float2(dm4);
349
- return dm4f.x*sumf_d - dm4f.y*sumf_m;
350
-#endif
351
-}
352
-
353
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
354
- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
355
- const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
356
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
357
- float sumf_d = 0.0f;
358
- float sumf_m = 0.0f;
359
-
360
-#pragma unroll
361
- for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
362
- int sumi_d = 0;
363
-
364
-#pragma unroll
365
- for (int j = 0; j < QI8_1; ++j) {
366
- sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
367
- }
368
-
369
- const float2 ds8f = __half22float2(ds8[i]);
370
-
371
- sumf_d += ds8f.x * (sc[i] * sumi_d);
372
- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
373
- }
374
-
375
- const float2 dm4f = __half22float2(dm4);
376
-
377
- return dm4f.x*sumf_d - dm4f.y*sumf_m;
378
-#endif
379
-}
380
-
381
-#define VDR_Q5_K_Q8_1_MMVQ 2
382
-#define VDR_Q5_K_Q8_1_MMQ 8
383
-
384
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
385
- const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
386
- const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
387
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
388
-
389
- float sumf_d = 0.0f;
390
- float sumf_m = 0.0f;
391
-
392
-#pragma unroll
393
- for (int i = 0; i < QR5_K; ++i) {
394
- const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
395
- const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
396
-
397
- const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
398
- const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
399
-
400
- const int v0i = vl0i | vh0i;
401
- const int v1i = vl1i | vh1i;
402
-
403
- const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
404
- const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u
405
-
406
- sumf_d += d8[i] * (dot1 * sc[i]);
407
- sumf_m += d8[i] * (dot2 * m[i]);
408
- }
409
-
410
- const float2 dm5f = __half22float2(dm5);
411
- return dm5f.x*sumf_d - dm5f.y*sumf_m;
412
-#endif
413
-}
414
-
415
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
416
- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
417
- const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
418
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
419
- float sumf_d = 0.0f;
420
- float sumf_m = 0.0f;
421
-
422
-#pragma unroll
423
- for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
424
- int sumi_d = 0;
425
-
426
-#pragma unroll
427
- for (int j = 0; j < QI8_1; ++j) {
428
- sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
429
- }
430
-
431
- const float2 ds8f = __half22float2(ds8[i]);
432
-
433
- sumf_d += ds8f.x * (sc[i] * sumi_d);
434
- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
435
- }
436
-
437
- const float2 dm4f = __half22float2(dm4);
438
-
439
- return dm4f.x*sumf_d - dm4f.y*sumf_m;
440
-#endif
441
-}
442
-
443
-#define VDR_Q6_K_Q8_1_MMVQ 1
444
-#define VDR_Q6_K_Q8_1_MMQ 8
445
-
446
-// contiguous v/x values
447
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
448
- const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
449
- const float & d, const float * __restrict__ d8) {
450
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
451
- float sumf = 0.0f;
452
-
453
-#pragma unroll
454
- for (int i = 0; i < QR6_K; ++i) {
455
- const int sc = scales[4*i];
456
- const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
457
- const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
458
- const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
459
-
460
- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
461
- }
462
-
463
- return d*sumf;
464
-#endif
465
-}
466
-
467
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
468
- const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
469
- const float & d6, const float * __restrict__ d8) {
470
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
471
- float sumf_d = 0.0f;
472
-
473
-#pragma unroll
474
- for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
475
- int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
476
-
477
-#pragma unroll
478
- for (int i = i0; i < i0 + 2; ++i) {
479
- sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
480
- sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
481
-
482
- sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
483
- sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
484
- }
485
-
486
- sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);
487
- }
488
-
489
- return d6 * sumf_d;
490
-#endif
491
-}
492
-
493
-static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
494
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
495
-
496
- const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;
497
-
498
- int v[VDR_Q4_0_Q8_1_MMVQ];
499
- int u[2*VDR_Q4_0_Q8_1_MMVQ];
500
-
501
-#pragma unroll
502
- for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
503
- v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);
504
- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
505
- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);
506
- }
507
-
508
- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, __half2float(bq4_0->d), bq8_1->ds);
509
-}
510
-
511
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
512
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
513
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI4_0) + mmq_y/QI4_0];
514
- *x_ql = tile_x_qs;
515
- *x_dm = (half2 *) tile_x_d;
516
-}
517
-
518
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(
519
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
520
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
521
- const int kbx = k / QI4_0;
522
- const int kqsx = k % QI4_0;
523
-
524
- const block_q4_0 * bx0 = (const block_q4_0 *) vx;
525
- float * x_dmf = (float *) x_dm;
526
-
527
-#pragma unroll
528
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
529
- int i = i0 + i_offset;
530
- if (need_check) {
531
- i = min(i, i_max);
532
- }
533
- const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;
534
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
535
- // x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbx] = bxi->d;
536
- }
537
-
538
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_0;
539
- const int kbxd = k % blocks_per_tile_x_row;
540
-
541
-#pragma unroll
542
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {
543
- int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;
544
- if (need_check) {
545
- i = min(i, i_max);
546
- }
547
- const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;
548
- x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
549
- }
550
-}
551
-
552
-static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(
553
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
554
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
555
- (void)x_qh; (void)x_sc;
556
-
557
- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
558
- const float * x_dmf = (const float *) x_dm;
559
-
560
- int u[2*VDR_Q4_0_Q8_1_MMQ];
561
-
562
-#pragma unroll
563
- for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
564
- u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
565
- u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_0) % WARP_SIZE_GGUF];
566
- }
567
-
568
- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
569
- (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], u, x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i/QI4_0 + k/QI4_0],
570
- y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
571
-}
572
-
573
-static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
574
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
575
-
576
- const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;
577
-
578
- int v[VDR_Q4_1_Q8_1_MMVQ];
579
- int u[2*VDR_Q4_1_Q8_1_MMVQ];
580
-
581
-#pragma unroll
582
- for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
583
- v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);
584
- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
585
- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);
586
- }
587
-
588
- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
589
-}
590
-
591
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
592
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + + mmq_y];
593
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI4_1) + mmq_y/QI4_1];
594
- *x_ql = tile_x_qs;
595
- *x_dm = tile_x_dm;
596
-}
597
-
598
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(
599
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
600
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
601
- const int kbx = k / QI4_1;
602
- const int kqsx = k % QI4_1;
603
-
604
- const block_q4_1 * bx0 = (const block_q4_1 *) vx;
605
-
606
-#pragma unroll
607
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
608
- int i = i0 + i_offset;
609
- if (need_check) {
610
- i = min(i, i_max);
611
- }
612
- const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;
613
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
614
- }
615
-
616
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_1;
617
- const int kbxd = k % blocks_per_tile_x_row;
618
-
619
-#pragma unroll
620
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {
621
- int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;
622
- if (need_check) {
623
- i = min(i, i_max);
624
- }
625
- const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;
626
- x_dm[i * (WARP_SIZE_GGUF/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
627
- }
628
-}
629
-
630
-static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(
631
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
632
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
633
- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
634
-
635
- int u[2*VDR_Q4_1_Q8_1_MMQ];
636
-
637
-#pragma unroll
638
- for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
639
- u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
640
- u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_1) % WARP_SIZE_GGUF];
641
- }
642
-
643
- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
644
- (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], u, x_dm[i * (WARP_SIZE_GGUF/QI4_1) + i/QI4_1 + k/QI4_1],
645
- y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
646
-}
647
-
648
-static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
649
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
650
-
651
- const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;
652
-
653
- int vl[VDR_Q5_0_Q8_1_MMVQ];
654
- int vh[VDR_Q5_0_Q8_1_MMVQ];
655
- int u[2*VDR_Q5_0_Q8_1_MMVQ];
656
-
657
-#pragma unroll
658
- for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
659
- vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);
660
- vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));
661
- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
662
- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);
663
- }
664
-
665
- return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, __half2float(bq5_0->d), bq8_1->ds);
666
-}
667
-
668
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
669
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
670
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI5_0) + mmq_y/QI5_0];
671
-
672
- *x_ql = tile_x_ql;
673
- *x_dm = (half2 *) tile_x_d;
674
-}
675
-
676
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(
677
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
678
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
679
- const int kbx = k / QI5_0;
680
- const int kqsx = k % QI5_0;
681
-
682
- const block_q5_0 * bx0 = (const block_q5_0 *) vx;
683
-
684
-#pragma unroll
685
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
686
- int i = i0 + i_offset;
687
-
688
- if (need_check) {
689
- i = min(i, i_max);
690
- }
691
- const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbx;
692
- const int ql = get_int_from_uint8(bxi->qs, kqsx);
693
- const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));
694
-
695
- int qs0 = (ql >> 0) & 0x0F0F0F0F;
696
- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
697
- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
698
- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
699
- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
700
- qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
701
-
702
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+0] = qs0;
703
-
704
- int qs1 = (ql >> 4) & 0x0F0F0F0F;
705
- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
706
- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
707
- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
708
- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
709
- qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
710
-
711
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+1] = qs1;
712
- }
713
-
714
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_0;
715
- const int kbxd = k % blocks_per_tile_x_row;
716
- float * x_dmf = (float *) x_dm;
717
-
718
-#pragma unroll
719
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {
720
- int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;
721
-
722
- if (need_check) {
723
- i = min(i, i_max);
724
- }
725
-
726
- const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;
727
- x_dmf[i * (WARP_SIZE_GGUF/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
728
- }
729
-}
730
-
731
-static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
732
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
733
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
734
- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
735
- const int index_bx = i * (WARP_SIZE_GGUF/QI5_0) + i/QI5_0 + k/QI5_0;
736
- const float * x_dmf = (const float *) x_dm;
737
- const float * y_df = (const float *) y_ds;
738
-
739
- int u[2*VDR_Q5_0_Q8_1_MMQ];
740
-
741
-#pragma unroll
742
- for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {
743
- u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
744
- u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_0) % WARP_SIZE_GGUF];
745
- }
746
-
747
- return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>
748
- (&x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
749
-}
750
-
751
-static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
752
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
753
-
754
- const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;
755
-
756
- int vl[VDR_Q5_1_Q8_1_MMVQ];
757
- int vh[VDR_Q5_1_Q8_1_MMVQ];
758
- int u[2*VDR_Q5_1_Q8_1_MMVQ];
759
-
760
-#pragma unroll
761
- for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
762
- vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);
763
- vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));
764
- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
765
- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);
766
- }
767
-
768
- return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
769
-}
770
-
771
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
772
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
773
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI5_1) + mmq_y/QI5_1];
774
-
775
- *x_ql = tile_x_ql;
776
- *x_dm = tile_x_dm;
777
-}
778
-
779
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(
780
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
781
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
782
- const int kbx = k / QI5_1;
783
- const int kqsx = k % QI5_1;
784
-
785
- const block_q5_1 * bx0 = (const block_q5_1 *) vx;
786
-
787
-#pragma unroll
788
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
789
- int i = i0 + i_offset;
790
-
791
- if (need_check) {
792
- i = min(i, i_max);
793
- }
794
-
795
- const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbx;
796
-
797
- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
798
- const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));
799
-
800
- int qs0 = (ql >> 0) & 0x0F0F0F0F;
801
- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
802
- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
803
- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
804
- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
805
-
806
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+0] = qs0;
807
-
808
- int qs1 = (ql >> 4) & 0x0F0F0F0F;
809
- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
810
- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
811
- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
812
- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
813
-
814
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+1] = qs1;
815
- }
816
-
817
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_1;
818
- const int kbxd = k % blocks_per_tile_x_row;
819
-
820
-#pragma unroll
821
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {
822
- int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;
823
-
824
- if (need_check) {
825
- i = min(i, i_max);
826
- }
827
-
828
- const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;
829
-
830
- x_dm[i * (WARP_SIZE_GGUF/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
831
- }
832
-}
833
-
834
-static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(
835
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
836
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
837
- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
838
- const int index_bx = i * (WARP_SIZE_GGUF/QI5_1) + + i/QI5_1 + k/QI5_1;
839
-
840
- int u[2*VDR_Q5_1_Q8_1_MMQ];
841
-
842
-#pragma unroll
843
- for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {
844
- u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
845
- u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_1) % WARP_SIZE_GGUF];
846
- }
847
-
848
- return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
849
- (&x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
850
-}
851
-
852
-static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
853
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
854
-
855
- const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;
856
-
857
- int v[VDR_Q8_0_Q8_1_MMVQ];
858
- int u[VDR_Q8_0_Q8_1_MMVQ];
859
-
860
-#pragma unroll
861
- for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
862
- v[i] = get_int_from_int8(bq8_0->qs, iqs + i);
863
- u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
864
- }
865
-
866
- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, __half2float(bq8_0->d), __low2float(bq8_1->ds));
867
-}
868
-
869
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
870
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
871
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI8_0) + mmq_y/QI8_0];
872
-
873
- *x_ql = tile_x_qs;
874
- *x_dm = (half2 *) tile_x_d;
875
-}
876
-
877
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(
878
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
879
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
880
- const int kbx = k / QI8_0;
881
- const int kqsx = k % QI8_0;
882
- float * x_dmf = (float *) x_dm;
883
-
884
- const block_q8_0 * bx0 = (const block_q8_0 *) vx;
885
-
886
-#pragma unroll
887
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
888
- int i = i0 + i_offset;
889
-
890
- if (need_check) {
891
- i = min(i, i_max);
892
- }
893
- const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;
894
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
895
- }
896
-
897
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI8_0;
898
- const int kbxd = k % blocks_per_tile_x_row;
899
-
900
-#pragma unroll
901
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {
902
- int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;
903
-
904
- if (need_check) {
905
- i = min(i, i_max);
906
- }
907
- const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;
908
- x_dmf[i * (WARP_SIZE_GGUF/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
909
- }
910
-}
911
-
912
-static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(
913
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
914
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
915
- const float * x_dmf = (const float *) x_dm;
916
- const float * y_df = (const float *) y_ds;
917
-
918
- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>
919
- (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], &y_qs[j * WARP_SIZE_GGUF + k], x_dmf[i * (WARP_SIZE_GGUF/QI8_0) + i/QI8_0 + k/QI8_0],
920
- y_df[j * (WARP_SIZE_GGUF/QI8_1) + k/QI8_1]);
921
-}
922
-
923
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
924
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
925
-
926
- const block_q2_K * bq2_K = (const block_q2_K *) vbq;
927
-
928
- const int bq8_offset = QR2_K * (iqs / QI8_1);
929
- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
930
-
931
- const uint8_t * scales = bq2_K->scales + scale_offset;
932
-
933
- const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);
934
- int u[QR2_K];
935
- float d8[QR2_K];
936
-
937
-#pragma unroll
938
- for (int i = 0; i < QR2_K; ++ i) {
939
- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
940
- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
941
- }
942
-
943
- return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
944
-}
945
-
946
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
947
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
948
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI2_K) + mmq_y/QI2_K];
949
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/4) + mmq_y/4];
950
-
951
- *x_ql = tile_x_ql;
952
- *x_dm = tile_x_dm;
953
- *x_sc = tile_x_sc;
954
-}
955
-
956
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(
957
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
958
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
959
- const int kbx = k / QI2_K;
960
- const int kqsx = k % QI2_K;
961
-
962
- const block_q2_K * bx0 = (const block_q2_K *) vx;
963
-
964
-#pragma unroll
965
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
966
- int i = i0 + i_offset;
967
-
968
- if (need_check) {
969
- i = min(i, i_max);
970
- }
971
- const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;
972
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
973
- }
974
-
975
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI2_K;
976
- const int kbxd = k % blocks_per_tile_x_row;
977
-
978
-#pragma unroll
979
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {
980
- int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;
981
-
982
- if (need_check) {
983
- i = min(i, i_max);
984
- }
985
- const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;
986
- x_dm[i * (WARP_SIZE_GGUF/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
987
- }
988
-
989
-#pragma unroll
990
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
991
- int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF/4);
992
-
993
- if (need_check) {
994
- i = min(i, i_max);
995
- }
996
- const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/4)) / (QI2_K/4);
997
- x_sc[i * (WARP_SIZE_GGUF/4) + i / 4 + k % (WARP_SIZE_GGUF/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
998
- }
999
-}
1000
-
1001
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
1002
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1003
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1004
- const int kbx = k / QI2_K;
1005
- const int ky = (k % QI2_K) * QR2_K;
1006
- const float * y_df = (const float *) y_ds;
1007
-
1008
- int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];
1009
-
1010
- const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
1011
- const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));
1012
-
1013
-#pragma unroll
1014
- for (int l = 0; l < QR2_K*VDR_Q2_K_Q8_1_MMQ; ++l) {
1015
- v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;
1016
- }
1017
-
1018
- const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/4) + i/4 + kbx*4]) + ky/4;
1019
-
1020
- const int index_y = j * WARP_SIZE_GGUF + (QR2_K*k) % WARP_SIZE_GGUF;
1021
- return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE_GGUF/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1022
-}
1023
-
1024
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
1025
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1026
-
1027
- const block_q3_K * bq3_K = (const block_q3_K *) vbq;
1028
-
1029
- const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
1030
- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
1031
-
1032
- const float d = __half2float(bq3_K->d);
1033
-
1034
- const int vl = get_int_from_uint8(bq3_K->qs, iqs);
1035
-
1036
- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1037
- const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
1038
-
1039
- int u[QR3_K];
1040
- float d8[QR3_K];
1041
-
1042
-#pragma unroll
1043
- for (int i = 0; i < QR3_K; ++i) {
1044
- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
1045
- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
1046
- }
1047
-
1048
- return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
1049
-}
1050
-
1051
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1052
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
1053
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI3_K) + mmq_y/QI3_K];
1054
- __shared__ int tile_x_qh[mmq_y * (WARP_SIZE_GGUF/2) + mmq_y/2];
1055
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/4) + mmq_y/4];
1056
-
1057
- *x_ql = tile_x_ql;
1058
- *x_dm = tile_x_dm;
1059
- *x_qh = tile_x_qh;
1060
- *x_sc = tile_x_sc;
1061
-}
1062
-
1063
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(
1064
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1065
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1066
- const int kbx = k / QI3_K;
1067
- const int kqsx = k % QI3_K;
1068
-
1069
- const block_q3_K * bx0 = (const block_q3_K *) vx;
1070
-
1071
-#pragma unroll
1072
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1073
- int i = i0 + i_offset;
1074
- if (need_check) {
1075
- i = min(i, i_max);
1076
- }
1077
- const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;
1078
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1079
- }
1080
-
1081
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI3_K;
1082
- const int kbxd = k % blocks_per_tile_x_row;
1083
- float * x_dmf = (float *) x_dm;
1084
-
1085
-#pragma unroll
1086
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {
1087
- int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;
1088
- if (need_check) {
1089
- i = min(i, i_max);
1090
- }
1091
- const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;
1092
- x_dmf[i * (WARP_SIZE_GGUF/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1093
- }
1094
-
1095
-#pragma unroll
1096
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {
1097
- int i = i0 + i_offset * 2 + k / (WARP_SIZE_GGUF/2);
1098
- if (need_check) {
1099
- i = min(i, i_max);
1100
- }
1101
- const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/2)) / (QI3_K/2);
1102
- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1103
- x_qh[i * (WARP_SIZE_GGUF/2) + i / 2 + k % (WARP_SIZE_GGUF/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1104
- }
1105
-
1106
-#pragma unroll
1107
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
1108
- int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF/4);
1109
- if (need_check) {
1110
- i = min(i, i_max);
1111
- }
1112
- const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/4)) / (QI3_K/4);
1113
-
1114
- const int ksc = k % (QI3_K/4);
1115
-
1116
- const int ksc_low = ksc % (QI3_K/8);
1117
- const int shift_low = 4 * (ksc / (QI3_K/8));
1118
- const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;
1119
-
1120
- const int ksc_high = QI3_K/8;
1121
- const int shift_high = 2 * ksc;
1122
- const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;
1123
-
1124
- const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
1125
-
1126
- x_sc[i * (WARP_SIZE_GGUF/4) + i / 4 + k % (WARP_SIZE_GGUF/4)] = sc;
1127
- }
1128
-}
1129
-
1130
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(
1131
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1132
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1133
-
1134
- const int kbx = k / QI3_K;
1135
- const int ky = (k % QI3_K) * QR3_K;
1136
- const float * x_dmf = (const float *) x_dm;
1137
- const float * y_df = (const float *) y_ds;
1138
-
1139
- const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE_GGUF/4) + i/4 + kbx*4)) + ky/4;
1140
-
1141
- int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];
1142
-
1143
-#pragma unroll
1144
- for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {
1145
- const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1146
- const int shift = 2 * ((ky % 32) / 8);
1147
- const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;
1148
-
1149
- const int vh = x_qh[i * (WARP_SIZE_GGUF/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1150
- const int vlh = (vh << 2) & 0x04040404;
1151
-
1152
- v[l] = __vsubss4(vll, vlh);
1153
- }
1154
-
1155
- const int index_y = j * WARP_SIZE_GGUF + (k*QR3_K) % WARP_SIZE_GGUF;
1156
- return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE_GGUF/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1157
-}
1158
-
1159
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
1160
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1161
- const block_q4_K * bq4_K = (const block_q4_K *) vbq;
1162
-
1163
- int v[2];
1164
- int u[2*QR4_K];
1165
- float d8[QR4_K];
1166
-
1167
- // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
1168
- const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
1169
-
1170
- // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
1171
- // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
1172
- // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
1173
- // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
1174
-
1175
- const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1176
- v[0] = q4[0];
1177
- v[1] = q4[4];
1178
-
1179
- const uint16_t * scales = (const uint16_t *)bq4_K->scales;
1180
- uint16_t aux[2];
1181
- const int j = bq8_offset/2;
1182
- if (j < 2) {
1183
- aux[0] = scales[j+0] & 0x3f3f;
1184
- aux[1] = scales[j+2] & 0x3f3f;
1185
- } else {
1186
- aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1187
- aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1188
- }
1189
- const uint8_t * sc = (const uint8_t *)aux;
1190
- const uint8_t * m = sc + 2;
1191
-
1192
- for (int i = 0; i < QR4_K; ++i) {
1193
- const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1194
- d8[i] = __low2float(bq8i->ds);
1195
-
1196
- const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1197
- u[2*i+0] = q8[0];
1198
- u[2*i+1] = q8[4];
1199
- }
1200
-
1201
- return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
1202
-}
1203
-
1204
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1205
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
1206
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI4_K) + mmq_y/QI4_K];
1207
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1208
-
1209
- *x_ql = tile_x_ql;
1210
- *x_dm = tile_x_dm;
1211
- *x_sc = tile_x_sc;
1212
-}
1213
-
1214
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(
1215
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1216
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1217
- const int kbx = k / QI4_K; // == 0 if QK_K == 256
1218
- const int kqsx = k % QI4_K; // == k if QK_K == 256
1219
-
1220
- const block_q4_K * bx0 = (const block_q4_K *) vx;
1221
-
1222
-#pragma unroll
1223
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1224
- int i = i0 + i_offset;
1225
-
1226
- if (need_check) {
1227
- i = min(i, i_max);
1228
- }
1229
- const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;
1230
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1231
- }
1232
-
1233
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_K; // == 1 if QK_K == 256
1234
- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1235
-
1236
-#pragma unroll
1237
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {
1238
- int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;
1239
- if (need_check) {
1240
- i = min(i, i_max);
1241
- }
1242
- const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;
1243
- x_dm[i * (WARP_SIZE_GGUF/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1244
- }
1245
-
1246
-#pragma unroll
1247
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1248
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1249
-
1250
- if (need_check) {
1251
- i = min(i, i_max);
1252
- }
1253
-
1254
- const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / (QI4_K/8);
1255
-
1256
- const int * scales = (const int *) bxi->scales;
1257
-
1258
- const int ksc = k % (WARP_SIZE_GGUF/8);
1259
- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1260
- int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1261
- scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1262
-
1263
- x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + ksc] = scales8;
1264
- }
1265
-}
1266
-
1267
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(
1268
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1269
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1270
- (void)x_qh;
1271
-
1272
- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1273
-
1274
- const int index_y = j * WARP_SIZE_GGUF + (QR4_K*k) % WARP_SIZE_GGUF;
1275
- return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE_GGUF + 1) + k], &y_qs[index_y], sc, sc+8,
1276
- x_dm[i * (WARP_SIZE_GGUF/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1277
-}
1278
-
1279
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
1280
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1281
-
1282
- const block_q5_K * bq5_K = (const block_q5_K *) vbq;
1283
-
1284
- int vl[2];
1285
- int vh[2];
1286
- int u[2*QR5_K];
1287
- float d8[QR5_K];
1288
-
1289
- const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
1290
- const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1291
- const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
1292
-
1293
- vl[0] = ql[0];
1294
- vl[1] = ql[4];
1295
-
1296
- vh[0] = qh[0] >> bq8_offset;
1297
- vh[1] = qh[4] >> bq8_offset;
1298
-
1299
- const uint16_t * scales = (const uint16_t *)bq5_K->scales;
1300
- uint16_t aux[2];
1301
- const int j = bq8_offset/2;
1302
- if (j < 2) {
1303
- aux[0] = scales[j+0] & 0x3f3f;
1304
- aux[1] = scales[j+2] & 0x3f3f;
1305
- } else {
1306
- aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1307
- aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1308
- }
1309
- const uint8_t * sc = (const uint8_t *)aux;
1310
- const uint8_t * m = sc + 2;
1311
-
1312
-#pragma unroll
1313
- for (int i = 0; i < QR5_K; ++i) {
1314
- const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1315
- d8[i] = __low2float(bq8i->ds);
1316
-
1317
- const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1318
- u[2*i+0] = q8[0];
1319
- u[2*i+1] = q8[4];
1320
- }
1321
-
1322
- return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
1323
-}
1324
-
1325
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1326
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
1327
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI5_K) + mmq_y/QI5_K];
1328
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1329
-
1330
- *x_ql = tile_x_ql;
1331
- *x_dm = tile_x_dm;
1332
- *x_sc = tile_x_sc;
1333
-}
1334
-
1335
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(
1336
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1337
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1338
- const int kbx = k / QI5_K; // == 0 if QK_K == 256
1339
- const int kqsx = k % QI5_K; // == k if QK_K == 256
1340
-
1341
- const block_q5_K * bx0 = (const block_q5_K *) vx;
1342
-
1343
-#pragma unroll
1344
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1345
- int i = i0 + i_offset;
1346
-
1347
- if (need_check) {
1348
- i = min(i, i_max);
1349
- }
1350
-
1351
- const block_q5_K * bxi = bx0 + i*blocks_per_row + kbx;
1352
- const int ky = QR5_K*kqsx;
1353
-
1354
- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
1355
- const int ql0 = (ql >> 0) & 0x0F0F0F0F;
1356
- const int ql1 = (ql >> 4) & 0x0F0F0F0F;
1357
-
1358
- const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K/4));
1359
- const int qh0 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 0)) << 4) & 0x10101010;
1360
- const int qh1 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 1)) << 4) & 0x10101010;
1361
-
1362
- const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;
1363
- const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);
1364
-
1365
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq0] = ql0 | qh0;
1366
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq1] = ql1 | qh1;
1367
- }
1368
-
1369
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_K; // == 1 if QK_K == 256
1370
- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1371
-
1372
-#pragma unroll
1373
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {
1374
- int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;
1375
-
1376
- if (need_check) {
1377
- i = min(i, i_max);
1378
- }
1379
-
1380
- const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;
1381
- x_dm[i * (WARP_SIZE_GGUF/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1382
- }
1383
-
1384
-#pragma unroll
1385
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1386
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1387
-
1388
- if (need_check) {
1389
- i = min(i, i_max);
1390
- }
1391
-
1392
- const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / (QI5_K/8);
1393
-
1394
- const int * scales = (const int *) bxi->scales;
1395
-
1396
- const int ksc = k % (WARP_SIZE_GGUF/8);
1397
-
1398
- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1399
- int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1400
- scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1401
-
1402
- x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + ksc] = scales8;
1403
- }
1404
-}
1405
-
1406
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(
1407
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1408
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1409
- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1410
-
1411
- const int index_x = i * (QR5_K*WARP_SIZE_GGUF + 1) + QR5_K*k;
1412
- const int index_y = j * WARP_SIZE_GGUF + (QR5_K*k) % WARP_SIZE_GGUF;
1413
- return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,
1414
- x_dm[i * (WARP_SIZE_GGUF/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1415
-}
1416
-
1417
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
1418
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1419
-
1420
- const block_q6_K * bq6_K = (const block_q6_K *) vbq;
1421
-
1422
- const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
1423
- const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
1424
- const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
1425
-
1426
- const int vl = get_int_from_uint8(bq6_K->ql, iqs);
1427
- const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
1428
-
1429
- const int8_t * scales = bq6_K->scales + scale_offset;
1430
-
1431
- int u[QR6_K];
1432
- float d8[QR6_K];
1433
-
1434
-#pragma unroll
1435
- for (int i = 0; i < QR6_K; ++i) {
1436
- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
1437
- d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
1438
- }
1439
-
1440
- return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, __half2float(bq6_K->d), d8);
1441
-}
1442
-
1443
-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1444
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
1445
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI6_K) + mmq_y/QI6_K];
1446
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1447
-
1448
- *x_ql = tile_x_ql;
1449
- *x_dm = tile_x_dm;
1450
- *x_sc = tile_x_sc;
1451
-}
1452
-
1453
-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(
1454
- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,
1455
- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {
1456
- const int kbx = k / QI6_K; // == 0 if QK_K == 256
1457
- const int kqsx = k % QI6_K; // == k if QK_K == 256
1458
-
1459
- const block_q6_K * bx0 = (const block_q6_K *) vx;
1460
-
1461
-#pragma unroll
1462
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
1463
- int i = i0 + i_offset;
1464
-
1465
- if (need_check) {
1466
- i = min(i, i_max);
1467
- }
1468
-
1469
- const block_q6_K * bxi = bx0 + i*blocks_per_row + kbx;
1470
- const int ky = QR6_K*kqsx;
1471
-
1472
- const int ql = get_int_from_uint8(bxi->ql, kqsx);
1473
- const int ql0 = (ql >> 0) & 0x0F0F0F0F;
1474
- const int ql1 = (ql >> 4) & 0x0F0F0F0F;
1475
-
1476
- const int qh = get_int_from_uint8(bxi->qh, (QI6_K/4) * (kqsx / (QI6_K/2)) + kqsx % (QI6_K/4));
1477
- const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) << 4) & 0x30303030;
1478
- const int qh1 = (qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) & 0x30303030;
1479
-
1480
- const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;
1481
- const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);
1482
-
1483
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1484
- x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1485
- }
1486
-
1487
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI6_K; // == 1 if QK_K == 256
1488
- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1489
- float * x_dmf = (float *) x_dm;
1490
-
1491
-#pragma unroll
1492
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {
1493
- int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;
1494
-
1495
- if (need_check) {
1496
- i = min(i, i_max);
1497
- }
1498
-
1499
- const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;
1500
-
1501
- x_dmf[i * (WARP_SIZE_GGUF/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1502
- }
1503
-
1504
-#pragma unroll
1505
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1506
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1507
-
1508
- if (need_check) {
1509
- i = min(i, i_max);
1510
- }
1511
-
1512
- const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / 4;
1513
-
1514
- x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + k % (WARP_SIZE_GGUF/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1515
- }
1516
-}
1517
-
1518
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(
1519
- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1520
- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1521
- const float * x_dmf = (const float *) x_dm;
1522
- const float * y_df = (const float *) y_ds;
1523
-
1524
- const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/8]);
1525
-
1526
- const int index_x = i * (QR6_K*WARP_SIZE_GGUF + 1) + QR6_K*k;
1527
- const int index_y = j * WARP_SIZE_GGUF + (QR6_K*k) % WARP_SIZE_GGUF;
1528
- return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE_GGUF/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1529
-}
1530
-
1531
-static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
1532
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1533
- const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;
1534
-
1535
- const int ib32 = iqs;
1536
- const uint16_t * q2 = bq2->qs + 4*ib32;
1537
- const uint8_t * aux8 = (const uint8_t *)q2;
1538
- const int8_t * q8 = bq8_1[ib32].qs;
1539
- uint32_t aux32 = q2[2] | (q2[3] << 16);
1540
- int sumi = 0;
1541
- for (int l = 0; l < 4; ++l) {
1542
- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
1543
- const uint8_t signs = ksigns_iq2xs[aux32 & 127];
1544
- for (int j = 0; j < 8; ++j) {
1545
- sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1546
- }
1547
- q8 += 8;
1548
- aux32 >>= 7;
1549
- }
1550
- const float d = __half2float(bq2->d) * (0.5f + aux32) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
1551
- return d * sumi;
1552
-}
1553
-
1554
-static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
1555
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1556
- const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;
1557
-
1558
- const int ib32 = iqs;
1559
- const uint16_t * q2 = bq2->qs + 4*ib32;
1560
- const int8_t * q8 = bq8_1[ib32].qs;
1561
- const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1562
- const uint8_t ls2 = bq2->scales[ib32] >> 4;
1563
- int sumi1 = 0;
1564
- for (int l = 0; l < 2; ++l) {
1565
- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1566
- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1567
- for (int j = 0; j < 8; ++j) {
1568
- sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1569
- }
1570
- q8 += 8;
1571
- }
1572
- int sumi2 = 0;
1573
- for (int l = 2; l < 4; ++l) {
1574
- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1575
- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1576
- for (int j = 0; j < 8; ++j) {
1577
- sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1578
- }
1579
- q8 += 8;
1580
- }
1581
- const float d = __half2float(bq2->d) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
1582
- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1583
-}
1584
-
1585
-static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1586
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1587
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1588
- const block_iq2_s * bq2 = (const block_iq2_s *) vbq;
1589
-
1590
- const int ib32 = iqs;
1591
- const int8_t * q8 = bq8_1[ib32].qs;
1592
- const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;
1593
- const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1594
- const uint8_t ls2 = bq2->scales[ib32] >> 4;
1595
- int sumi1 = 0;
1596
- for (int l = 0; l < 2; ++l) {
1597
- const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1598
- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1599
- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1600
- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
1601
- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
1602
- sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);
1603
- sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);
1604
- q8 += 8;
1605
- }
1606
- int sumi2 = 0;
1607
- for (int l = 2; l < 4; ++l) {
1608
- const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1609
- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1610
- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1611
- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
1612
- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
1613
- sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);
1614
- sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);
1615
- q8 += 8;
1616
- }
1617
- const float d = __half2float(bq2->d) * __low2float(bq8_1[ib32].ds) * 0.25f;
1618
- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1619
-#endif
1620
-}
1621
-
1622
-static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1623
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1624
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1625
- const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;
1626
-
1627
- const int ib32 = iqs;
1628
- const uint8_t * q3 = bq2->qs + 8*ib32;
1629
- const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;
1630
- const int8_t * q8 = bq8_1[ib32].qs;
1631
- uint32_t aux32 = gas[0] | (gas[1] << 16);
1632
- int sumi = 0;
1633
- for (int l = 0; l < 4; ++l) {
1634
- const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];
1635
- const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];
1636
- const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));
1637
- const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);
1638
- const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);
1639
- sumi = __dp4a(grid_l, *((int *)q8+0), sumi);
1640
- sumi = __dp4a(grid_h, *((int *)q8+1), sumi);
1641
- q8 += 8;
1642
- aux32 >>= 7;
1643
- }
1644
- const float d = __half2float(bq2->d) * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;
1645
- return d * sumi;
1646
-#endif
1647
-}
1648
-
1649
-static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1650
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1651
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1652
- const block_iq3_s * bq2 = (const block_iq3_s *) vbq;
1653
-
1654
- const int ib32 = iqs;
1655
- const uint8_t * qs = bq2->qs + 8*ib32;
1656
- const int8_t * q8 = bq8_1[ib32].qs;
1657
- int sumi = 0;
1658
- for (int l = 0; l < 4; ++l) {
1659
- const uint32_t * grid1 = iq3xs_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));
1660
- const uint32_t * grid2 = iq3xs_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));
1661
- uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
1662
- uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
1663
- const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);
1664
- const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);
1665
- sumi = __dp4a(grid_l, *((int *)q8+0), sumi);
1666
- sumi = __dp4a(grid_h, *((int *)q8+1), sumi);
1667
- q8 += 8;
1668
- }
1669
- const float d = __half2float(bq2->d) * (0.5f + ((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds) * 0.5f;
1670
- return d * sumi;
1671
-#endif
1672
-}
1673
-
1674
-static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1675
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1676
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1677
- const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1678
-
1679
- const int qs_packed = get_int_b2(bq1->qs, iqs);
1680
- const uint8_t * qs = (const uint8_t *) &qs_packed;
1681
-
1682
- const int qh = bq1->qh[iqs];
1683
-
1684
- int sumi = 0;
1685
-#pragma unroll
1686
- for (int l0 = 0; l0 < 8; l0 += 2) {
1687
- const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];
1688
-
1689
- const int grid0 = (grid >> 0) & 0x0F0F0F0F;
1690
- const int grid1 = (grid >> 4) & 0x0F0F0F0F;
1691
-
1692
- const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
1693
- const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
1694
-
1695
- sumi = __dp4a(grid0, u0, sumi);
1696
- sumi = __dp4a(grid1, u1, sumi);
1697
- }
1698
-
1699
- const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);
1700
- const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
1701
- const float2 ds = __half22float2(bq8_1[iqs].ds);
1702
- return d1q * (ds.x*sumi + ds.y*delta);
1703
-#endif
1704
-}
1705
-
1706
-static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
1707
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1708
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1709
-
1710
- const block_iq1_m * bq1 = (const block_iq1_m *) vbq;
1711
-
1712
- const int qs_packed = get_int_b4(bq1->qs, iqs);
1713
- const uint8_t * qs = (const uint8_t *) &qs_packed;
1714
-
1715
- int sumi[2] = {0};
1716
- float sumf[2] = {0.0f};
1717
-#pragma unroll
1718
- for (int l0 = 0; l0 < 8; l0 += 2) {
1719
- const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));
1720
-
1721
- const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];
1722
-
1723
- const int grid0 = (grid >> 0) & 0x0F0F0F0F;
1724
- const int grid1 = (grid >> 4) & 0x0F0F0F0F;
1725
-
1726
- const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
1727
- const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
1728
-
1729
- sumi[l0/4] = __dp4a(grid0, u0, sumi[l0/4]);
1730
- sumi[l0/4] = __dp4a(grid1, u1, sumi[l0/4]);
1731
-
1732
- const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);
1733
- int sumy = 0;
1734
- sumy = __dp4a(u0, 0x01010101, sumy);
1735
- sumy = __dp4a(u1, 0x01010101, sumy);
1736
- sumf[l0/4] += delta*sumy;
1737
- }
1738
-
1739
- const uint16_t * sc = (const uint16_t *) bq1->scales;
1740
-
1741
- iq1m_scale_t scale;
1742
- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
1743
- const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);
1744
-
1745
- const int tmp = sc[iqs/2] >> (6*(iqs%2));
1746
- const int sc0 = 2*((tmp >> 0) & 0x07) + 1;
1747
- const int sc1 = 2*((tmp >> 3) & 0x07) + 1;
1748
- return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
1749
-#endif
1750
-}
1751
-
1752
-static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,
1753
- int & val1, int & val2) {
1754
-
1755
- uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;
1756
- aux32 = q4 & 0x0f0f0f0f;
1757
- uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);
1758
- uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);
1759
- val1 = v1 | (v2 << 16);
1760
- aux32 = (q4 >> 4) & 0x0f0f0f0f;
1761
- v1 = values[q8[0]] | (values[q8[1]] << 8);
1762
- v2 = values[q8[2]] | (values[q8[3]] << 8);
1763
- val2 = v1 | (v2 << 16);
1764
-}
1765
-
1766
-static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1767
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1768
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1769
-
1770
- const block_iq4_nl * bq = (const block_iq4_nl *) vbq;
1771
-
1772
- const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;
1773
- const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;
1774
-
1775
- const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1776
-
1777
- int v1, v2;
1778
- int sumi1 = 0, sumi2 = 0;
1779
- for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
1780
- const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);
1781
- get_int_from_table_16(aux, values, v1, v2);
1782
- sumi1 = __dp4a(v1, q8[l+0], sumi1);
1783
- sumi2 = __dp4a(v2, q8[l+4], sumi2);
1784
- }
1785
- const float d = __half2float(bq->d) * __low2float(bq8_1->ds);
1786
- return d * (sumi1 + sumi2);
1787
-#endif
1788
-}
1789
-
1790
-
1791
-static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
1792
- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1793
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1794
- const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;
1795
- const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1796
-
1797
- // iqs is 0...7
1798
- const int ib32 = iqs;
1799
- const int32_t * q8 = (const int *)bq8_1[ib32].qs;
1800
- const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;
1801
- const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);
1802
- const float d = __half2float(bq4->d) * (ls - 32) * __low2float(bq8_1[ib32].ds);
1803
- int v1, v2;
1804
- int sumi1 = 0, sumi2 = 0;
1805
- for (int j = 0; j < 4; ++j) {
1806
- get_int_from_table_16(q4[j], values, v1, v2);
1807
- sumi1 = __dp4a(v1, q8[j+0], sumi1);
1808
- sumi2 = __dp4a(v2, q8[j+4], sumi2);
1809
- }
1810
- return d * (sumi1 + sumi2);
1811
-#endif
1812
-}
\ No newline at end of file
csrc/libtorch_stable/torch_bindings.cpp
+1
-37
@@ -557,34 +557,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
557
// Post processing for GPTQ.
558
ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
559
560
- // Dequantization for GGML.
561
- ops.def(
562
- "ggml_dequantize(Tensor W, int type, SymInt m, SymInt n, ScalarType? "
563
- "dtype) -> Tensor");
564
-
565
- // mmvq kernel for GGML.
566
- ops.def(
567
- "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, SymInt row) "
568
- "-> Tensor");
569
-
570
- // mmq kernel for GGML.
571
- ops.def(
572
- "ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor");
573
-
574
- // moe kernel for GGML.
575
- ops.def(
576
- "ggml_moe_a8(Tensor X, Tensor W, "
577
- "Tensor sorted_token_ids, Tensor expert_ids, Tensor "
578
- "num_tokens_post_padded, "
579
- "int type, SymInt row, SymInt top_k, SymInt tokens) -> Tensor");
580
-
581
- ops.def(
582
- "ggml_moe_a8_vec(Tensor X, Tensor W, "
583
- "Tensor topk_ids, int top_k, "
584
- "int type, SymInt row, SymInt tokens) -> Tensor");
585
-
586
- ops.def("ggml_moe_get_block_size(int type) -> int");
587
-
560
// Mamba selective scan kernel
561
ops.def(
562
"selective_scan_fwd(Tensor! u, Tensor! delta,"
@@ -741,12 +713,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
713
ops.impl("gptq_gemm", TORCH_BOX(&gptq_gemm));
714
ops.impl("gptq_shuffle", TORCH_BOX(&gptq_shuffle));
715
744
- // GGML kernels
745
- ops.impl("ggml_dequantize", TORCH_BOX(&ggml_dequantize));
746
- ops.impl("ggml_mul_mat_vec_a8", TORCH_BOX(&ggml_mul_mat_vec_a8));
747
- ops.impl("ggml_mul_mat_a8", TORCH_BOX(&ggml_mul_mat_a8));
748
- ops.impl("ggml_moe_a8", TORCH_BOX(&ggml_moe_a8));
749
- ops.impl("ggml_moe_a8_vec", TORCH_BOX(&ggml_moe_a8_vec));
716
+ // Mamba kernels
717
ops.impl("selective_scan_fwd", TORCH_BOX(&selective_scan_fwd));
718
719
ops.impl("paged_attention_v1", TORCH_BOX(&paged_attention_v1));
@@ -790,9 +757,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
757
ops.impl("cutlass_scaled_mm_supports_fp4",
758
TORCH_BOX(&cutlass_scaled_mm_supports_fp4));
759
#endif
793
-
794
- // GGML block size lookup (no tensor args)
795
- ops.impl("ggml_moe_get_block_size", TORCH_BOX(&ggml_moe_get_block_size));
760
}
761
762
// Cache ops
docs/features/quantization/README.md
-1
@@ -9,7 +9,6 @@ The following are the supported quantization formats for vLLM:
9
10
- [AutoAWQ](auto_awq.md)
11
- [BitsAndBytes](bnb.md)
12
-- [GGUF](gguf.md)
12
- [GPTQModel](gptqmodel.md)
13
- [Intel Neural Compressor](inc.md)
14
- [LLM Compressor](llm_compressor/README.md)
docs/features/quantization/gguf.md
+8
-2
@@ -3,8 +3,14 @@
3
!!! warning
4
Please note that GGUF support in vLLM is highly experimental and under-optimized at the moment, it might be incompatible with other features. Currently, you can use GGUF as a way to reduce memory footprint. If you encounter any issues, please report them to the vLLM team.
5
6
-!!! warning
7
- Currently, vllm only supports loading single-file GGUF models. If you have a multi-files GGUF model, you can use [gguf-split](https://github.com/ggerganov/llama.cpp/pull/6135) tool to merge them to a single-file model.
6
+!!! note
7
+ GGUF support has migrated to OOT [vllm-gguf-plugin](https://github.com/vllm-project/vllm-gguf-plugin). Make sure you have GGUF plugin installed before serving a GGUF model.
8
+
9
+Before serving a GGUF model, make sure to install the [vllm-gguf-plugin](https://github.com/vllm-project/vllm-gguf-plugin):
10
+
11
+```bash
12
+uv pip install vllm-gguf-plugin
13
+```
14
15
To run a GGUF model with vLLM, you can use the `repo_id:quant_type` format to load directly from HuggingFace. For example, to load a Q4_K_M quantized model from [unsloth/Qwen3-0.6B-GGUF](https://huggingface.co/unsloth/Qwen3-0.6B-GGUF):
16
docs/mkdocs/hooks/generate_examples.py
-1
@@ -32,7 +32,6 @@ def title(text: str) -> str:
32
"mae": "MAE",
33
"ner": "NER",
34
"tpu": "TPU",
35
- "gguf": "GGUF",
35
"lora": "LoRA",
36
"nccl": "NCCL",
37
"rlhf": "RLHF",
requirements/common.txt
-1
@@ -31,7 +31,6 @@ filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/31
31
partial-json-parser # used for parsing partial JSON outputs
32
pyzmq >= 25.0.0
33
msgspec
34
-gguf >= 0.17.0
34
mistral_common[image] >= 1.11.3
35
opencv-python-headless >= 4.13.0 # required for video IO
36
pyyaml
requirements/test/rocm.txt
-8
@@ -279,10 +279,6 @@ genai-perf==0.0.16
279
# via -r requirements/test/rocm.in
280
genson==1.3.0
281
# via datamodel-code-generator
282
-gguf==0.18.0
283
- # via
284
- # -c requirements/common.txt
285
- # -r requirements/test/../common.txt
282
google-api-core==2.30.0
283
# via
284
# google-cloud-core
@@ -589,7 +585,6 @@ numpy==2.2.6
585
# evaluate
586
# fastparquet
587
# genai-perf
592
- # gguf
588
# imagehash
589
# imageio
590
# librosa
@@ -959,7 +954,6 @@ pyyaml==6.0.3
954
# datamodel-code-generator
955
# datasets
956
# genai-perf
962
- # gguf
957
# huggingface-hub
958
# lm-format-enforcer
959
# optuna
@@ -1004,7 +998,6 @@ requests==2.32.5
998
# datasets
999
# docker
1000
# evaluate
1007
- # gguf
1001
# google-api-core
1002
# google-cloud-storage
1003
# gpt-oss
@@ -1231,7 +1224,6 @@ tqdm==4.67.3
1224
# -r requirements/test/../common.txt
1225
# datasets
1226
# evaluate
1234
- # gguf
1227
# huggingface-hub
1228
# lm-eval
1229
# mteb
setup.py
+2
@@ -1239,6 +1239,8 @@ setup(
1239
"opentelemetry-exporter-otlp>=1.26.0",
1240
"opentelemetry-semantic-conventions-ai>=0.4.1",
1241
],
1242
+ # extra quantization plugin
1243
+ "extra-quant": ["vllm-gguf-plugin>=0.0.2"],
1244
},
1245
cmdclass=cmdclass,
1246
package_data=package_data,
tests/compile/fullgraph/test_full_graph.py
-6
@@ -39,12 +39,6 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
39
]
40
)
41
42
- # TODO: figure out why this fails.
43
- if False and is_quant_method_supported("gguf"): # noqa: SIM223
44
- TEST_MODELS.append(
45
- ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", {"quantization": "gguf"})
46
- )
47
-
42
if is_quant_method_supported("gptq"):
43
TEST_MODELS.append(
44
("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {"quantization": "gptq"})
tests/kernels/quantization/test_ggml.py
deleted
-54
@@ -1,54 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import gguf
5
-import pytest
6
-import torch
7
-
8
-from tests.kernels.utils import opcheck
9
-from vllm import _custom_ops as ops # noqa: F401
10
-
11
-
12
-@pytest.mark.parametrize("quant_type", [12])
13
-def test_ggml_opcheck(quant_type):
14
- block_size, type_size = gguf.GGML_QUANT_SIZES[quant_type]
15
- shape = [256, 1152]
16
- qweight = torch.randint(0, 100, shape, device="cuda", dtype=torch.uint8)
17
- m = qweight.shape[0]
18
- n = qweight.shape[1] // type_size * block_size
19
- opcheck(torch.ops._C.ggml_dequantize, (qweight, quant_type, m, n, torch.float16))
20
-
21
- x = torch.rand((m, 512), device="cuda", dtype=torch.float16)
22
- opcheck(torch.ops._C.ggml_mul_mat_a8, (qweight, x, quant_type, qweight.shape[0]))
23
- opcheck(
24
- torch.ops._C.ggml_mul_mat_vec_a8, (qweight, x, quant_type, qweight.shape[0])
25
- )
26
-
27
- shape = [256, 1024, 336]
28
- qweight = torch.randint(0, 100, shape, device="cuda", dtype=torch.uint8)
29
- x = torch.rand((1, 1024), device="cuda", dtype=torch.float16)
30
- sorted_token_ids = torch.arange(776, device="cuda")
31
- expert_ids = torch.randint(0, 256, (194,), device="cuda")
32
- num_tokens_post_padded = torch.tensor([1], dtype=torch.int64, device="cuda")
33
-
34
- opcheck(
35
- torch.ops._C.ggml_moe_a8,
36
- (
37
- x,
38
- qweight,
39
- sorted_token_ids,
40
- expert_ids,
41
- num_tokens_post_padded,
42
- quant_type,
43
- qweight.shape[0],
44
- 1,
45
- x.shape[0],
46
- ),
47
- )
48
-
49
- topk_ids = torch.zeros((1, 1), device="cuda", dtype=torch.int32)
50
-
51
- opcheck(
52
- torch.ops._C.ggml_moe_a8_vec,
53
- (x, qweight, topk_ids, 1, quant_type, qweight.shape[0], x.shape[0]),
54
- )
tests/kernels/quantization/test_gguf.py
deleted
-207
@@ -1,207 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from pathlib import Path
5
-
6
-import pytest
7
-import torch
8
-from gguf import GGMLQuantizationType, GGUFReader, ReaderTensor, dequantize
9
-from huggingface_hub import snapshot_download
10
-
11
-import vllm._custom_ops as ops
12
-from vllm.model_executor.layers.fused_moe import fused_experts
13
-from vllm.model_executor.layers.quantization.gguf import _fused_moe_gguf
14
-from vllm.utils.torch_utils import set_random_seed
15
-
16
-GGUF_SAMPLE = snapshot_download("Isotr0py/test-gguf-sample")
17
-GGUF_SAMPLE_MOE = snapshot_download("SzymonOzog/test-gguf-moe-sample")
18
-
19
-
20
-def get_gguf_sample_tensors(
21
- hidden_size: int, quant_type: GGMLQuantizationType
22
-) -> list[ReaderTensor]:
23
- sample_dir = GGUF_SAMPLE
24
- filename = f"Quant_{quant_type.name}_{hidden_size}.gguf"
25
- sample_file = Path(sample_dir) / filename
26
- return GGUFReader(sample_file).tensors
27
-
28
-
29
-def get_gguf_MoE_tensors(
30
- hidden_size: int, quant_type: GGMLQuantizationType
31
-) -> list[ReaderTensor]:
32
- sample_dir = GGUF_SAMPLE_MOE
33
- filename = f"Quant_{quant_type.name}_{hidden_size}.gguf"
34
- sample_file = Path(sample_dir) / filename
35
- return GGUFReader(sample_file).tensors
36
-
37
-
38
-DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
39
-# Hidden_size for testing, must match the sample file in HF repo,
40
-# we have `hidden_size = 256, 1024` for test in HF repo currently.
41
-HIDDEN_SIZES = [256, 1024]
42
-NUM_TOKENS = [7, 2050] # Arbitrary values for testing
43
-SEEDS = [0]
44
-QUANT_TYPES = [
45
- # i-matrix
46
- GGMLQuantizationType.IQ1_M,
47
- GGMLQuantizationType.IQ1_S,
48
- GGMLQuantizationType.IQ2_S,
49
- GGMLQuantizationType.IQ2_XS,
50
- GGMLQuantizationType.IQ3_S,
51
- GGMLQuantizationType.IQ3_XXS,
52
- GGMLQuantizationType.IQ4_NL,
53
- GGMLQuantizationType.IQ4_XS,
54
- # k-quants
55
- GGMLQuantizationType.Q2_K,
56
- GGMLQuantizationType.Q3_K,
57
- GGMLQuantizationType.Q4_K,
58
- GGMLQuantizationType.Q5_K,
59
- GGMLQuantizationType.Q6_K,
60
- # standard quantization
61
- GGMLQuantizationType.Q4_0,
62
- GGMLQuantizationType.Q5_0,
63
- GGMLQuantizationType.Q8_0,
64
-]
65
-
66
-
67
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
68
-@pytest.mark.parametrize("dtype", DTYPES)
69
-@pytest.mark.parametrize("quant_type", QUANT_TYPES)
70
-@torch.inference_mode()
71
-def test_dequantize(
72
- hidden_size: int, dtype: torch.dtype, quant_type: GGMLQuantizationType
73
-):
74
- tensors = get_gguf_sample_tensors(hidden_size, quant_type)
75
- for tensor in tensors:
76
- shape_str = tensor.name.split("_")[-1]
77
- shape = map(int, shape_str.split("x"))
78
-
79
- ref_output = torch.tensor(
80
- dequantize(tensor.data, quant_type), device="cuda"
81
- ).to(dtype)
82
- output = ops.ggml_dequantize(
83
- torch.tensor(tensor.data, device="cuda"), quant_type, *list(shape), dtype
84
- )
85
-
86
- torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=4e-2)
87
-
88
-
89
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
90
-@pytest.mark.parametrize("dtype", DTYPES)
91
-@pytest.mark.parametrize("quant_type", QUANT_TYPES)
92
-@torch.inference_mode()
93
-def test_mmvq(hidden_size: int, dtype: torch.dtype, quant_type: GGMLQuantizationType):
94
- set_random_seed(0)
95
-
96
- tensors = get_gguf_sample_tensors(hidden_size, quant_type)
97
- x = torch.rand((1, hidden_size), dtype=dtype, device="cuda")
98
- for tensor in tensors:
99
- weight = torch.tensor(dequantize(tensor.data, quant_type), device="cuda").to(
100
- dtype
101
- )
102
- ref_output = x @ weight.T
103
-
104
- qweight = torch.tensor(tensor.data, device="cuda")
105
- output = ops.ggml_mul_mat_vec_a8(qweight, x, quant_type, qweight.shape[0]).to(
106
- dtype
107
- )
108
-
109
- torch.testing.assert_close(output, ref_output, atol=1, rtol=1e-1)
110
-
111
-
112
-@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
113
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
114
-@pytest.mark.parametrize("dtype", DTYPES)
115
-@pytest.mark.parametrize(
116
- "quant_type",
117
- [
118
- # k-quants
119
- GGMLQuantizationType.Q2_K,
120
- GGMLQuantizationType.Q3_K,
121
- GGMLQuantizationType.Q4_K,
122
- GGMLQuantizationType.Q5_K,
123
- GGMLQuantizationType.Q6_K,
124
- # standard quants
125
- GGMLQuantizationType.Q4_0,
126
- GGMLQuantizationType.Q5_0,
127
- GGMLQuantizationType.Q8_0,
128
- ],
129
-)
130
-@torch.inference_mode()
131
-def test_mmq(
132
- num_tokens: int,
133
- hidden_size: int,
134
- dtype: torch.dtype,
135
- quant_type: GGMLQuantizationType,
136
-):
137
- set_random_seed(0)
138
-
139
- tensors = get_gguf_sample_tensors(hidden_size, quant_type)
140
- x = torch.rand((num_tokens, hidden_size), dtype=dtype, device="cuda")
141
- for tensor in tensors:
142
- weight = torch.tensor(dequantize(tensor.data, quant_type), device="cuda").to(
143
- dtype
144
- )
145
- ref_output = x @ weight.T
146
-
147
- qweight = torch.tensor(tensor.data, device="cuda")
148
- output = ops.ggml_mul_mat_a8(qweight, x, quant_type, qweight.shape[0])
149
- atols = {torch.half: 1, torch.bfloat16: 1.5, torch.float: 1.2}
150
- # test matrix has inputs centered around 0 and lower precision from
151
- # bfloat16 tends to accumulate and can greatly inflate rtol
152
- # since outputs are also very close to 0
153
- rtols = {torch.half: 1e-1, torch.bfloat16: 1e4, torch.float: 2e1}
154
- torch.testing.assert_close(
155
- output, ref_output, atol=atols[dtype], rtol=rtols[dtype]
156
- )
157
-
158
-
159
-@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
160
-@pytest.mark.parametrize("hidden_size", [512])
161
-@pytest.mark.parametrize("top_k", [4, 8])
162
-@pytest.mark.parametrize("dtype", DTYPES)
163
-@pytest.mark.parametrize("quant_type", QUANT_TYPES)
164
-@torch.inference_mode()
165
-def test_moe(
166
- num_tokens: int,
167
- hidden_size: int,
168
- dtype: torch.dtype,
169
- quant_type: GGMLQuantizationType,
170
- top_k: int,
171
-):
172
- set_random_seed(0)
173
- H, E = 1024, 256
174
-
175
- x = torch.rand((num_tokens, H), dtype=dtype, device="cuda")
176
-
177
- topk_weights = torch.rand(num_tokens, top_k, device="cuda", dtype=dtype)
178
- topk_ids = torch.randint(
179
- 0, E, (num_tokens, top_k), device="cuda", dtype=torch.int32
180
- )
181
-
182
- tensors = get_gguf_MoE_tensors(hidden_size, quant_type)
183
-
184
- w13 = tensors[0]
185
- w2 = tensors[1]
186
-
187
- w13_dequant = torch.tensor(dequantize(w13.data, quant_type), device="cuda").to(
188
- dtype
189
- )
190
-
191
- w2_dequant = torch.tensor(dequantize(w2.data, quant_type), device="cuda").to(dtype)
192
-
193
- output = _fused_moe_gguf(
194
- x,
195
- torch.tensor(w13.data, device="cuda"),
196
- torch.tensor(w2.data, device="cuda"),
197
- topk_weights,
198
- topk_ids,
199
- quant_type,
200
- quant_type,
201
- "silu",
202
- )
203
-
204
- ref_output = fused_experts(
205
- x, w13_dequant, w2_dequant, topk_weights, topk_ids
206
- ).reshape(output.shape)
207
- torch.testing.assert_close(output, ref_output, atol=1, rtol=1e-1)
tests/models/test_gguf_download.py
deleted
-224
@@ -1,224 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from unittest.mock import MagicMock, patch
5
-
6
-import pytest
7
-
8
-from vllm.config import ModelConfig
9
-from vllm.config.load import LoadConfig
10
-from vllm.model_executor.model_loader.gguf_loader import GGUFModelLoader
11
-from vllm.model_executor.model_loader.weight_utils import download_gguf
12
-
13
-
14
-class TestGGUFDownload:
15
- """Test GGUF model downloading functionality."""
16
-
17
- @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
18
- def test_download_gguf_single_file(self, mock_download):
19
- """Test downloading a single GGUF file."""
20
- # Setup mock
21
- mock_folder = "/tmp/mock_cache"
22
- mock_download.return_value = mock_folder
23
-
24
- # Mock glob to return a single file
25
- with patch("glob.glob") as mock_glob:
26
- mock_glob.side_effect = lambda pattern, **kwargs: (
27
- [f"{mock_folder}/model-IQ1_S.gguf"] if "IQ1_S" in pattern else []
28
- )
29
-
30
- result = download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
31
-
32
- # Verify download_weights_from_hf was called with correct patterns
33
- mock_download.assert_called_once_with(
34
- model_name_or_path="unsloth/Qwen3-0.6B-GGUF",
35
- cache_dir=None,
36
- allow_patterns=[
37
- "*-IQ1_S.gguf",
38
- "*-IQ1_S-*.gguf",
39
- "*/*-IQ1_S.gguf",
40
- "*/*-IQ1_S-*.gguf",
41
- ],
42
- revision=None,
43
- ignore_patterns=None,
44
- )
45
-
46
- # Verify result is the file path, not folder
47
- assert result == f"{mock_folder}/model-IQ1_S.gguf"
48
-
49
- @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
50
- def test_download_gguf_sharded_files(self, mock_download):
51
- """Test downloading sharded GGUF files."""
52
- mock_folder = "/tmp/mock_cache"
53
- mock_download.return_value = mock_folder
54
-
55
- # Mock glob to return sharded files
56
- with patch("glob.glob") as mock_glob:
57
- mock_glob.side_effect = lambda pattern, **kwargs: (
58
- [
59
- f"{mock_folder}/model-Q2_K-00001-of-00002.gguf",
60
- f"{mock_folder}/model-Q2_K-00002-of-00002.gguf",
61
- ]
62
- if "Q2_K" in pattern
63
- else []
64
- )
65
-
66
- result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
67
-
68
- # Should return the first file after sorting
69
- assert result == f"{mock_folder}/model-Q2_K-00001-of-00002.gguf"
70
-
71
- @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
72
- def test_download_gguf_subdir(self, mock_download):
73
- """Test downloading GGUF files from subdirectory."""
74
- mock_folder = "/tmp/mock_cache"
75
- mock_download.return_value = mock_folder
76
-
77
- with patch("glob.glob") as mock_glob:
78
- mock_glob.side_effect = lambda pattern, **kwargs: (
79
- [f"{mock_folder}/Q2_K/model-Q2_K.gguf"]
80
- if "Q2_K" in pattern or "**/*.gguf" in pattern
81
- else []
82
- )
83
-
84
- result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
85
-
86
- assert result == f"{mock_folder}/Q2_K/model-Q2_K.gguf"
87
-
88
- @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
89
- @patch("glob.glob", return_value=[])
90
- def test_download_gguf_no_files_found(self, mock_glob, mock_download):
91
- """Test error when no GGUF files are found."""
92
- mock_folder = "/tmp/mock_cache"
93
- mock_download.return_value = mock_folder
94
-
95
- with pytest.raises(ValueError, match="Downloaded GGUF files not found"):
96
- download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
97
-
98
-
99
-class TestGGUFModelLoader:
100
- """Test GGUFModelLoader class methods."""
101
-
102
- @patch("os.path.isfile", return_value=True)
103
- def test_prepare_weights_local_file(self, mock_isfile):
104
- """Test _prepare_weights with local file."""
105
- load_config = LoadConfig(load_format="gguf")
106
- loader = GGUFModelLoader(load_config)
107
-
108
- # Create a simple mock ModelConfig with only the model attribute
109
- model_config = MagicMock()
110
- model_config.model = "/path/to/model.gguf"
111
-
112
- result = loader._prepare_weights(model_config)
113
- assert result == "/path/to/model.gguf"
114
- mock_isfile.assert_called_once_with("/path/to/model.gguf")
115
-
116
- @patch("vllm.model_executor.model_loader.gguf_loader.hf_hub_download")
117
- @patch("os.path.isfile", return_value=False)
118
- def test_prepare_weights_repo_filename(self, mock_isfile, mock_hf_download):
119
- """Test _prepare_weights with repo_id/filename.gguf format."""
120
- load_config = LoadConfig(load_format="gguf")
121
- loader = GGUFModelLoader(load_config)
122
-
123
- mock_hf_download.return_value = "/downloaded/model.gguf"
124
-
125
- model_config = MagicMock()
126
- model_config.model = "unsloth/Qwen3-0.6B-GGUF/model.gguf"
127
- model_config.revision = "abc123"
128
-
129
- result = loader._prepare_weights(model_config)
130
- assert result == "/downloaded/model.gguf"
131
- mock_hf_download.assert_called_once_with(
132
- repo_id="unsloth/Qwen3-0.6B-GGUF",
133
- filename="model.gguf",
134
- revision="abc123",
135
- cache_dir=None,
136
- )
137
-
138
- @patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
139
- @patch("vllm.transformers_utils.config.file_or_path_exists", return_value=True)
140
- @patch("vllm.config.model.get_config")
141
- @patch("vllm.config.model.is_gguf", return_value=True)
142
- @patch("vllm.model_executor.model_loader.gguf_loader.download_gguf")
143
- @patch("os.path.isfile", return_value=False)
144
- def test_prepare_weights_repo_quant_type(
145
- self,
146
- mock_isfile,
147
- mock_download_gguf,
148
- mock_is_gguf,
149
- mock_get_config,
150
- mock_file_exists,
151
- mock_get_image_config,
152
- ):
153
- """Test _prepare_weights with repo_id:quant_type format."""
154
- mock_hf_config = MagicMock()
155
- mock_hf_config.architectures = ["Qwen3ForCausalLM"]
156
-
157
- class MockTextConfig:
158
- max_position_embeddings = 4096
159
- sliding_window = None
160
- model_type = "qwen3"
161
- num_attention_heads = 32
162
-
163
- mock_text_config = MockTextConfig()
164
- mock_hf_config.get_text_config.return_value = mock_text_config
165
- mock_hf_config.dtype = "bfloat16"
166
- mock_get_config.return_value = mock_hf_config
167
-
168
- load_config = LoadConfig(load_format="gguf")
169
- loader = GGUFModelLoader(load_config)
170
-
171
- mock_download_gguf.return_value = "/downloaded/model-IQ1_S.gguf"
172
-
173
- model_config = ModelConfig(
174
- model="unsloth/Qwen3-0.6B-GGUF:IQ1_S", tokenizer="Qwen/Qwen3-0.6B"
175
- )
176
- result = loader._prepare_weights(model_config)
177
- # The actual result will be the downloaded file path from mock
178
- assert result == "/downloaded/model-IQ1_S.gguf"
179
- mock_download_gguf.assert_called_once_with(
180
- "unsloth/Qwen3-0.6B-GGUF",
181
- "IQ1_S",
182
- cache_dir=None,
183
- revision=None,
184
- ignore_patterns=["original/**/*"],
185
- )
186
-
187
- @patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
188
- @patch("vllm.config.model.get_config")
189
- @patch("vllm.config.model.is_gguf", return_value=False)
190
- @patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=False)
191
- @patch("os.path.isfile", return_value=False)
192
- def test_prepare_weights_invalid_format(
193
- self,
194
- mock_isfile,
195
- mock_check_gguf,
196
- mock_is_gguf,
197
- mock_get_config,
198
- mock_get_image_config,
199
- ):
200
- """Test _prepare_weights with invalid format."""
201
- mock_hf_config = MagicMock()
202
- mock_hf_config.architectures = ["Qwen3ForCausalLM"]
203
-
204
- class MockTextConfig:
205
- max_position_embeddings = 4096
206
- sliding_window = None
207
- model_type = "qwen3"
208
- num_attention_heads = 32
209
-
210
- mock_text_config = MockTextConfig()
211
- mock_hf_config.get_text_config.return_value = mock_text_config
212
- mock_hf_config.dtype = "bfloat16"
213
- mock_get_config.return_value = mock_hf_config
214
-
215
- load_config = LoadConfig(load_format="gguf")
216
- loader = GGUFModelLoader(load_config)
217
-
218
- # Create ModelConfig with a valid repo_id to avoid validation errors
219
- # Then test _prepare_weights with invalid format
220
- model_config = ModelConfig(model="unsloth/Qwen3-0.6B")
221
- # Manually set model to invalid format after creation
222
- model_config.model = "invalid-format"
223
- with pytest.raises(ValueError, match="Unrecognised GGUF reference"):
224
- loader._prepare_weights(model_config)
tests/plugins_tests/gguf/__init__.py
tests/plugins_tests/gguf/test_gguf_plugin_generate.py
renamed
+12
-84
@@ -1,23 +1,18 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""
4
-Tests gguf models against unquantized models generations
5
-Note: To pass the test, quantization higher than Q4 should be used
4
+E2E tests for GGUF plugin functionality.
5
"""
6
7
import os
8
from typing import NamedTuple
9
10
import pytest
12
-from huggingface_hub import hf_hub_download
13
-from pytest import MarkDecorator
11
from transformers import AutoTokenizer
12
16
-from tests.quantization.utils import is_quant_method_supported
17
-
13
from ...conftest import VllmRunner
14
+from ...models.utils import check_logprobs_close
15
from ...utils import multi_gpu_test
20
-from ..utils import check_logprobs_close
16
17
os.environ["TOKENIZERS_PARALLELISM"] = "true"
18
@@ -26,80 +21,24 @@ MAX_MODEL_LEN = 1024
21
22
class GGUFTestConfig(NamedTuple):
23
original_model: str
29
- gguf_repo: str
30
- gguf_filename: str
31
- marks: list[MarkDecorator] = []
32
-
33
- @property
34
- def gguf_model(self):
35
- return hf_hub_download(self.gguf_repo, filename=self.gguf_filename)
24
+ gguf_model_path: str # Full path to .gguf file
25
26
38
-LLAMA_CONFIG = GGUFTestConfig(
39
- original_model="meta-llama/Llama-3.2-1B-Instruct",
40
- gguf_repo="bartowski/Llama-3.2-1B-Instruct-GGUF",
41
- gguf_filename="Llama-3.2-1B-Instruct-Q6_K.gguf",
42
-)
43
-
44
-QWEN2_CONFIG = GGUFTestConfig(
45
- original_model="Qwen/Qwen2.5-1.5B-Instruct",
46
- gguf_repo="Qwen/Qwen2.5-1.5B-Instruct-GGUF",
47
- gguf_filename="qwen2.5-1.5b-instruct-q6_k.gguf",
48
-)
49
-
27
QWEN3_CONFIG = GGUFTestConfig(
28
original_model="Qwen/Qwen3-0.6B",
52
- gguf_repo="unsloth/Qwen3-0.6B-GGUF",
53
- gguf_filename="Qwen3-0.6B-BF16.gguf",
54
-)
55
-
56
-PHI3_CONFIG = GGUFTestConfig(
57
- original_model="microsoft/Phi-3.5-mini-instruct",
58
- gguf_repo="bartowski/Phi-3.5-mini-instruct-GGUF",
59
- gguf_filename="Phi-3.5-mini-instruct-IQ4_XS.gguf",
60
-)
61
-
62
-GPT2_CONFIG = GGUFTestConfig(
63
- original_model="openai-community/gpt2-large",
64
- gguf_repo="QuantFactory/gpt2-large-GGUF",
65
- gguf_filename="gpt2-large.Q4_K_M.gguf",
29
+ gguf_model_path="unsloth/Qwen3-0.6B-GGUF:Q8_0",
30
)
31
68
-STABLELM_CONFIG = GGUFTestConfig(
69
- original_model="stabilityai/stablelm-3b-4e1t",
70
- gguf_repo="afrideva/stablelm-3b-4e1t-GGUF",
71
- gguf_filename="stablelm-3b-4e1t.q4_k_m.gguf",
72
-)
32
74
-STARCODER_CONFIG = GGUFTestConfig(
75
- original_model="bigcode/starcoder2-3b",
76
- gguf_repo="QuantFactory/starcoder2-3b-GGUF",
77
- gguf_filename="starcoder2-3b.Q6_K.gguf",
33
+OLMOE_CONFIG = GGUFTestConfig(
34
+ original_model="allenai/OLMoE-1B-7B-0125",
35
+ gguf_model_path="allenai/OLMoE-1B-7B-0125-GGUF:Q6_K",
36
)
37
80
-DOLPHIN_CONFIG = GGUFTestConfig(
81
- # Test VocabParallelEmbedding sharding issue.
82
- original_model="cognitivecomputations/TinyDolphin-2.8-1.1b",
83
- gguf_repo="tsunemoto/TinyDolphin-2.8-1.1b-GGUF",
84
- gguf_filename="tinydolphin-2.8-1.1b.Q6_K.gguf",
85
-)
86
-
87
-GEMMA3_CONFIG = GGUFTestConfig(
88
- original_model="google/gemma-3-270m-it",
89
- gguf_repo="ggml-org/gemma-3-270m-it-qat-GGUF",
90
- gguf_filename="gemma-3-270m-it-qat-Q4_0.gguf",
91
-)
38
39
MODELS = [
94
- # LLAMA_CONFIG, # broken: https://github.com/vllm-project/vllm/issues/19458
95
- QWEN2_CONFIG,
40
QWEN3_CONFIG,
97
- PHI3_CONFIG,
98
- GPT2_CONFIG,
99
- STABLELM_CONFIG,
100
- DOLPHIN_CONFIG,
101
- GEMMA3_CONFIG,
102
- # STARCODER_CONFIG, # broken
41
+ OLMOE_CONFIG,
42
]
43
44
@@ -121,7 +60,7 @@ def check_model_outputs(
60
61
# Run gguf model.
62
with vllm_runner(
124
- model_name=model.gguf_model,
63
+ model_name=model.gguf_model_path,
64
enforce_eager=True,
65
tokenizer_name=model.original_model,
66
dtype=dtype,
@@ -154,17 +93,10 @@ def check_model_outputs(
93
)
94
95
157
-@pytest.mark.skipif(
158
- not is_quant_method_supported("gguf"),
159
- reason="gguf is not supported on this GPU type.",
160
-)
161
-@pytest.mark.parametrize(
162
- "model",
163
- [pytest.param(test_config, marks=test_config.marks) for test_config in MODELS],
164
-)
96
+@pytest.mark.parametrize("model", MODELS)
97
@pytest.mark.parametrize("dtype", ["bfloat16"])
98
@pytest.mark.parametrize("max_tokens", [32])
167
-@pytest.mark.parametrize("num_logprobs", [5])
99
+@pytest.mark.parametrize("num_logprobs", [8])
100
@pytest.mark.parametrize("tp_size", [1])
101
def test_models(
102
vllm_runner: type[VllmRunner],
@@ -180,11 +112,7 @@ def test_models(
112
)
113
114
183
-@pytest.mark.skipif(
184
- not is_quant_method_supported("gguf"),
185
- reason="gguf is not supported on this GPU type.",
186
-)
187
-@pytest.mark.parametrize("model", [LLAMA_CONFIG])
115
+@pytest.mark.parametrize("model", MODELS)
116
@pytest.mark.parametrize("dtype", ["half"])
117
@pytest.mark.parametrize("max_tokens", [8])
118
@pytest.mark.parametrize("num_logprobs", [5])
tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py
renamed
+6
-19
@@ -12,13 +12,12 @@ from huggingface_hub import hf_hub_download
12
from pytest import MarkDecorator
13
from transformers import AutoModelForImageTextToText
14
15
-from tests.quantization.utils import is_quant_method_supported
15
from vllm.assets.image import ImageAsset
16
from vllm.multimodal.image import rescale_image_size
17
from vllm.utils.torch_utils import set_default_torch_num_threads
18
20
-from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner
21
-from ...utils import check_logprobs_close
19
+from ...conftest import IMAGE_ASSETS, HfRunner, VllmRunner
20
+from ...models.utils import check_logprobs_close
21
22
23
class GGUFMMTestConfig(NamedTuple):
@@ -66,20 +65,18 @@ GEMMA3_CONFIG = GGUFMMTestConfig(
65
prompt=_GEMMA3_PROMPTS,
66
image_names=_GEMMA3_IMAGE_NAMES,
67
max_model_len=4096,
69
- marks=[pytest.mark.core_model],
68
mm_processor_kwargs={},
69
)
70
71
# Pan-and-scan multimodal - uses unquantized BF16 GGUF
72
GEMMA3_CONFIG_PAN_AND_SCAN = GGUFMMTestConfig(
73
original_model="google/gemma-3-4b-it",
76
- gguf_repo="unsloth/gemma-3-4b-it-GGUF",
77
- gguf_backbone="gemma-3-4b-it-BF16.gguf",
78
- gguf_mmproj="mmproj-BF16.gguf",
74
+ gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
75
+ gguf_backbone="gemma-3-4b-it-q4_0.gguf",
76
+ gguf_mmproj="mmproj-model-f16-4B.gguf",
77
prompt=_GEMMA3_PROMPTS,
78
image_names=_GEMMA3_IMAGE_NAMES,
79
max_model_len=4096,
82
- marks=[pytest.mark.core_model],
80
mm_processor_kwargs={"do_pan_and_scan": True},
81
)
82
@@ -153,17 +150,7 @@ def run_multimodal_gguf_test(
150
)
151
152
156
-@pytest.mark.skipif(
157
- not is_quant_method_supported("gguf"),
158
- reason="gguf is not supported on this GPU type.",
159
-)
160
-@pytest.mark.parametrize(
161
- "model",
162
- [
163
- pytest.param(test_config, marks=test_config.marks)
164
- for test_config in MODELS_TO_TEST
165
- ],
166
-)
153
+@pytest.mark.parametrize("model", MODELS_TO_TEST)
154
@pytest.mark.parametrize("dtype", ["bfloat16"])
155
@pytest.mark.parametrize("max_tokens", [32])
156
@pytest.mark.parametrize("num_logprobs", [10])
tests/transformers_utils/test_utils.py
-210
@@ -1,15 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from pathlib import Path
4
-from unittest.mock import patch
5
-
6
-import pytest
7
-
8
-from vllm.transformers_utils.gguf_utils import (
9
- is_gguf,
10
- is_remote_gguf,
11
- split_remote_gguf,
12
-)
3
from vllm.transformers_utils.utils import (
4
is_azure,
5
is_cloud_storage,
@@ -45,203 +35,3 @@ def test_is_cloud_storage():
35
assert is_cloud_storage("az://model-container/path")
36
assert not is_cloud_storage("/unix/local/path")
37
assert not is_cloud_storage("nfs://nfs-fqdn.local")
48
-
49
-
50
-class TestIsRemoteGGUF:
51
- """Test is_remote_gguf utility function."""
52
-
53
- def test_is_remote_gguf_with_colon_and_slash(self):
54
- """Test is_remote_gguf with repo_id:quant_type format."""
55
- # Valid quant types (exact GGML types)
56
- assert is_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
57
- assert is_remote_gguf("user/repo:Q2_K")
58
- assert is_remote_gguf("repo/model:Q4_K")
59
- assert is_remote_gguf("repo/model:Q8_0")
60
-
61
- # Invalid quant types should return False
62
- assert not is_remote_gguf("repo/model:quant")
63
- assert not is_remote_gguf("repo/model:INVALID")
64
- assert not is_remote_gguf("repo/model:invalid_type")
65
-
66
- def test_is_remote_gguf_extended_quant_types(self):
67
- """Test is_remote_gguf with extended quant type naming conventions."""
68
- # Extended quant types with _M, _S, _L suffixes
69
- assert is_remote_gguf("repo/model:Q4_K_M")
70
- assert is_remote_gguf("repo/model:Q4_K_S")
71
- assert is_remote_gguf("repo/model:Q3_K_L")
72
- assert is_remote_gguf("repo/model:Q5_K_M")
73
- assert is_remote_gguf("repo/model:Q3_K_S")
74
-
75
- # Extended quant types with _XL, _XS, _XXS suffixes
76
- assert is_remote_gguf("repo/model:Q5_K_XL")
77
- assert is_remote_gguf("repo/model:IQ4_XS")
78
- assert is_remote_gguf("repo/model:IQ3_XXS")
79
-
80
- # Invalid extended types (base type doesn't exist)
81
- assert not is_remote_gguf("repo/model:INVALID_M")
82
- assert not is_remote_gguf("repo/model:Q9_K_M")
83
-
84
- def test_is_remote_gguf_nonstandard_quant_type(self):
85
- """Test is_remote_gguf with non-standard quant types containing
86
- a known GGML type."""
87
- # Non-standard quant types with known GGML type after prefix
88
- assert is_remote_gguf("unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL")
89
- assert is_remote_gguf("user/Model:UD-Q4_K_M")
90
- assert is_remote_gguf("user/SomeModel:Custom-Q8_0")
91
-
92
- # Exact GGML type after prefix (no suffix stripping needed)
93
- assert is_remote_gguf("user/Model-GGUF:UD-IQ4_NL")
94
- assert is_remote_gguf("user/Model-GGUF:UD-Q8_0")
95
-
96
- # Completely unknown quant types should still fail
97
- assert not is_remote_gguf("repo/model:TOTALLY-RANDOM")
98
- assert not is_remote_gguf("user/Model:UD-INVALID")
99
-
100
- # No dash separator → not recognized as prefixed
101
- assert not is_remote_gguf("repo/model:UDIQ4NL")
102
-
103
- def test_is_remote_gguf_without_colon(self):
104
- """Test is_remote_gguf without colon."""
105
- assert not is_remote_gguf("repo/model")
106
- assert not is_remote_gguf("unsloth/Qwen3-0.6B-GGUF")
107
-
108
- def test_is_remote_gguf_without_slash(self):
109
- """Test is_remote_gguf without slash."""
110
- assert not is_remote_gguf("model.gguf")
111
- # Even with valid quant_type, no slash means not remote GGUF
112
- assert not is_remote_gguf("model:IQ1_S")
113
- assert not is_remote_gguf("model:quant")
114
-
115
- def test_is_remote_gguf_local_path(self):
116
- """Test is_remote_gguf with local file path."""
117
- assert not is_remote_gguf("/path/to/model.gguf")
118
- assert not is_remote_gguf("./model.gguf")
119
-
120
- def test_is_remote_gguf_with_path_object(self):
121
- """Test is_remote_gguf with Path object."""
122
- assert is_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
123
- assert not is_remote_gguf(Path("repo/model"))
124
-
125
- def test_is_remote_gguf_with_http_https(self):
126
- """Test is_remote_gguf with HTTP/HTTPS URLs."""
127
- # HTTP/HTTPS URLs should return False even with valid quant_type
128
- assert not is_remote_gguf("http://example.com/repo/model:IQ1_S")
129
- assert not is_remote_gguf("https://huggingface.co/repo/model:Q2_K")
130
- assert not is_remote_gguf("http://repo/model:Q4_K")
131
- assert not is_remote_gguf("https://repo/model:Q8_0")
132
-
133
- def test_is_remote_gguf_with_cloud_storage(self):
134
- """Test is_remote_gguf with cloud storage paths."""
135
- # Cloud storage paths should return False even with valid quant_type
136
- assert not is_remote_gguf("s3://bucket/repo/model:IQ1_S")
137
- assert not is_remote_gguf("gs://bucket/repo/model:Q2_K")
138
- assert not is_remote_gguf("s3://repo/model:Q4_K")
139
- assert not is_remote_gguf("gs://repo/model:Q8_0")
140
-
141
-
142
-class TestSplitRemoteGGUF:
143
- """Test split_remote_gguf utility function."""
144
-
145
- def test_split_remote_gguf_valid(self):
146
- """Test split_remote_gguf with valid repo_id:quant_type format."""
147
- repo_id, quant_type = split_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
148
- assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
149
- assert quant_type == "IQ1_S"
150
-
151
- repo_id, quant_type = split_remote_gguf("repo/model:Q2_K")
152
- assert repo_id == "repo/model"
153
- assert quant_type == "Q2_K"
154
-
155
- def test_split_remote_gguf_extended_quant_types(self):
156
- """Test split_remote_gguf with extended quant type naming conventions."""
157
- repo_id, quant_type = split_remote_gguf("unsloth/Qwen3-0.6B-GGUF:Q4_K_M")
158
- assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
159
- assert quant_type == "Q4_K_M"
160
-
161
- repo_id, quant_type = split_remote_gguf("repo/model:Q3_K_S")
162
- assert repo_id == "repo/model"
163
- assert quant_type == "Q3_K_S"
164
-
165
- def test_split_remote_gguf_nonstandard_quant_type(self):
166
- """Test split_remote_gguf with non-standard quant types in GGUF repos."""
167
- repo_id, quant_type = split_remote_gguf(
168
- "unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL"
169
- )
170
- assert repo_id == "unsloth/Qwen3.5-35B-A3B-GGUF"
171
- assert quant_type == "UD-Q4_K_XL"
172
-
173
- def test_split_remote_gguf_with_path_object(self):
174
- """Test split_remote_gguf with Path object."""
175
- repo_id, quant_type = split_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
176
- assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
177
- assert quant_type == "IQ1_S"
178
-
179
- def test_split_remote_gguf_invalid(self):
180
- """Test split_remote_gguf with invalid format."""
181
- # Invalid format (no colon) - is_remote_gguf returns False
182
- with pytest.raises(ValueError, match="Wrong GGUF model"):
183
- split_remote_gguf("repo/model")
184
-
185
- # Invalid quant type - is_remote_gguf returns False
186
- with pytest.raises(ValueError, match="Wrong GGUF model"):
187
- split_remote_gguf("repo/model:INVALID_TYPE")
188
-
189
- # HTTP URL - is_remote_gguf returns False
190
- with pytest.raises(ValueError, match="Wrong GGUF model"):
191
- split_remote_gguf("http://repo/model:IQ1_S")
192
-
193
- # Cloud storage - is_remote_gguf returns False
194
- with pytest.raises(ValueError, match="Wrong GGUF model"):
195
- split_remote_gguf("s3://bucket/repo/model:Q2_K")
196
-
197
-
198
-class TestIsGGUF:
199
- """Test is_gguf utility function."""
200
-
201
- @patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=True)
202
- def test_is_gguf_with_local_file(self, mock_check_gguf):
203
- """Test is_gguf with local GGUF file."""
204
- assert is_gguf("/path/to/model.gguf")
205
- assert is_gguf("./model.gguf")
206
-
207
- def test_is_gguf_with_remote_gguf(self):
208
- """Test is_gguf with remote GGUF format."""
209
- # Valid remote GGUF format (repo_id:quant_type with valid quant_type)
210
- assert is_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
211
- assert is_gguf("repo/model:Q2_K")
212
- assert is_gguf("repo/model:Q4_K")
213
-
214
- # Extended quant types with suffixes
215
- assert is_gguf("repo/model:Q4_K_M")
216
- assert is_gguf("repo/model:Q3_K_S")
217
- assert is_gguf("repo/model:Q5_K_L")
218
-
219
- # Invalid quant_type should return False
220
- assert not is_gguf("repo/model:quant")
221
- assert not is_gguf("repo/model:INVALID")
222
-
223
- @patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=False)
224
- def test_is_gguf_false(self, mock_check_gguf):
225
- """Test is_gguf returns False for non-GGUF models."""
226
- assert not is_gguf("unsloth/Qwen3-0.6B")
227
- assert not is_gguf("repo/model")
228
- assert not is_gguf("model")
229
-
230
- def test_is_gguf_edge_cases(self):
231
- """Test is_gguf with edge cases."""
232
- # Empty string
233
- assert not is_gguf("")
234
-
235
- # Only colon, no slash (even with valid quant_type)
236
- assert not is_gguf("model:IQ1_S")
237
-
238
- # Only slash, no colon
239
- assert not is_gguf("repo/model")
240
-
241
- # HTTP/HTTPS URLs
242
- assert not is_gguf("http://repo/model:IQ1_S")
243
- assert not is_gguf("https://repo/model:Q2_K")
244
-
245
- # Cloud storage
246
- assert not is_gguf("s3://bucket/repo/model:IQ1_S")
247
- assert not is_gguf("gs://bucket/repo/model:Q2_K")
vllm/_custom_ops.py
-128
@@ -768,69 +768,6 @@ if hasattr(torch.ops._C, "allspark_w8a16_gemm"):
768
return torch.empty((m, n), device=a.device, dtype=a.dtype)
769
770
771
-if hasattr(torch.ops._C, "ggml_dequantize"):
772
-
773
- @register_fake("_C::ggml_dequantize")
774
- def _ggml_dequantize_fake(
775
- W: torch.Tensor,
776
- quant_type: int,
777
- m: torch.SymInt,
778
- n: torch.SymInt,
779
- dtype: torch.dtype | None = None,
780
- ) -> torch.Tensor:
781
- return torch.empty((m, n), dtype=torch.float16, device=W.device)
782
-
783
- @register_fake("_C::ggml_mul_mat_vec_a8")
784
- def _ggml_mul_mat_vec_a8_fake(
785
- W: torch.Tensor,
786
- X: torch.Tensor,
787
- quant_type: int,
788
- row: torch.SymInt,
789
- ) -> torch.Tensor:
790
- return torch.empty((X.shape[0], row), dtype=X.dtype, device=W.device)
791
-
792
- @register_fake("_C::ggml_mul_mat_a8")
793
- def _ggml_mul_mat_a8_fake(
794
- W: torch.Tensor,
795
- X: torch.Tensor,
796
- quant_type: int,
797
- row: torch.SymInt,
798
- ) -> torch.Tensor:
799
- batch = X.size(0)
800
- return torch.empty((batch, row), dtype=X.dtype, device=W.device)
801
-
802
- @register_fake("_C::ggml_moe_a8")
803
- def _ggml_moe_a8_fake(
804
- X: torch.Tensor,
805
- W: torch.Tensor,
806
- sorted_token_ids: torch.Tensor,
807
- expert_ids: torch.Tensor,
808
- num_tokens_post_padded: torch.Tensor,
809
- quant_type: int,
810
- row: torch.SymInt,
811
- top_k: torch.SymInt,
812
- tokens: torch.SymInt,
813
- ) -> torch.Tensor:
814
- tokens = X.size(0)
815
- return torch.empty((tokens * top_k, row), dtype=torch.float16, device=W.device)
816
-
817
-
818
-if hasattr(torch.ops._C, "ggml_moe_a8_vec"):
819
-
820
- @register_fake("_C::ggml_moe_a8_vec")
821
- def _ggml_moe_a8_vec_fake(
822
- X: torch.Tensor,
823
- W: torch.Tensor,
824
- topk_ids: torch.Tensor,
825
- top_k: int,
826
- quant_type: int,
827
- row: torch.SymInt,
828
- tokens: torch.SymInt,
829
- ) -> torch.Tensor:
830
- tokens = X.size(0)
831
- return torch.empty((tokens * top_k, row), dtype=X.dtype, device=W.device)
832
-
833
-
771
# cutlass
772
def cutlass_scaled_mm_supports_fp4(cuda_device_capability: int) -> bool:
773
return torch.ops._C.cutlass_scaled_mm_supports_fp4(cuda_device_capability)
@@ -2195,71 +2132,6 @@ def scaled_int8_quant(
2132
return output, input_scales, input_azp
2133
2134
2198
-# gguf
2199
-def ggml_dequantize(
2200
- W: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None
2201
-) -> torch.Tensor:
2202
- return torch.ops._C.ggml_dequantize(W, quant_type, m, n, dtype)
2203
-
2204
-
2205
-def ggml_mul_mat_vec_a8(
2206
- W: torch.Tensor,
2207
- X: torch.Tensor,
2208
- quant_type: int,
2209
- row: int,
2210
-) -> torch.Tensor:
2211
- return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
2212
-
2213
-
2214
-def ggml_mul_mat_a8(
2215
- W: torch.Tensor,
2216
- X: torch.Tensor,
2217
- quant_type: int,
2218
- row: int,
2219
-) -> torch.Tensor:
2220
- return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
2221
-
2222
-
2223
-def ggml_moe_a8(
2224
- X: torch.Tensor,
2225
- W: torch.Tensor,
2226
- sorted_token_ids: torch.Tensor,
2227
- expert_ids: torch.Tensor,
2228
- num_tokens_post_padded: torch.Tensor,
2229
- quant_type: int,
2230
- row: int,
2231
- top_k: int,
2232
- tokens: int,
2233
-) -> torch.Tensor:
2234
- return torch.ops._C.ggml_moe_a8(
2235
- X,
2236
- W,
2237
- sorted_token_ids,
2238
- expert_ids,
2239
- num_tokens_post_padded,
2240
- quant_type,
2241
- row,
2242
- top_k,
2243
- tokens,
2244
- )
2245
-
2246
-
2247
-def ggml_moe_a8_vec(
2248
- X: torch.Tensor,
2249
- W: torch.Tensor,
2250
- topk_ids: torch.Tensor,
2251
- top_k: int,
2252
- quant_type: int,
2253
- row: torch.SymInt,
2254
- tokens: torch.SymInt,
2255
-) -> torch.Tensor:
2256
- return torch.ops._C.ggml_moe_a8_vec(X, W, topk_ids, top_k, quant_type, row, tokens)
2257
-
2258
-
2259
-def ggml_moe_get_block_size(quant_type: int) -> int:
2260
- return torch.ops._C.ggml_moe_get_block_size(quant_type)
2261
-
2262
-
2135
# mamba
2136
def selective_scan_fwd(
2137
u: torch.Tensor,
vllm/config/load.py
-2
@@ -51,8 +51,6 @@ class LoadConfig:
51
- "bitsandbytes" will load the weights using bitsandbytes quantization.
52
- "sharded_state" will load weights from pre-sharded checkpoint files,
53
supporting efficient loading of tensor-parallel models.
54
- - "gguf" will load weights from GGUF format files (details specified in
55
- https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).
54
- "mistral" will load weights from consolidated safetensors files used by
55
Mistral models.
56
- "modelexpress" will load weights using ModelExpress.
vllm/config/model.py
+1
-24
@@ -42,12 +42,6 @@ from vllm.transformers_utils.config import (
42
uses_mrope,
43
uses_xdrope_dim,
44
)
45
-from vllm.transformers_utils.gguf_utils import (
46
- is_gguf,
47
- is_remote_gguf,
48
- maybe_patch_hf_config_from_gguf,
49
- split_remote_gguf,
50
-)
45
from vllm.transformers_utils.model_arch_config_convertor import (
46
MODEL_ARCH_CONFIG_CONVERTORS,
47
ModelArchConfigConvertorBase,
@@ -547,11 +541,6 @@ class ModelConfig:
541
hf_overrides_fn=hf_overrides_fn,
542
token=self.hf_token,
543
)
550
- hf_config = maybe_patch_hf_config_from_gguf(
551
- self.model,
552
- hf_config,
553
- )
554
-
544
self.hf_config = hf_config
545
if dict_overrides:
546
self._apply_dict_overrides(hf_config, dict_overrides)
@@ -724,14 +713,6 @@ class ModelConfig:
713
"disable the cache with --mm-processor-cache-gb 0."
714
)
715
727
- # Multimodal GGUF models must use original repo for mm processing
728
- if is_gguf(self.tokenizer) and self.is_multimodal_model:
729
- raise ValueError(
730
- "Loading a multimodal GGUF model needs to use original "
731
- "tokenizer. Please specify the unquantized hf model's "
732
- "repo name or path using the --tokenizer argument."
733
- )
734
-
716
if self.disable_sliding_window:
717
# Set after get_and_verify_max_len to ensure that max_model_len
718
# can be correctly capped to sliding window size
@@ -884,10 +865,7 @@ class ModelConfig:
865
self.tokenizer = object_storage_tokenizer.dir
866
867
def _get_encoder_config(self) -> dict[str, Any] | None:
887
- model = self.model
888
- if is_remote_gguf(model):
889
- model, _ = split_remote_gguf(model)
890
- return get_sentence_transformer_tokenizer_config(model, self.revision)
868
+ return get_sentence_transformer_tokenizer_config(self.model, self.revision)
869
870
def _get_default_runner_type(
871
self,
@@ -1019,7 +997,6 @@ class ModelConfig:
997
"gpt_oss_mxfp4",
998
"deepseek_v4_fp8",
999
"humming",
1022
- "gguf",
1000
]
1001
# if the user specifies humming, we should always use humming
1002
if self.quantization == "humming":
vllm/engine/arg_utils.py
-5
@@ -103,7 +103,6 @@ from vllm.transformers_utils.config import (
103
is_interleaved,
104
maybe_override_with_speculators,
105
)
106
-from vllm.transformers_utils.gguf_utils import is_gguf
106
from vllm.transformers_utils.repo_utils import get_model_path
107
from vllm.transformers_utils.utils import is_cloud_storage
108
from vllm.utils.argparse_utils import (
@@ -1558,10 +1557,6 @@ class EngineArgs:
1557
return engine_args
1558
1559
def create_model_config(self) -> ModelConfig:
1561
- # gguf file needs a specific model loader
1562
- if is_gguf(self.model):
1563
- self.quantization = self.load_format = "gguf"
1564
-
1560
if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
1561
logger.warning(
1562
"The global random seed is set to %d. Since "
vllm/model_executor/layers/fused_moe/routed_experts.py
-20
@@ -6,7 +6,6 @@ from enum import Enum
6
from typing import TYPE_CHECKING, Any, Literal, cast, overload
7
8
import torch
9
-from torch.nn.parameter import UninitializedParameter
9
10
from vllm.distributed.eplb.eplb_state import EplbState
11
from vllm.logger import init_logger
@@ -625,13 +624,6 @@ class RoutedExperts(PluggableLayer):
624
# dimension intermediate_size_per_partition is used.
625
SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
626
628
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
629
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
630
- if is_gguf_weight_type:
631
- param.weight_type = loaded_weight.item()
632
- param.data.copy_(loaded_weight)
633
- return True if return_success else None
634
-
627
# Case for BitsAndBytes
628
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
629
if use_bitsandbytes_4bit:
@@ -677,18 +669,6 @@ class RoutedExperts(PluggableLayer):
669
if full_load:
670
shard_dim += 1
671
680
- # Materialize GGUF UninitializedParameter accounting merged weights
681
- if is_gguf_weight and isinstance(param, UninitializedParameter):
682
- # To materialize a tensor, we must have full shape including
683
- # number of experts, making this portion to require `full_load`.
684
- assert full_load
685
- final_shape = list(loaded_weight.shape)
686
- # w1 and w3 are merged per expert.
687
- if shard_id in {"w1", "w3"}:
688
- final_shape[1] *= 2
689
- final_shape[shard_dim] = final_shape[shard_dim] // self.moe_config.tp_size
690
- param.materialize(final_shape, dtype=loaded_weight.dtype)
691
-
672
expert_data = param.data if full_load else param.data[expert_id]
673
674
# Case input scale: input_scale loading is only supported for fp8
vllm/model_executor/layers/linear.py
+1
-96
@@ -5,7 +5,7 @@ import itertools
5
from abc import abstractmethod
6
7
import torch
8
-from torch.nn.parameter import Parameter, UninitializedParameter
8
+from torch.nn.parameter import Parameter
9
10
import vllm.envs as envs
11
from vllm.distributed import (
@@ -360,19 +360,6 @@ class ReplicatedLinear(LinearBase):
360
self.register_parameter("bias", None)
361
362
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
363
- # If the weight on disk does not have a shape, give it one
364
- # (such scales for AutoFp8).
365
- # Special case for GGUF
366
-
367
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
368
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
369
- if is_gguf_weight_type:
370
- param.weight_type = loaded_weight.item()
371
-
372
- # Materialize GGUF UninitializedParameter
373
- if is_gguf_weight and isinstance(param, UninitializedParameter):
374
- param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
375
-
363
if len(loaded_weight.shape) == 0:
364
loaded_weight = loaded_weight.reshape(1)
365
@@ -536,20 +523,6 @@ class ColumnParallelLinear(LinearBase):
523
# no need to narrow
524
is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
525
539
- # Special case for GGUF
540
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
541
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
542
- if is_gguf_weight_type:
543
- param.weight_type = loaded_weight.item()
544
-
545
- # Materialize GGUF UninitializedParameter
546
- if is_gguf_weight and isinstance(param, UninitializedParameter):
547
- final_shape = list(loaded_weight.shape)
548
- if output_dim is not None:
549
- assert final_shape[output_dim] % self.tp_size == 0
550
- final_shape[output_dim] = final_shape[output_dim] // self.tp_size
551
- param.materialize(final_shape, dtype=loaded_weight.dtype)
552
-
526
param_data = param.data
527
if output_dim is not None and not is_sharded_weight:
528
shard_size = param_data.shape[output_dim]
@@ -693,37 +666,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
666
loaded_shard_id: tuple[int, ...] | int | None = None,
667
):
668
self.validate_shard_id(loaded_shard_id)
696
- # Special case for GGUF
697
- # initialize GGUF param after we know the quantize type
698
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
699
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
700
- if isinstance(loaded_shard_id, tuple) and (
701
- is_gguf_weight or is_gguf_weight_type
702
- ):
703
- raise NotImplementedError(
704
- "Shard id with multiple indices is not supported for GGUF."
705
- )
706
- if is_gguf_weight_type:
707
- if loaded_shard_id is not None:
708
- param.data[loaded_shard_id].copy_(loaded_weight)
709
- param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
710
- else:
711
- param.shard_weight_type = {
712
- i: loaded_weight.item() for i, _ in enumerate(self.output_sizes)
713
- }
714
- return
715
-
716
- if is_gguf_weight:
717
- output_dim = getattr(param, "output_dim", None)
718
- shard_size = loaded_weight.size(output_dim) // self.tp_size
719
- start_idx = self.tp_rank * shard_size
720
-
721
- if loaded_shard_id is not None:
722
- loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
723
- param.shard_id.append(loaded_shard_id)
724
- param.shard_id_map[loaded_shard_id] = len(param.data_container)
725
- param.data_container.append(loaded_weight)
726
- return
669
670
param_data = param.data
671
output_dim = getattr(param, "output_dim", None)
@@ -1186,30 +1128,6 @@ class QKVParallelLinear(ColumnParallelLinear):
1128
loaded_shard_id: str | None = None,
1129
):
1130
self.validate_shard_id(loaded_shard_id)
1189
- # Special case for GGUF
1190
- # initialize GGUF param after we know the quantize type
1191
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
1192
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
1193
- if is_gguf_weight_type:
1194
- idx_map = {"q": 0, "k": 1, "v": 2}
1195
- if loaded_shard_id is not None:
1196
- param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
1197
- param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
1198
- else:
1199
- param.shard_weight_type = {k: loaded_weight.item() for k in idx_map}
1200
- return
1201
-
1202
- if is_gguf_weight:
1203
- output_dim = getattr(param, "output_dim", None)
1204
- shard_size = loaded_weight.size(output_dim) // self.tp_size
1205
- start_idx = self.tp_rank * shard_size
1206
-
1207
- if loaded_shard_id is not None:
1208
- loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
1209
- param.shard_id.append(loaded_shard_id)
1210
- param.shard_id_map[loaded_shard_id] = len(param.data_container)
1211
- param.data_container.append(loaded_weight)
1212
- return
1131
1132
param_data = param.data
1133
output_dim = getattr(param, "output_dim", None)
@@ -1498,19 +1416,6 @@ class RowParallelLinear(LinearBase):
1416
# no need to narrow
1417
is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
1418
1501
- # Special case for GGUF
1502
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
1503
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
1504
- if is_gguf_weight_type:
1505
- param.weight_type = loaded_weight.item()
1506
-
1507
- # Materialize GGUF UninitializedParameter
1508
- if is_gguf_weight and isinstance(param, UninitializedParameter):
1509
- weight_shape = list(loaded_weight.shape)
1510
- if input_dim:
1511
- weight_shape[input_dim] = weight_shape[input_dim] // self.tp_size
1512
- param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
1513
-
1419
param_data = param.data
1420
if input_dim is not None and not is_sharded_weight:
1421
shard_size = param_data.shape[input_dim]
vllm/model_executor/layers/quantization/__init__.py
-3
@@ -18,7 +18,6 @@ QuantizationMethods = Literal[
18
"modelopt_fp4",
19
"modelopt_mxfp8",
20
"modelopt_mixed",
21
- "gguf",
21
"auto_gptq",
22
"gptq",
23
"gptq_marlin",
@@ -125,7 +124,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
124
from .fbgemm_fp8 import FBGEMMFp8Config
125
from .fp8 import Fp8Config
126
from .fp_quant import FPQuantConfig
128
- from .gguf import GGUFConfig
127
from .humming import HummingConfig
128
from .inc import INCConfig
129
from .modelopt import (
@@ -148,7 +146,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
146
"modelopt_fp4": ModelOptNvFp4Config,
147
"modelopt_mxfp8": ModelOptMxFp8Config,
148
"modelopt_mixed": ModelOptMixedPrecisionConfig,
151
- "gguf": GGUFConfig,
149
"auto_gptq": AutoGPTQConfig,
150
"gptq": AutoGPTQConfig,
151
"gptq_marlin": AutoGPTQConfig,
vllm/model_executor/layers/quantization/base_config.py
+7
@@ -47,6 +47,13 @@ class QuantizeMethodBase(ABC):
47
Expects create_weights to have been called before on the layer."""
48
raise NotImplementedError
49
50
+ # Not required functions
51
+ def tie_weights(self, layer: torch.nn.Module, *args, **kwargs):
52
+ """Tie layer's weights for the layer from another layer/tensors.
53
+
54
+ Expects create_weights to have been called before on the layer."""
55
+ raise NotImplementedError
56
+
57
def process_weights_after_loading(self, layer: nn.Module) -> None:
58
"""Process the weight after loading.
59
vllm/model_executor/layers/quantization/gguf.py
deleted
-690
@@ -1,690 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from collections.abc import Mapping
5
-from types import MappingProxyType
6
-from typing import TYPE_CHECKING, Any
7
-
8
-if TYPE_CHECKING:
9
- from vllm.model_executor.layers.quantization import QuantizationMethods
10
-
11
-import gguf
12
-import torch
13
-from gguf import GGMLQuantizationType as WeightType
14
-from torch.nn.parameter import Parameter, UninitializedParameter
15
-
16
-from vllm import _custom_ops as ops
17
-from vllm.logger import init_logger
18
-from vllm.model_executor.layers.fused_moe import (
19
- FusedMoEConfig,
20
- FusedMoEMethodBase,
21
- FusedMoEQuantConfig,
22
- MoEActivation,
23
- RoutedExperts,
24
- SharedExperts,
25
- apply_moe_activation,
26
-)
27
-from vllm.model_executor.layers.linear import (
28
- LinearBase,
29
- LinearMethodBase,
30
- UnquantizedLinearMethod,
31
-)
32
-from vllm.model_executor.layers.quantization import QuantizationMethods
33
-from vllm.model_executor.layers.quantization.base_config import (
34
- QuantizationConfig,
35
- QuantizeMethodBase,
36
-)
37
-from vllm.model_executor.layers.vocab_parallel_embedding import (
38
- UnquantizedEmbeddingMethod,
39
- VocabParallelEmbedding,
40
-)
41
-from vllm.model_executor.models.utils import WeightsMapper
42
-from vllm.model_executor.utils import set_weight_attrs
43
-from vllm.platforms import current_platform
44
-from vllm.utils.torch_utils import direct_register_custom_op
45
-
46
-logger = init_logger(__name__)
47
-
48
-
49
-class GGUFConfig(QuantizationConfig):
50
- """Config class for GGUF."""
51
-
52
- def __init__(self, unquantized_modules: list[str] | None = None) -> None:
53
- super().__init__()
54
- self.unquantized_modules = unquantized_modules or []
55
-
56
- def __repr__(self) -> str:
57
- return "GGUFConfig()"
58
-
59
- def get_name(self) -> QuantizationMethods:
60
- return "gguf"
61
-
62
- def get_supported_act_dtypes(self) -> list[torch.dtype]:
63
- # GGUF dequantization kernels use half precision (fp16) internally.
64
- # bfloat16 has precision issues on Blackwell devices.
65
- if current_platform.has_device_capability(100):
66
- logger.warning_once("GGUF has precision issues with bfloat16 on Blackwell.")
67
- return [torch.half, torch.float32]
68
- return [torch.half, torch.bfloat16, torch.float32]
69
-
70
- @classmethod
71
- def get_min_capability(cls) -> int:
72
- return 60
73
-
74
- @classmethod
75
- def get_config_filenames(cls) -> list[str]:
76
- return [] # no extra configs.
77
-
78
- @classmethod
79
- def from_config(cls, config: dict[str, Any]) -> "GGUFConfig":
80
- return cls()
81
-
82
- @classmethod
83
- def override_quantization_method(
84
- cls, hf_quant_cfg: dict[str, Any], user_quant: str | None, hf_config=None
85
- ) -> "QuantizationMethods | None":
86
- # When user explicitly specifies --quantization gguf, override
87
- # whatever quantization method is in the HF model config (e.g. fp8).
88
- if user_quant == "gguf":
89
- return "gguf"
90
- return None
91
-
92
- def get_quant_method(
93
- self, layer: torch.nn.Module, prefix: str
94
- ) -> "QuantizeMethodBase | None":
95
- if isinstance(layer, LinearBase):
96
- if is_layer_skipped_gguf(
97
- prefix, self.unquantized_modules, self.packed_modules_mapping
98
- ):
99
- return UnquantizedLinearMethod()
100
- return GGUFLinearMethod(self)
101
- elif isinstance(layer, VocabParallelEmbedding):
102
- if is_layer_skipped_gguf(
103
- prefix, self.unquantized_modules, self.packed_modules_mapping
104
- ):
105
- return UnquantizedEmbeddingMethod()
106
- return GGUFEmbeddingMethod(self)
107
- elif isinstance(layer, RoutedExperts):
108
- # TODO: Select UnquantizedFusedMoEMethod on unquantized layers.
109
- return GGUFMoEMethod(self, layer.moe_config)
110
- return None
111
-
112
- def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
113
- """
114
- Interface for models to update module names referenced in
115
- quantization configs in order to reflect the vllm model structure
116
-
117
- Args:
118
- hf_to_vllm_mapper: maps from hf model structure (the assumed
119
- structure of the qconfig) to vllm model structure
120
- """
121
- if self.unquantized_modules is not None:
122
- self.unquantized_modules = hf_to_vllm_mapper.apply_list(
123
- self.unquantized_modules
124
- )
125
-
126
-
127
-def is_layer_skipped_gguf(
128
- prefix: str,
129
- unquantized_modules: list[str],
130
- fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
131
-):
132
- # Fused layers like gate_up_proj or qkv_proj will not be fused
133
- # in the safetensors checkpoint. So, we convert the name
134
- # from the fused version to unfused + check to make sure that
135
- # each shard of the fused layer has the same scheme.
136
- proj_name = prefix.split(".")[-1]
137
- if proj_name in fused_mapping:
138
- shard_prefixes = [
139
- prefix.replace(proj_name, shard_proj_name)
140
- for shard_proj_name in fused_mapping[proj_name]
141
- ]
142
-
143
- is_skipped = None
144
- for shard_prefix in shard_prefixes:
145
- is_shard_skipped = any(
146
- shard_prefix in module_name for module_name in unquantized_modules
147
- )
148
-
149
- if is_skipped is None:
150
- is_skipped = is_shard_skipped
151
- elif is_shard_skipped != is_skipped:
152
- raise ValueError(
153
- f"Detected some but not all shards of {prefix} "
154
- "are quantized. All shards of fused layers "
155
- "to have the same precision."
156
- )
157
- else:
158
- is_skipped = any(module_name in prefix for module_name in unquantized_modules)
159
-
160
- assert is_skipped is not None
161
- return is_skipped
162
-
163
-
164
-UNQUANTIZED_TYPES = {WeightType.F32, WeightType.F16, WeightType.BF16}
165
-STANDARD_QUANT_TYPES = {
166
- WeightType.Q4_0,
167
- WeightType.Q4_1,
168
- WeightType.Q5_0,
169
- WeightType.Q5_1,
170
- WeightType.Q8_0,
171
- WeightType.Q8_1,
172
-}
173
-KQUANT_TYPES = {
174
- WeightType.Q2_K,
175
- WeightType.Q3_K,
176
- WeightType.Q4_K,
177
- WeightType.Q5_K,
178
- WeightType.Q6_K,
179
-}
180
-IMATRIX_QUANT_TYPES = {
181
- WeightType.IQ1_M,
182
- WeightType.IQ1_S,
183
- WeightType.IQ2_XXS,
184
- WeightType.IQ2_XS,
185
- WeightType.IQ2_S,
186
- WeightType.IQ3_XXS,
187
- WeightType.IQ3_S,
188
- WeightType.IQ4_XS,
189
- WeightType.IQ4_NL,
190
-}
191
-# TODO(Isotr0py): Currently, we don't have MMQ kernel for I-Matrix quantization.
192
-# Consolidate DEQUANT_TYPES, MMVQ_QUANT_TYPES and MMQ_QUANT_TYPES after we add
193
-# MMQ kernel for I-Matrix quantization.
194
-DEQUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
195
-MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
196
-MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
197
-
198
-
199
-def _fused_mul_mat_gguf(
200
- x: torch.Tensor, qweight: torch.Tensor, qweight_type: int
201
-) -> torch.Tensor:
202
- if qweight_type in IMATRIX_QUANT_TYPES:
203
- mmvq_safe = 8 if qweight.shape[0] > 5120 else 16
204
- else:
205
- mmvq_safe = 2 if qweight.shape[0] > 5120 else 6
206
- # HACK: when doing chunked prefill we don't generate output tokens
207
- # so input to logits generator is empty which causes invalid parameter
208
- if x.shape[0] == 0:
209
- return torch.empty(x.shape[0], qweight.shape[0], dtype=x.dtype, device=x.device)
210
- # there is no need to call any kernel for fp16/bf16
211
- if qweight_type in UNQUANTIZED_TYPES:
212
- return x @ qweight.T
213
- # enable MMVQ in contiguous batching with batch_size=1
214
- if x.shape[0] <= mmvq_safe and qweight_type in MMVQ_QUANT_TYPES:
215
- y = ops.ggml_mul_mat_vec_a8(qweight, x, qweight_type, qweight.shape[0])
216
- # Use MMQ Kernel if it's available (standard + k-quants)
217
- elif qweight_type in MMQ_QUANT_TYPES:
218
- y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
219
- # If there is no available MMQ kernel, fallback to dequantize
220
- elif qweight_type in DEQUANT_TYPES:
221
- block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
222
- shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
223
- weight = ops.ggml_dequantize(qweight, qweight_type, *shape, x.dtype)
224
- y = x @ weight.T
225
- else:
226
- # Raise an error if the quantization type is not supported.
227
- # Might be useful if llama.cpp adds a new quantization type.
228
- # Wrap to GGMLQuantizationType IntEnum to make sure it's a valid type.
229
- qweight_type = WeightType(qweight_type)
230
- raise NotImplementedError(f"Unsupported GGUF quantization type: {qweight_type}")
231
- return y
232
-
233
-
234
-def _fused_mul_mat_gguf_fake(
235
- x: torch.Tensor,
236
- qweight: torch.Tensor,
237
- qweight_type: int,
238
-) -> torch.Tensor:
239
- return torch.empty(x.shape[0], qweight.shape[0], dtype=x.dtype, device=x.device)
240
-
241
-
242
-try:
243
- direct_register_custom_op(
244
- op_name="_fused_mul_mat_gguf",
245
- op_func=_fused_mul_mat_gguf,
246
- fake_impl=_fused_mul_mat_gguf_fake,
247
- )
248
- fused_mul_mat_gguf = torch.ops.vllm._fused_mul_mat_gguf
249
-
250
-except AttributeError as error:
251
- raise error
252
-
253
-
254
-def _fused_moe_gguf(
255
- x: torch.Tensor,
256
- w1: torch.Tensor,
257
- w2: torch.Tensor,
258
- topk_weights: torch.Tensor,
259
- topk_ids: torch.Tensor,
260
- qweight_type: int,
261
- qweight_type2: int,
262
- activation: str,
263
-) -> torch.Tensor:
264
- activation_enum = MoEActivation.from_str(activation)
265
-
266
- def act(x: torch.Tensor):
267
- d = x.shape[-1] // 2
268
- output_shape = x.shape[:-1] + (d,)
269
- out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
270
- apply_moe_activation(activation_enum, out, x)
271
- return out
272
-
273
- # lazy import to avoid triggering triton import in CPU backend
274
- from vllm.model_executor.layers.fused_moe.fused_moe import moe_align_block_size
275
-
276
- out_hidden_states = torch.empty_like(x)
277
- # unless we decent expert reuse we are better off running moe_vec kernel
278
- if (
279
- qweight_type2 in MMQ_QUANT_TYPES
280
- and qweight_type in MMQ_QUANT_TYPES
281
- and x.shape[0] > 64
282
- ):
283
- num_tokens, _ = x.shape
284
- E, N, _ = w1.shape
285
- top_k = topk_ids.shape[1]
286
- BLOCK_SIZE = ops.ggml_moe_get_block_size(qweight_type)
287
-
288
- sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
289
- topk_ids, BLOCK_SIZE, E
290
- )
291
- out = ops.ggml_moe_a8(
292
- x,
293
- w1,
294
- sorted_token_ids,
295
- expert_ids,
296
- num_tokens_post_padded,
297
- qweight_type,
298
- N,
299
- top_k,
300
- num_tokens,
301
- )
302
- out = act(out)
303
- out = ops.ggml_moe_a8(
304
- out,
305
- w2,
306
- sorted_token_ids,
307
- expert_ids,
308
- num_tokens_post_padded,
309
- qweight_type2,
310
- w2.shape[1],
311
- 1,
312
- num_tokens * top_k,
313
- )
314
- out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
315
- topk_weights.view(num_tokens, top_k, 1)
316
- )
317
- ops.moe_sum(out, out_hidden_states)
318
- elif qweight_type2 in MMVQ_QUANT_TYPES and qweight_type in MMVQ_QUANT_TYPES:
319
- num_tokens, _ = x.shape
320
- E, N, _ = w1.shape
321
- top_k = topk_ids.shape[1]
322
-
323
- out = ops.ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens)
324
- out = act(out)
325
-
326
- out = ops.ggml_moe_a8_vec(
327
- out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
328
- )
329
- out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
330
- topk_weights.view(num_tokens, top_k, 1)
331
- )
332
- ops.moe_sum(out, out_hidden_states)
333
- else:
334
- logger.warning_once(
335
- "There is no support for fast MoE kernel "
336
- "for current quantization method. "
337
- "Falling back to slow implementation. "
338
- )
339
- for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
340
- inp = x[tok].reshape((1,) + x.shape[1:])
341
- current_hidden_state = None
342
- for ww, ii in zip(w, idx):
343
- expert_up = w1[ii]
344
-
345
- out = fused_mul_mat_gguf(inp, expert_up, qweight_type)
346
- out = act(out)
347
-
348
- expert_down = w2[ii]
349
- current_state = fused_mul_mat_gguf(
350
- out, expert_down, qweight_type2
351
- ).mul_(ww)
352
- if current_hidden_state is None:
353
- current_hidden_state = current_state
354
- else:
355
- current_hidden_state.add_(current_state)
356
- out_hidden_states[tok] = current_hidden_state
357
- return out_hidden_states
358
-
359
-
360
-def _fused_moe_gguf_fake(
361
- x: torch.Tensor,
362
- w1: torch.Tensor,
363
- w2: torch.Tensor,
364
- topk_weights: torch.Tensor,
365
- topk_ids: torch.Tensor,
366
- qweight_type: int,
367
- qweight_type2: int,
368
- activation: str,
369
-) -> torch.Tensor:
370
- return torch.empty_like(x)
371
-
372
-
373
-try:
374
- direct_register_custom_op(
375
- op_name="_fused_moe_gguf",
376
- op_func=_fused_moe_gguf,
377
- fake_impl=_fused_moe_gguf_fake,
378
- )
379
- fused_moe_gguf = torch.ops.vllm._fused_moe_gguf
380
-
381
-except AttributeError as error:
382
- raise error
383
-
384
-
385
-def _apply_gguf_embedding(
386
- x: torch.Tensor,
387
- qweight: torch.Tensor,
388
- qweight_type: int,
389
- hidden_size: int,
390
- dtype: torch.dtype | None = None,
391
-) -> torch.Tensor:
392
- if qweight_type in UNQUANTIZED_TYPES:
393
- return torch.embedding(qweight, x)
394
- elif qweight_type in DEQUANT_TYPES:
395
- block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
396
- x_flat = x.flatten()
397
- assert hidden_size == qweight.shape[1] // type_size * block_size
398
- quant = torch.index_select(qweight, dim=0, index=x_flat)
399
- dequant = ops.ggml_dequantize(
400
- quant, qweight_type, hidden_size, x_flat.shape[0], dtype
401
- )
402
- return dequant.view(*x.shape, hidden_size)
403
- else:
404
- qweight_type = WeightType(qweight_type)
405
- raise NotImplementedError(f"Unsupported GGUF quantization type: {qweight_type}")
406
-
407
-
408
-def _apply_gguf_embedding_fake(
409
- x: torch.Tensor,
410
- qweight: torch.Tensor,
411
- qweight_type: int,
412
- hidden_size: int,
413
- dtype: torch.dtype | None = None,
414
-) -> torch.Tensor:
415
- return torch.empty(x.shape[0], hidden_size, dtype=dtype, device=x.device)
416
-
417
-
418
-try:
419
- direct_register_custom_op(
420
- op_name="_apply_gguf_embedding",
421
- op_func=_apply_gguf_embedding,
422
- fake_impl=_apply_gguf_embedding_fake,
423
- )
424
- apply_gguf_embedding = torch.ops.vllm._apply_gguf_embedding
425
-
426
-except AttributeError as error:
427
- raise error
428
-
429
-
430
-class GGUFLinearMethod(LinearMethodBase):
431
- """Linear method for GGUF.
432
-
433
- Args:
434
- quant_config: The GGUF quantization config.
435
- """
436
-
437
- def __init__(self, quant_config: GGUFConfig):
438
- self.quant_config = quant_config
439
-
440
- def create_weights(
441
- self,
442
- layer: torch.nn.Module,
443
- input_size_per_partition: int,
444
- output_partition_sizes: list[int],
445
- input_size: int,
446
- output_size: int,
447
- params_dtype: torch.dtype,
448
- **extra_weight_attrs,
449
- ):
450
- self.params_dtype = params_dtype
451
- output_size_per_partition = sum(output_partition_sizes)
452
-
453
- tensor_shape = (output_size_per_partition, input_size_per_partition)
454
- qweight = GGUFUninitializedParameter(requires_grad=False)
455
- set_weight_attrs(
456
- qweight,
457
- {
458
- "input_dim": 1,
459
- "output_dim": 0,
460
- "tensor_shape": tensor_shape,
461
- "is_gguf_weight": True,
462
- "data_container": [],
463
- "shard_id": [],
464
- "shard_id_map": {},
465
- },
466
- )
467
- set_weight_attrs(qweight, extra_weight_attrs)
468
- layer.register_parameter("qweight", qweight)
469
-
470
- qweight_type = Parameter(
471
- torch.empty(len(output_partition_sizes), dtype=torch.uint8),
472
- requires_grad=False,
473
- )
474
- set_weight_attrs(
475
- qweight_type,
476
- {
477
- "is_gguf_weight_type": True,
478
- "weight_type": 0,
479
- "shard_weight_type": {},
480
- "ignore_warning": True,
481
- },
482
- )
483
- set_weight_attrs(qweight_type, extra_weight_attrs)
484
- layer.register_parameter("qweight_type", qweight_type)
485
-
486
- def process_weights_after_loading(self, layer: torch.nn.Module):
487
- qweight_type = layer.qweight_type.weight_type
488
- if not (qweight_type in UNQUANTIZED_TYPES or qweight_type in DEQUANT_TYPES):
489
- qweight_type = WeightType(qweight_type)
490
- raise ValueError(
491
- f"Unsupported GGUF quantization type {qweight_type} in layer {layer}."
492
- )
493
- # For MergedColumnParallelLinear and QKVParallelLinear, we need to
494
- # materialize the padded weight parameter for CUDA Graph compatibility.
495
- self._create_padded_weight_param(layer)
496
-
497
- def _create_padded_weight_param(self, layer: torch.nn.Module):
498
- """Create padded weight parameter for GGUF MergedLinear layer."""
499
- qweight = layer.qweight
500
- shard_id_map = qweight.shard_id_map
501
- shard_id = qweight.shard_id
502
- if len(data_container := qweight.data_container) > 1:
503
- dtype = {data.dtype for data in data_container}
504
- assert len(dtype) == 1, ValueError(
505
- f"Data container has mixed dtypes: {dtype}"
506
- )
507
- dtype = next(iter(dtype))
508
- # concat dim0 and pad dim1
509
- padded_side = max(x.size(1) for x in data_container)
510
- concat_side = sum(x.size(0) for x in data_container)
511
- # Pad the quantized weights to dense tensor, and create a map
512
- # with the location of each shard in the padded tensor.
513
- padded_data = torch.zeros(
514
- (concat_side, padded_side), dtype=dtype, device=qweight.device
515
- )
516
- # (dim0_start, dim0_end, dim1_size)
517
- shard_offset_map = dict[str, tuple[int, int, int]]()
518
- for idx in shard_id:
519
- id_in_container = shard_id_map[idx]
520
- start = sum(x.size(0) for x in data_container[:id_in_container])
521
- end = start + data_container[id_in_container].size(0)
522
- size = data_container[id_in_container].size(1)
523
- padded_data[start:end, :size] = data_container[id_in_container]
524
- shard_offset_map[idx] = (start, end, size)
525
- qweight.data_container.clear()
526
- padded_param = Parameter(padded_data, requires_grad=False)
527
- set_weight_attrs(padded_param, vars(qweight))
528
- set_weight_attrs(padded_param, {"shard_offset_map": shard_offset_map})
529
- layer.register_parameter("qweight", padded_param)
530
-
531
- def apply(
532
- self,
533
- layer: torch.nn.Module,
534
- x: torch.Tensor,
535
- bias: torch.Tensor | None = None,
536
- ) -> torch.Tensor:
537
- shard_id = layer.qweight.shard_id
538
-
539
- if shard_id:
540
- # dequantize shard weights respectively
541
- shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id
542
- qweight = layer.qweight
543
- result = []
544
- for idx in shard_id:
545
- start, end, offset = layer.qweight.shard_offset_map[idx]
546
- qweight_type = layer.qweight_type.shard_weight_type[idx]
547
- result.append(
548
- fused_mul_mat_gguf(
549
- x, qweight[start:end, :offset].contiguous(), qweight_type
550
- )
551
- )
552
- out = torch.cat(result, axis=1)
553
- else:
554
- qweight = layer.qweight
555
- qweight_type = layer.qweight_type.weight_type
556
- out = fused_mul_mat_gguf(x, qweight, qweight_type)
557
- if bias is not None:
558
- out.add_(bias)
559
- return out
560
-
561
-
562
-class GGUFMoEMethod(FusedMoEMethodBase):
563
- """MoE method for GGUF.
564
-
565
- Args:
566
- quant_config: The GGUF quantization config.
567
- """
568
-
569
- def __init__(
570
- self,
571
- quant_config: GGUFConfig,
572
- moe: FusedMoEConfig,
573
- ):
574
- super().__init__(moe)
575
- self.quant_config = quant_config
576
-
577
- def create_weights(
578
- self,
579
- layer: RoutedExperts,
580
- num_experts: int,
581
- hidden_size: int,
582
- intermediate_size_per_partition: int,
583
- params_dtype: torch.dtype,
584
- **extra_weight_attrs,
585
- ):
586
- tensor_shape = (num_experts, 2 * intermediate_size_per_partition, hidden_size)
587
- # gate up proj
588
- w13_qweight = GGUFUninitializedParameter(requires_grad=False)
589
- set_weight_attrs(
590
- w13_qweight,
591
- {
592
- "input_dim": 1,
593
- "output_dim": 0,
594
- "tensor_shape": tensor_shape,
595
- "is_gguf_weight": True,
596
- "data_container": [],
597
- },
598
- )
599
- set_weight_attrs(w13_qweight, extra_weight_attrs)
600
- layer.register_parameter("w13_qweight", w13_qweight)
601
-
602
- w13_qweight_type = Parameter(
603
- torch.empty(1, dtype=torch.uint8), requires_grad=False
604
- )
605
- set_weight_attrs(
606
- w13_qweight_type,
607
- {"is_gguf_weight_type": True, "weight_type": 0, "ignore_warning": True},
608
- )
609
- set_weight_attrs(w13_qweight_type, extra_weight_attrs)
610
- layer.register_parameter("w13_qweight_type", w13_qweight_type)
611
-
612
- tensor_shape = (num_experts, intermediate_size_per_partition, hidden_size)
613
- # gate down proj
614
- w2_qweight = GGUFUninitializedParameter(requires_grad=False)
615
- set_weight_attrs(
616
- w2_qweight,
617
- {
618
- "input_dim": 1,
619
- "output_dim": 0,
620
- "tensor_shape": tensor_shape,
621
- "is_gguf_weight": True,
622
- "data_container": [],
623
- },
624
- )
625
- set_weight_attrs(w2_qweight, extra_weight_attrs)
626
- layer.register_parameter("w2_qweight", w2_qweight)
627
-
628
- w2_qweight_type = Parameter(
629
- torch.empty(1, dtype=torch.uint8), requires_grad=False
630
- )
631
- set_weight_attrs(
632
- w2_qweight_type,
633
- {"is_gguf_weight_type": True, "weight_type": 0, "ignore_warning": True},
634
- )
635
-
636
- set_weight_attrs(w2_qweight_type, extra_weight_attrs)
637
- layer.register_parameter("w2_qweight_type", w2_qweight_type)
638
-
639
- def get_fused_moe_quant_config(
640
- self, layer: RoutedExperts
641
- ) -> FusedMoEQuantConfig | None:
642
- return None
643
-
644
- def apply(
645
- self,
646
- layer: RoutedExperts,
647
- x: torch.Tensor,
648
- topk_weights: torch.Tensor,
649
- topk_ids: torch.Tensor,
650
- shared_experts: SharedExperts | None,
651
- shared_experts_input: torch.Tensor | None,
652
- ) -> torch.Tensor:
653
- if layer.apply_router_weight_on_input:
654
- raise NotImplementedError(
655
- "Apply router weight on input is not supported for"
656
- "fused GGUF MoE method."
657
- )
658
-
659
- return fused_moe_gguf(
660
- x,
661
- layer.w13_qweight,
662
- layer.w2_qweight,
663
- topk_weights,
664
- topk_ids,
665
- layer.w13_qweight_type.weight_type,
666
- layer.w2_qweight_type.weight_type,
667
- layer.activation.value,
668
- )
669
-
670
-
671
-class GGUFEmbeddingMethod(GGUFLinearMethod):
672
- """Embedding method for GGUF.
673
-
674
- Args:
675
- quant_config: The GGUF quantization config.
676
- """
677
-
678
- def embedding(self, layer: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:
679
- qweight = layer.qweight
680
- qweight_type = layer.qweight_type.weight_type
681
- hidden_size = qweight.tensor_shape[1]
682
-
683
- return apply_gguf_embedding(
684
- x, qweight, qweight_type, hidden_size, dtype=self.params_dtype
685
- )
686
-
687
-
688
-class GGUFUninitializedParameter(UninitializedParameter):
689
- cls_to_become = Parameter
690
- data_container: list[torch.Tensor]
vllm/model_executor/layers/vocab_parallel_embedding.py
+8
-18
@@ -6,7 +6,7 @@ from dataclasses import dataclass
6
7
import torch
8
import torch.nn.functional as F
9
-from torch.nn.parameter import Parameter, UninitializedParameter
9
+from torch.nn.parameter import Parameter
10
11
import vllm.envs as envs
12
from vllm.distributed import (
@@ -77,6 +77,12 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
77
def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor:
78
return F.embedding(input_, layer.weight)
79
80
+ def tie_weights(
81
+ self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding"
82
+ ):
83
+ layer.weight = embed_tokens.weight
84
+ return layer
85
+
86
87
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
88
"""Pad the vocab size to the given value."""
@@ -425,17 +431,6 @@ class VocabParallelEmbedding(PluggableLayer):
431
output_dim = getattr(param, "output_dim", None)
432
packed_dim = getattr(param, "packed_dim", None)
433
428
- # If the parameter is a gguf weight, then load it directly.
429
- if getattr(param, "is_gguf_weight_type", None):
430
- param.data.copy_(loaded_weight)
431
- param.weight_type = loaded_weight.item()
432
- return
433
- elif isinstance(param, UninitializedParameter):
434
- shape = list(loaded_weight.shape)
435
- if output_dim is not None:
436
- shape[output_dim] = self.num_embeddings_per_partition
437
- param.materialize(tuple(shape), dtype=loaded_weight.dtype)
438
-
434
# If parameter does not have output dim, then it should
435
# be copied onto all gpus (e.g. g_idx for act_order gptq).
436
if output_dim is None:
@@ -562,12 +557,7 @@ class ParallelLMHead(VocabParallelEmbedding):
557
558
def tie_weights(self, embed_tokens: VocabParallelEmbedding):
559
"""Tie the weights with word embeddings."""
565
- # GGUF quantized embed_tokens.
566
- if self.quant_config and self.quant_config.get_name() == "gguf":
567
- return embed_tokens
568
- else:
569
- self.weight = embed_tokens.weight
570
- return self
560
+ return self.quant_method.tie_weights(self, embed_tokens)
561
562
def forward(self, input_):
563
del input_
vllm/model_executor/model_loader/__init__.py
-4
@@ -12,7 +12,6 @@ from vllm.model_executor.model_loader.base_loader import BaseModelLoader
12
from vllm.model_executor.model_loader.bitsandbytes_loader import BitsAndBytesModelLoader
13
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
14
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
15
-from vllm.model_executor.model_loader.gguf_loader import GGUFModelLoader
15
from vllm.model_executor.model_loader.modelexpress_loader import (
16
ModelExpressModelLoader,
17
)
@@ -37,7 +36,6 @@ LoadFormats = Literal[
36
"bitsandbytes",
37
"dummy",
38
"fastsafetensors",
40
- "gguf",
39
"instanttensor",
40
"mistral",
41
"modelexpress",
@@ -55,7 +53,6 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {
53
"bitsandbytes": BitsAndBytesModelLoader,
54
"dummy": DummyModelLoader,
55
"fastsafetensors": DefaultModelLoader,
58
- "gguf": GGUFModelLoader,
56
"instanttensor": DefaultModelLoader,
57
"mistral": DefaultModelLoader,
58
"modelexpress": ModelExpressModelLoader,
@@ -154,7 +151,6 @@ __all__ = [
151
"register_model_loader",
152
"BaseModelLoader",
153
"BitsAndBytesModelLoader",
157
- "GGUFModelLoader",
154
"ModelExpressModelLoader",
155
"DefaultModelLoader",
156
"DummyModelLoader",
vllm/model_executor/model_loader/gguf_loader.py
deleted
-453
@@ -1,453 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import os
4
-from collections.abc import Generator
5
-from typing import TYPE_CHECKING, cast
6
-
7
-import gguf
8
-import regex as re
9
-import torch
10
-import torch.nn as nn
11
-from transformers import AutoModelForCausalLM, AutoModelForImageTextToText
12
-
13
-from vllm.config import ModelConfig, VllmConfig
14
-from vllm.config.load import LoadConfig
15
-from vllm.logger import init_logger
16
-from vllm.model_executor.model_loader.base_loader import BaseModelLoader
17
-from vllm.model_executor.model_loader.utils import (
18
- initialize_model,
19
- process_weights_after_loading,
20
-)
21
-from vllm.model_executor.model_loader.weight_utils import (
22
- download_gguf,
23
- get_gguf_extra_tensor_names,
24
- get_gguf_weight_type_map,
25
- gguf_quant_weights_iterator,
26
- gguf_quant_weights_iterator_multi,
27
-)
28
-from vllm.transformers_utils.gguf_utils import detect_gguf_multimodal
29
-from vllm.transformers_utils.repo_utils import hf_api
30
-from vllm.utils.torch_utils import set_default_torch_dtype
31
-
32
-if TYPE_CHECKING:
33
- from vllm.model_executor.layers.quantization.gguf import GGUFConfig
34
-
35
-logger = init_logger(__name__)
36
-
37
-
38
-class GGUFModelLoader(BaseModelLoader):
39
- """
40
- Model loader that can load GGUF files. This is useful for loading models
41
- that are quantized with GGUF and saved in the GGUF format. This loader
42
- supports loading both full models and sharded models.
43
- """
44
-
45
- def __init__(self, load_config: LoadConfig):
46
- super().__init__(load_config)
47
- if load_config.model_loader_extra_config:
48
- raise ValueError(
49
- f"Model loader extra config is not supported for "
50
- f"load format {load_config.load_format}"
51
- )
52
-
53
- def _prepare_weights(self, model_config: ModelConfig):
54
- model_name_or_path = model_config.model
55
- if os.path.isfile(model_name_or_path):
56
- return model_name_or_path
57
- # repo id/filename.gguf
58
- if "/" in model_name_or_path and model_name_or_path.endswith(".gguf"):
59
- repo_id, filename = model_name_or_path.rsplit("/", 1)
60
- return hf_api().hf_hub_download(
61
- repo_id=repo_id,
62
- filename=filename,
63
- revision=model_config.revision,
64
- cache_dir=self.load_config.download_dir,
65
- )
66
- # repo_id:quant_type
67
- elif "/" in model_name_or_path and ":" in model_name_or_path:
68
- repo_id, quant_type = model_name_or_path.rsplit(":", 1)
69
- return download_gguf(
70
- repo_id,
71
- quant_type,
72
- cache_dir=self.load_config.download_dir,
73
- revision=model_config.revision,
74
- ignore_patterns=self.load_config.ignore_patterns,
75
- )
76
-
77
- raise ValueError(
78
- f"Unrecognised GGUF reference: {model_name_or_path} "
79
- "(expected local file, <repo_id>/<filename>.gguf, "
80
- "or <repo_id>:<quant_type>)"
81
- )
82
-
83
- @staticmethod
84
- def _get_all_gguf_files(model_path: str) -> list[str]:
85
- """Discover all GGUF shard files from a single shard path.
86
-
87
- Supports variable-width shard indices by dynamically detecting
88
- the padding from the original filename.
89
- E.g. ``*-00001-of-00005.gguf`` → all 5 shards,
90
- ``*-01-of-15.gguf`` → all 15 shards.
91
- """
92
- match = re.search(r"-(\d+)-of-(\d+)\.gguf$", model_path)
93
- if not match:
94
- return [model_path]
95
- total = int(match.group(2))
96
- num_digits = len(match.group(1))
97
- prefix = model_path[: match.start(1)]
98
- suffix = model_path[match.end(2) :]
99
- files = []
100
- for i in range(1, total + 1):
101
- shard_path = f"{prefix}{i:0{num_digits}d}-of-{total:0{num_digits}d}{suffix}"
102
- if os.path.isfile(shard_path):
103
- files.append(shard_path)
104
- if files:
105
- logger.info("Discovered %d GGUF shard files", len(files))
106
- return files if files else [model_path]
107
-
108
- def _get_gguf_weights_map(self, model_config: ModelConfig):
109
- """
110
- GGUF uses this naming convention for their tensors from HF checkpoint:
111
- `blk.N.BB.weight` and `blk.N.BB.bias`
112
- where N signifies the block number of a layer, and BB signifies the
113
- attention/mlp layer components.
114
- See "Standardized tensor names" in
115
- https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details.
116
- """
117
- config = model_config.hf_config
118
- # Get text config to handle both nested (multimodal) and flat
119
- # (text-only) config structures. For multimodal models like
120
- # Gemma3Config, this returns config.text_config. For text-only
121
- # models, this returns config itself.
122
- text_config = config.get_text_config()
123
- model_type = config.model_type
124
- is_multimodal = (
125
- hasattr(config, "vision_config") and config.vision_config is not None
126
- )
127
- gguf_to_hf_name_map = {}
128
- sideload_params: list[re.Pattern] = []
129
- # hack: ggufs have a different name than transformers
130
- if model_type == "cohere":
131
- model_type = "command-r"
132
- if model_type == "gemma3_text":
133
- # Gemma3 models use "gemma3_text" in HuggingFace but
134
- # "gemma3" in GGUF architecture naming
135
- model_type = "gemma3"
136
- if model_type in ("deepseek_v3", "deepseek_v2"):
137
- model_type = "deepseek2"
138
- # GGUF layer map assumes that we will have a merged expert weights
139
- # so we need to map them manually
140
- for idx in range(config.num_hidden_layers):
141
- gguf_to_hf_name_map[f"blk.{idx}.exp_probs_b.bias"] = (
142
- f"model.layers.{idx}.mlp.gate.e_score_correction_bias"
143
- )
144
- gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (
145
- f"model.layers.{idx}.mlp.experts.0.down_proj.weight"
146
- )
147
- gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (
148
- f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"
149
- )
150
- gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
151
- f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
152
- )
153
- sideload_params.append(
154
- re.compile(
155
- f"model\\.layers\\.{idx}"
156
- r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"
157
- )
158
- )
159
- if model_type in ("qwen2_moe", "qwen3_moe"):
160
- model_type = model_type.replace("_", "")
161
- # GGUF layer map assumes that we will have a merged expert weights
162
- # so we need to map them manually
163
- for idx in range(config.num_hidden_layers):
164
- gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (
165
- f"model.layers.{idx}.mlp.experts.0.down_proj.weight"
166
- )
167
- gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (
168
- f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"
169
- )
170
- gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
171
- f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
172
- )
173
- sideload_params.append(
174
- re.compile(
175
- f"model\\.layers\\.{idx}"
176
- r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"
177
- )
178
- )
179
- if model_type == "minimax_m2":
180
- model_type = "minimax-m2"
181
- # GGUF layer map assumes merged expert weights
182
- # map them manually like deepseek2
183
- for idx in range(config.num_hidden_layers):
184
- gguf_to_hf_name_map[f"blk.{idx}.exp_probs_b.bias"] = (
185
- f"model.layers.{idx}.block_sparse_moe.e_score_correction_bias"
186
- )
187
- gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (
188
- f"model.layers.{idx}.block_sparse_moe.experts.0.w2.weight"
189
- )
190
- gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (
191
- f"model.layers.{idx}.block_sparse_moe.experts.0.w1.weight"
192
- )
193
- gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
194
- f"model.layers.{idx}.block_sparse_moe.experts.0.w3.weight"
195
- )
196
- sideload_params.append(
197
- re.compile(
198
- f"model\\.layers\\.{idx}"
199
- r"\.block_sparse_moe\.experts\.(gate_up_proj|down_proj)"
200
- )
201
- )
202
-
203
- arch = None
204
- for key, value in gguf.MODEL_ARCH_NAMES.items():
205
- if value == model_type:
206
- arch = key
207
- break
208
- if arch is None:
209
- raise RuntimeError(f"Unknown gguf model_type: {model_type}")
210
- text_num_layers = text_config.num_hidden_layers
211
- text_name_map = gguf.get_tensor_name_map(arch, text_num_layers)
212
-
213
- if is_multimodal:
214
- mm_proj_arch = gguf.MODEL_ARCH.MMPROJ
215
- vision_num_layers = config.vision_config.num_hidden_layers
216
- vision_name_map = gguf.get_tensor_name_map(mm_proj_arch, vision_num_layers)
217
- else:
218
- vision_name_map = None
219
-
220
- # Create dummy model to extract parameter names
221
- # For multimodal: use AutoModelForImageTextToText to get
222
- # language + vision + projector params
223
- # For text-only: use AutoModelForCausalLM to get language model params
224
- auto_cls = (
225
- AutoModelForImageTextToText if is_multimodal else AutoModelForCausalLM
226
- )
227
- with torch.device("meta"):
228
- dummy_model = auto_cls.from_config(
229
- config, trust_remote_code=model_config.trust_remote_code
230
- )
231
-
232
- state_dict = dummy_model.state_dict()
233
- if hf_checkpoint_map := getattr(
234
- dummy_model, "_checkpoint_conversion_mapping", None
235
- ):
236
-
237
- def revert_hf_rename(name: str) -> str:
238
- for original_name, hf_name in hf_checkpoint_map.items():
239
- if hf_name in name:
240
- name = name.replace(hf_name, original_name).lstrip("^")
241
- return name
242
-
243
- state_dict = {
244
- revert_hf_rename(name): tensor for name, tensor in state_dict.items()
245
- }
246
-
247
- if model_type == "minimax-m2" and not hf_checkpoint_map:
248
- # Reverse HF convention: mlp -> block_sparse_moe
249
- state_dict = {
250
- name.replace(".mlp.", ".block_sparse_moe."): tensor
251
- for name, tensor in state_dict.items()
252
- }
253
-
254
- def find_hf_name_in_tensor_map(hf_name: str) -> str | None:
255
- """
256
- Map HuggingFace parameter name to GGUF tensor name.
257
-
258
- This function handles the mismatch between HF parameter naming
259
- conventions and gguf-py's expected format:
260
- 1. Strips 'model.' prefix (common in multimodal models)
261
- 2. Converts '_weight' suffix to '.weight' (Gemma3 compatibility)
262
- 3. Searches vision_name_map for multimodal parameters
263
- 4. Falls back to text_name_map for language model parameters
264
-
265
- Args:
266
- hf_name: Full HuggingFace parameter name (e.g.,
267
- 'model.multi_modal_projector.mm_soft_emb_norm.weight')
268
-
269
- Returns:
270
- GGUF tensor name with suffix (e.g., 'mm.soft_emb_norm.weight')
271
- or None if no mapping found
272
- """
273
- # In transformers v5, multimodal models (e.g. Gemma3) wrap
274
- # all sub-models under an outer 'model.' attribute, producing
275
- # state_dict keys like 'model.language_model.layers.0...' and
276
- # 'model.vision_tower.vision_model...'. Strip this outer
277
- # prefix so the keys match what gguf-py expects.
278
- if is_multimodal and hf_name.startswith("model."):
279
- hf_name = hf_name[6:] # Remove outer 'model.'
280
-
281
- # Strip 'language_model.' prefix for multimodal models - gguf-py
282
- # tensor mappings expect parameter names without this prefix.
283
- # Note: 'model.' prefix should be KEPT for text-only models as
284
- # gguf-py expects it.
285
- if hf_name.startswith("language_model."):
286
- hf_name = hf_name[15:] # Remove 'language_model.'
287
- # Re-add 'model.' prefix because gguf-py text tensor maps
288
- # expect 'model.layers...' format.
289
- if is_multimodal:
290
- hf_name = "model." + hf_name
291
-
292
- # Parse parameter name and suffix
293
- if hf_name.endswith((".weight", ".bias")):
294
- base_name, suffix = hf_name.rsplit(".", 1)
295
- else:
296
- base_name, suffix = hf_name, ""
297
- # Handle '_weight' suffix (Gemma3 naming: parameter ends with
298
- # '_weight' instead of '.weight')
299
- if base_name.endswith("_weight"):
300
- base_name = base_name[:-7] # Remove '_weight'
301
- suffix = "weight"
302
-
303
- gguf_name = None
304
- # Priority 1: Search vision/projector parameters for multimodal models
305
- if vision_name_map is not None:
306
- gguf_name = vision_name_map.get_name(base_name)
307
-
308
- # Priority 2: Search text backbone parameters
309
- if gguf_name is None:
310
- gguf_name = text_name_map.get_name(base_name)
311
-
312
- if gguf_name is None:
313
- return None
314
-
315
- return gguf_name + "." + suffix
316
-
317
- # Build mapping and track unmapped parameters
318
- unmapped_params = []
319
- for hf_name in state_dict:
320
- gguf_name_with_suffix = find_hf_name_in_tensor_map(hf_name)
321
-
322
- # Track mapping success
323
- if gguf_name_with_suffix is not None:
324
- gguf_to_hf_name_map[gguf_name_with_suffix] = hf_name
325
- logger.debug("Mapped GGUF %s → HF %s", gguf_name_with_suffix, hf_name)
326
- elif hf_name not in gguf_to_hf_name_map.values():
327
- # Parameter not in manual overrides either
328
- unmapped_params.append(hf_name)
329
-
330
- # All parameters (except those initialized by other means) must be mapped:
331
- # both vision/projector and backbone
332
- if unmapped_params:
333
- unmapped_params = list(
334
- filter(
335
- lambda x: not any(re.fullmatch(p, x) for p in sideload_params),
336
- unmapped_params,
337
- )
338
- )
339
- if unmapped_params:
340
- raise RuntimeError(
341
- f"Failed to map GGUF parameters "
342
- f"({len(unmapped_params)}): "
343
- f"{unmapped_params}"
344
- )
345
- return gguf_to_hf_name_map
346
-
347
- def _get_gguf_weight_type(
348
- self,
349
- model_config: ModelConfig,
350
- model_name_or_path: str,
351
- gguf_to_hf_name_map: dict[str, str],
352
- ) -> dict[str, str]:
353
- gguf_files = self._get_all_gguf_files(model_name_or_path)
354
- weight_type_map = {}
355
- for f in gguf_files:
356
- weight_type_map.update(get_gguf_weight_type_map(f, gguf_to_hf_name_map))
357
- is_multimodal = hasattr(model_config.hf_config, "vision_config")
358
- if is_multimodal:
359
- mmproj_file = detect_gguf_multimodal(model_name_or_path)
360
- assert mmproj_file is not None, (
361
- "Could not find mm_proj file for multimodal GGUF model"
362
- )
363
- logger.info("Loading extra mm_proj weights from %s...", mmproj_file)
364
- mm_proj_weight_type_map = get_gguf_weight_type_map(
365
- mmproj_file, gguf_to_hf_name_map
366
- )
367
- weight_type_map.update(mm_proj_weight_type_map)
368
- return weight_type_map
369
-
370
- def _get_weights_iterator(
371
- self,
372
- model_config: ModelConfig,
373
- model_name_or_path: str,
374
- gguf_to_hf_name_map: dict[str, str],
375
- ) -> Generator[tuple[str, torch.Tensor], None, None]:
376
- """
377
- Iterate over GGUF model weights, loading from both main model file and
378
- mmproj.gguf for multimodal Gemma3 models.
379
-
380
- For Gemma3 multimodal GGUF models:
381
- - Main file (gemma-3-*.gguf): Language model weights (model.*)
382
- - mmproj file (mmproj*.gguf): Vision tower + projector weights (v.*, mm.*)
383
-
384
- Yields:
385
- Tuples of (parameter_name, tensor) for all model weights
386
- """
387
- hf_config = model_config.hf_config
388
- is_multimodal = hasattr(hf_config, "vision_config")
389
-
390
- if is_multimodal:
391
- # Load mm_proj (mm_encoder + projector) for multimodal weights
392
- mmproj_file = detect_gguf_multimodal(model_name_or_path)
393
- assert mmproj_file is not None, (
394
- "Could not find mm_proj file for multimodal GGUF model"
395
- )
396
- yield from gguf_quant_weights_iterator(mmproj_file, gguf_to_hf_name_map)
397
-
398
- gguf_files = self._get_all_gguf_files(model_name_or_path)
399
- if len(gguf_files) > 1:
400
- yield from gguf_quant_weights_iterator_multi(
401
- gguf_files, gguf_to_hf_name_map
402
- )
403
- else:
404
- yield from gguf_quant_weights_iterator(
405
- model_name_or_path, gguf_to_hf_name_map
406
- )
407
-
408
- def download_model(self, model_config: ModelConfig) -> None:
409
- self._prepare_weights(model_config)
410
-
411
- def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
412
- local_model_path = self._prepare_weights(model_config)
413
- gguf_weights_map = self._get_gguf_weights_map(model_config)
414
- model.load_weights(
415
- self._get_weights_iterator(model_config, local_model_path, gguf_weights_map)
416
- )
417
-
418
- def load_model(
419
- self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str = ""
420
- ) -> nn.Module:
421
- device_config = vllm_config.device_config
422
- local_model_path = self._prepare_weights(model_config)
423
- gguf_weights_map = self._get_gguf_weights_map(model_config)
424
- # we can only know if tie word embeddings after mapping weights
425
- gguf_files = self._get_all_gguf_files(local_model_path)
426
- all_extra_names = []
427
- for f in gguf_files:
428
- all_extra_names.extend(get_gguf_extra_tensor_names(f, gguf_weights_map))
429
- if "lm_head.weight" in all_extra_names:
430
- model_config.hf_config.update({"tie_word_embeddings": True})
431
-
432
- weight_type_map = self._get_gguf_weight_type(
433
- model_config, local_model_path, gguf_weights_map
434
- )
435
- # filter out unquantized modules to skip
436
- unquant_names = [
437
- name.removesuffix(".weight")
438
- for name, weight_type in weight_type_map.items()
439
- if weight_type in ("F32", "F16", "BF16") and name.endswith(".weight")
440
- ]
441
- logger.debug("GGUF unquantized modules: %s", unquant_names)
442
- if TYPE_CHECKING:
443
- vllm_config.quant_config = cast(GGUFConfig, vllm_config.quant_config)
444
- vllm_config.quant_config.unquantized_modules.extend(unquant_names)
445
-
446
- target_device = torch.device(device_config.device)
447
- with set_default_torch_dtype(model_config.dtype):
448
- with target_device:
449
- model = initialize_model(vllm_config=vllm_config, prefix=prefix)
450
- self.load_weights(model, model_config)
451
-
452
- process_weights_after_loading(model, model_config, target_device)
453
- return model
vllm/model_executor/model_loader/weight_utils.py
-167
@@ -54,11 +54,6 @@ except ImportError:
54
runai_model_streamer = PlaceholderModule("runai_model_streamer") # type: ignore[assignment]
55
SafetensorsStreamer = runai_model_streamer.placeholder_attr("SafetensorsStreamer")
56
57
-try:
58
- import gguf
59
-except ImportError:
60
- gguf = PlaceholderModule("gguf")
61
-
57
try:
58
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
59
except ImportError:
@@ -250,10 +245,6 @@ def get_quant_config(
245
raise ValueError("Model quantization method is not specified in the config.")
246
quant_cls = get_quantization_config(model_config.quantization)
247
253
- # GGUF doesn't have config file
254
- if model_config.quantization == "gguf":
255
- return quant_cls()
256
-
248
# Read the quantization config from the HF model config, if available.
249
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
250
# some vision model may keep quantization_config in their text_config
@@ -437,52 +428,6 @@ def get_sparse_attention_config(
428
return config
429
430
440
-def download_gguf(
441
- repo_id: str,
442
- quant_type: str,
443
- cache_dir: str | None = None,
444
- revision: str | None = None,
445
- ignore_patterns: str | list[str] | None = None,
446
-) -> str:
447
- # Use patterns that snapshot_download can handle directly
448
- # Patterns to match:
449
- # - *-{quant_type}.gguf (root)
450
- # - *-{quant_type}-*.gguf (root sharded)
451
- # - */*-{quant_type}.gguf (subdir)
452
- # - */*-{quant_type}-*.gguf (subdir sharded)
453
- allow_patterns = [
454
- f"*-{quant_type}.gguf",
455
- f"*-{quant_type}-*.gguf",
456
- f"*/*-{quant_type}.gguf",
457
- f"*/*-{quant_type}-*.gguf",
458
- ]
459
-
460
- # Use download_weights_from_hf which handles caching and downloading
461
- folder = download_weights_from_hf(
462
- model_name_or_path=repo_id,
463
- cache_dir=cache_dir,
464
- allow_patterns=allow_patterns,
465
- revision=revision,
466
- ignore_patterns=ignore_patterns,
467
- )
468
-
469
- # Find the downloaded file(s) in the folder
470
- local_files = []
471
- for pattern in allow_patterns:
472
- # Convert pattern to glob pattern for local filesystem
473
- glob_pattern = os.path.join(folder, pattern)
474
- local_files.extend(glob.glob(glob_pattern))
475
-
476
- if not local_files:
477
- raise ValueError(
478
- f"Downloaded GGUF files not found in {folder} for quant_type {quant_type}"
479
- )
480
-
481
- # Sort to ensure consistent ordering (prefer non-sharded files)
482
- local_files.sort(key=lambda x: (x.count("-"), x))
483
- return local_files[0]
484
-
485
-
431
@instrument(span_name="Download weights - HF")
432
def download_weights_from_hf(
433
model_name_or_path: str,
@@ -1237,118 +1182,6 @@ def multi_thread_pt_weights_iterator(
1182
del state
1183
1184
1240
-def get_gguf_extra_tensor_names(
1241
- gguf_file: str | Path, gguf_to_hf_name_map: dict[str, str]
1242
-) -> list[str]:
1243
- reader = gguf.GGUFReader(gguf_file)
1244
- expected_gguf_keys = set(gguf_to_hf_name_map.keys())
1245
- exact_gguf_keys = set([tensor.name for tensor in reader.tensors])
1246
- extra_keys = expected_gguf_keys - exact_gguf_keys
1247
- return [gguf_to_hf_name_map[key] for key in extra_keys]
1248
-
1249
-
1250
-def get_gguf_weight_type_map(
1251
- gguf_file: str | Path, gguf_to_hf_name_map: dict[str, str]
1252
-) -> dict[str, str]:
1253
- """
1254
- Return GGUF mapped weight's name and its quant type
1255
- """
1256
- reader = gguf.GGUFReader(gguf_file)
1257
- return {
1258
- gguf_to_hf_name_map[tensor.name]: tensor.tensor_type.name
1259
- for tensor in reader.tensors
1260
- if tensor.name in gguf_to_hf_name_map
1261
- }
1262
-
1263
-
1264
-def gguf_quant_weights_iterator(
1265
- gguf_file: str | Path, gguf_to_hf_name_map: dict[str, str]
1266
-) -> Generator[tuple[str, torch.Tensor], None, None]:
1267
- """
1268
- Iterate over the quant weights in the model gguf files and convert
1269
- them to torch tensors.
1270
- Be careful of the order of yielding weight types and weights data,
1271
- we have to yield all weight types first before yielding any weights.
1272
- Otherwise it would cause issue when loading weights with for packed
1273
- layer with different quant types.
1274
- """
1275
-
1276
- reader = gguf.GGUFReader(gguf_file)
1277
-
1278
- for tensor in reader.tensors:
1279
- if tensor.name in gguf_to_hf_name_map:
1280
- weight_type = tensor.tensor_type
1281
- name = gguf_to_hf_name_map[tensor.name]
1282
-
1283
- if weight_type.name not in ("F32", "BF16", "F16"):
1284
- weight_type_name = name.replace("weight", "qweight_type")
1285
- weight_type = torch.tensor(weight_type)
1286
- yield weight_type_name, weight_type
1287
-
1288
- for tensor in reader.tensors:
1289
- if tensor.name in gguf_to_hf_name_map:
1290
- weight = tensor.data
1291
- weight_type = tensor.tensor_type
1292
- name = gguf_to_hf_name_map[tensor.name]
1293
- if weight_type.name not in ("F32", "BF16", "F16"):
1294
- name = name.replace("weight", "qweight")
1295
- if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:
1296
- # BF16 is currently the only "quantization" type that isn't
1297
- # actually quantized but is read as a raw byte tensor.
1298
- # Reinterpret as `torch.bfloat16` tensor.
1299
- weight = weight.view(np.uint16)
1300
- if reader.byte_order == "S":
1301
- # GGUF endianness != system endianness
1302
- weight = weight.byteswap()
1303
- param = torch.tensor(weight).view(torch.bfloat16)
1304
- else:
1305
- param = torch.tensor(weight)
1306
- yield name, param
1307
-
1308
-
1309
-def gguf_quant_weights_iterator_multi(
1310
- gguf_files: list[str], gguf_to_hf_name_map: dict[str, str]
1311
-) -> Generator[tuple[str, torch.Tensor], None, None]:
1312
- """
1313
- Iterate over the quant weights across multiple GGUF shard files
1314
- and convert them to torch tensors.
1315
-
1316
- Like gguf_quant_weights_iterator, we yield all weight types first
1317
- before yielding any weights data to avoid issues with packed layers
1318
- that have different quant types.
1319
- """
1320
- readers = [gguf.GGUFReader(f) for f in gguf_files]
1321
-
1322
- # First pass: yield all weight types across all shards
1323
- for reader in readers:
1324
- for tensor in reader.tensors:
1325
- if tensor.name in gguf_to_hf_name_map:
1326
- weight_type = tensor.tensor_type
1327
- name = gguf_to_hf_name_map[tensor.name]
1328
- if weight_type.name not in ("F32", "BF16", "F16"):
1329
- weight_type_name = name.replace("weight", "qweight_type")
1330
- weight_type = torch.tensor(weight_type)
1331
- yield weight_type_name, weight_type
1332
-
1333
- # Second pass: yield all weight data across all shards
1334
- for reader in readers:
1335
- for tensor in reader.tensors:
1336
- if tensor.name in gguf_to_hf_name_map:
1337
- weight = tensor.data
1338
- weight_type = tensor.tensor_type
1339
- name = gguf_to_hf_name_map[tensor.name]
1340
- if weight_type.name not in ("F32", "BF16", "F16"):
1341
- name = name.replace("weight", "qweight")
1342
- if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:
1343
- weight = weight.view(np.uint16)
1344
- if reader.byte_order == "S":
1345
- weight = weight.byteswap()
1346
- param = torch.tensor(weight).view(torch.bfloat16)
1347
- else:
1348
- param = torch.tensor(weight)
1349
- yield name, param
1350
-
1351
-
1185
def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
1186
"""convert PySafeSlice object from safetensors to torch.Tensor
1187
vllm/model_executor/models/apertus.py
-3
@@ -228,9 +228,6 @@ class ApertusAttention(nn.Module):
228
quant_config: QuantizationConfig | None,
229
) -> None:
230
is_neox_style = True
231
- is_gguf = quant_config and quant_config.get_name() == "gguf"
232
- if is_gguf and config.model_type == "apertus":
233
- is_neox_style = False
231
232
self.rotary_emb = get_rope(
233
self.head_dim,
vllm/model_executor/models/exaone.py
-2
@@ -162,8 +162,6 @@ class ExaoneAttention(nn.Module):
162
)
163
164
is_neox_style = True
165
- if quant_config is not None and quant_config.get_name() == "gguf":
166
- is_neox_style = False
165
166
self.rotary_emb = get_rope(
167
self.head_dim,
vllm/model_executor/models/exaone4.py
-2
@@ -168,8 +168,6 @@ class Exaone4Attention(nn.Module):
168
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
169
170
is_neox_style = True
171
- if quant_config is not None and quant_config.get_name() == "gguf":
172
- is_neox_style = False
171
172
layer_idx = extract_layer_index(prefix)
173
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
vllm/model_executor/models/gemma3.py
-9
@@ -377,15 +377,6 @@ class Gemma3Model(nn.Module):
377
params_dict = dict(self.named_parameters())
378
loaded_params: set[str] = set()
379
for name, loaded_weight in weights:
380
- # Revert +1 during llama.cpp conversion
381
- # see: https://github.com/ggml-org/llama.cpp/blob/be7c3034108473beda214fd1d7c98fd6a7a3bdf5/convert_hf_to_gguf.py#L3397-L3400
382
- if (
383
- self.quant_config
384
- and self.quant_config.get_name() == "gguf"
385
- and name.endswith("norm.weight")
386
- ):
387
- loaded_weight -= 1
388
-
380
# Check if this is a scale parameter that needs remapping first
381
if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")):
382
# Try to remap the scale name first
vllm/model_executor/models/jais2.py
-3
@@ -161,9 +161,6 @@ class Jais2Attention(nn.Module):
161
)
162
163
is_neox_style = True
164
- if quant_config is not None and quant_config.get_name() == "gguf":
165
- is_neox_style = False
166
-
164
self.rotary_emb = get_rope(
165
self.head_dim,
166
max_position=max_position_embeddings,
vllm/model_executor/models/llama.py
-3
@@ -239,9 +239,6 @@ class LlamaAttention(nn.Module):
239
quant_config: QuantizationConfig | None,
240
) -> None:
241
is_neox_style = True
242
- is_gguf = quant_config and quant_config.get_name() == "gguf"
243
- if is_gguf and config.model_type == "llama":
244
- is_neox_style = False
242
243
self.rotary_emb = get_rope(
244
self.head_dim,
vllm/model_executor/models/llama4.py
-3
@@ -238,9 +238,6 @@ class Llama4Attention(nn.Module):
238
prefix=f"{prefix}.o_proj",
239
)
240
is_neox_style = True
241
- is_gguf = quant_config and quant_config.get_name() == "gguf"
242
- if is_gguf and config.model_type == "llama":
243
- is_neox_style = False
241
242
self.rotary_emb = (
243
get_rope(
vllm/model_executor/models/olmoe.py
+2
@@ -279,12 +279,14 @@ class OlmoeModel(nn.Module):
279
super().__init__()
280
281
config = vllm_config.model_config.hf_config
282
+ quant_config = vllm_config.quant_config
283
284
self.vocab_size = config.vocab_size
285
self.config = config
286
self.embed_tokens = VocabParallelEmbedding(
287
config.vocab_size,
288
config.hidden_size,
289
+ quant_config=quant_config,
290
)
291
self.start_layer, self.end_layer, self.layers = make_layers(
292
config.num_hidden_layers,
vllm/model_executor/models/openpangu.py
-18
@@ -517,10 +517,6 @@ class OpenPanguEmbeddedAttention(nn.Module):
517
quant_config: QuantizationConfig | None,
518
) -> None:
519
is_neox_style = True
520
- is_gguf = quant_config and quant_config.get_name() == "gguf"
521
- if is_gguf and config.model_type == "PanguEmbedded":
522
- is_neox_style = False
523
-
520
rope_parameters = config.rope_parameters or {}
521
if rope_parameters is not None and rope_parameters.get(
522
"mrope_interleaved", False
@@ -716,20 +712,6 @@ class OpenPanguSinkAttention(nn.Module):
712
# no need to narrow
713
is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
714
719
- # Special case for GGUF
720
- is_gguf_weight = getattr(param, "is_gguf_weight", False)
721
- is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
722
- if is_gguf_weight_type:
723
- param.weight_type = loaded_weight.item()
724
-
725
- # Materialize GGUF UninitializedParameter
726
- if is_gguf_weight and isinstance(param, nn.UninitializedParameter):
727
- final_shape = list(loaded_weight.shape)
728
- if output_dim is not None:
729
- assert final_shape[output_dim] % self.tp_size == 0
730
- final_shape[output_dim] = final_shape[output_dim] // self.tp_size
731
- param.materialize(final_shape, dtype=loaded_weight.dtype)
732
-
715
param_data = param.data
716
if output_dim is not None and not is_sharded_weight:
717
shard_size = param_data.shape[output_dim]
vllm/model_executor/models/siglip.py
-26
@@ -952,38 +952,12 @@ class SiglipVisionModel(nn.Module):
952
break
953
else:
954
param = params_dict[name]
955
- param = maybe_swap_ffn_param(
956
- name, param, loaded_weight, params_dict, self.quant_config
957
- )
955
weight_loader = getattr(param, "weight_loader", default_weight_loader)
956
weight_loader(param, loaded_weight)
957
loaded_params.add(name)
958
return loaded_params
959
960
964
-def maybe_swap_ffn_param(
965
- name: str,
966
- param: torch.Tensor,
967
- loaded_weight: torch.Tensor,
968
- params_dict: dict[str, torch.Tensor],
969
- quant_config: QuantizationConfig,
970
-) -> torch.Tensor:
971
- if not (quant_config and quant_config.get_name() == "gguf") or ".fc" not in name:
972
- return param
973
- # Some GGUF models have fc1 and fc2 weights swapped
974
- tp_size = get_tensor_model_parallel_world_size()
975
- output_dim = getattr(param, "output_dim", 0)
976
- output_size = param.size(output_dim) * tp_size
977
- weight_out_size = loaded_weight.size(output_dim)
978
- if ".fc1." in name and output_size != weight_out_size:
979
- new_name = name.replace(".fc1.", ".fc2.")
980
- param = params_dict[new_name]
981
- elif ".fc2." in name and output_size != weight_out_size:
982
- new_name = name.replace(".fc2.", ".fc1.")
983
- param = params_dict[new_name]
984
- return param
985
-
986
-
961
# Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200
962
class SiglipTextEmbeddings(nn.Module):
963
def __init__(self, config: SiglipTextConfig):
vllm/platforms/rocm.py
-1
@@ -448,7 +448,6 @@ class RocmPlatform(Platform):
448
"deepseek_v4_fp8",
449
"compressed-tensors",
450
"fbgemm_fp8",
451
- "gguf",
451
"quark",
452
"mxfp4",
453
"mxfp8",
vllm/tokenizers/registry.py
-22
@@ -12,13 +12,6 @@ from typing_extensions import TypeVar, assert_never
12
import vllm.envs as envs
13
from vllm.logger import init_logger
14
from vllm.transformers_utils.config import get_config
15
-from vllm.transformers_utils.gguf_utils import (
16
- check_gguf_file,
17
- get_gguf_file_path_from_hf,
18
- is_gguf,
19
- is_remote_gguf,
20
- split_remote_gguf,
21
-)
15
from vllm.transformers_utils.repo_utils import (
16
any_pattern_in_repo_files,
17
is_mistral_model_repo,
@@ -124,21 +117,6 @@ def resolve_tokenizer_args(
117
)
118
tokenizer_name = tokenizer_path
119
127
- # Separate model folder from file path for GGUF models
128
- if is_gguf(tokenizer_name):
129
- if check_gguf_file(tokenizer_name):
130
- kwargs["gguf_file"] = Path(tokenizer_name).name
131
- tokenizer_name = Path(tokenizer_name).parent
132
- elif is_remote_gguf(tokenizer_name):
133
- tokenizer_name, quant_type = split_remote_gguf(tokenizer_name)
134
- # Get the HuggingFace Hub path for the GGUF file
135
- gguf_file = get_gguf_file_path_from_hf(
136
- tokenizer_name,
137
- quant_type,
138
- revision=revision,
139
- )
140
- kwargs["gguf_file"] = gguf_file
141
-
120
if "truncation_side" not in kwargs:
121
if runner_type == "generate" or runner_type == "draft":
122
kwargs["truncation_side"] = "left"
vllm/transformers_utils/config.py
+2
-92
@@ -19,7 +19,6 @@ from transformers import GenerationConfig, PretrainedConfig
19
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
20
from transformers.models.auto.image_processing_auto import get_image_processor_config
21
from transformers.models.auto.modeling_auto import (
22
- MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
22
MODEL_MAPPING_NAMES,
23
)
24
from transformers.models.auto.tokenization_auto import get_tokenizer_config
@@ -35,12 +34,6 @@ from vllm.transformers_utils.utils import (
34
from vllm.utils.torch_utils import common_broadcastable_dtype
35
36
from .config_parser_base import ConfigParserBase
38
-from .gguf_utils import (
39
- check_gguf_file,
40
- is_gguf,
41
- is_remote_gguf,
42
- split_remote_gguf,
43
-)
37
from .repo_utils import (
38
file_or_path_exists,
39
get_hf_file_to_dict,
@@ -611,17 +604,9 @@ def maybe_override_with_speculators(
604
Returns:
605
Tuple of (resolved_model, resolved_tokenizer, speculative_config)
606
"""
614
- if check_gguf_file(model):
615
- kwargs["gguf_file"] = Path(model).name
616
- gguf_model_repo = Path(model).parent
617
- elif is_remote_gguf(model):
618
- repo_id, _ = split_remote_gguf(model)
619
- gguf_model_repo = Path(repo_id)
620
- else:
621
- gguf_model_repo = None
607
kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
608
config_dict, _ = PretrainedConfig.get_config_dict(
624
- model if gguf_model_repo is None else gguf_model_repo,
609
+ model,
610
revision=revision,
611
token=hf_token,
612
**without_trust_remote_code(kwargs),
@@ -659,21 +644,6 @@ def get_config(
644
hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
645
**kwargs,
646
) -> PretrainedConfig:
662
- # Separate model folder from file path for GGUF models
663
-
664
- _is_gguf = is_gguf(model)
665
- _is_remote_gguf = is_remote_gguf(model)
666
- if _is_gguf:
667
- if check_gguf_file(model):
668
- # Local GGUF file
669
- kwargs["gguf_file"] = Path(model).name
670
- model = Path(model).parent
671
- elif _is_remote_gguf:
672
- # Remote GGUF - extract repo_id from repo_id:quant_type format
673
- # The actual GGUF file will be downloaded later by GGUFModelLoader
674
- # Keep model as repo_id:quant_type for download, but use repo_id for config
675
- model, _ = split_remote_gguf(model)
676
-
647
if config_format == "auto":
648
try:
649
# First check for Mistral to avoid defaulting to
@@ -684,25 +654,8 @@ def get_config(
654
model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
655
):
656
config_format = "mistral"
687
- elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
688
- model, HF_CONFIG_NAME, revision=revision
689
- ):
657
+ elif file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
658
config_format = "hf"
691
- # Remote GGUF models must have config.json in repo,
692
- # otherwise the config can't be parsed correctly.
693
- # FIXME(Isotr0py): Support remote GGUF repos without config.json
694
- elif _is_remote_gguf and not file_or_path_exists(
695
- model, HF_CONFIG_NAME, revision=revision
696
- ):
697
- err_msg = (
698
- "Could not find config.json for remote GGUF model repo. "
699
- "To load remote GGUF model through `<repo_id>:<quant_type>`, "
700
- "ensure your model has config.json (HF format) file. "
701
- "Otherwise please specify --hf-config-path <original_repo> "
702
- "in engine args to fetch config from unquantized hf model."
703
- )
704
- logger.error(err_msg)
705
- raise ValueError(err_msg)
659
else:
660
raise ValueError(
661
"Could not detect config format for no config file found. "
@@ -737,34 +690,6 @@ def get_config(
690
**kwargs,
691
)
692
740
- # Patching defaults for GGUF models
741
- if _is_gguf:
742
- # Some models have different default values between GGUF and HF.
743
- def apply_gguf_default(key: str, gguf_default: Any):
744
- """
745
- Apply GGUF defaults unless explicitly configured.
746
-
747
- This function reads/writes external `config` and `config_dict`.
748
- If the specified `key` is not in `config_dict` (i.e. not explicitly
749
- configured and the default HF value is used), it updates the
750
- corresponding `config` value to `gguf_default`.
751
- """
752
- if key not in config_dict:
753
- config.update({key: gguf_default})
754
-
755
- # Apply architecture-specific GGUF defaults.
756
- if config.model_type in {"qwen3_moe"}:
757
- # Qwen3 MoE: norm_topk_prob is always true.
758
- # Note that, this parameter is always false (HF default) on Qwen2 MoE.
759
- apply_gguf_default("norm_topk_prob", True)
760
-
761
- # Special architecture mapping check for GGUF models
762
- if _is_gguf:
763
- if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
764
- raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
765
- model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
766
- config.update({"architectures": [model_type]})
767
-
693
# Architecture mapping for models without explicit architectures field
694
if not config.architectures:
695
if config.model_type not in MODEL_MAPPING_NAMES:
@@ -856,9 +781,6 @@ def get_pooling_config(
781
A dictionary containing the pooling type and whether
782
normalization is used, or None if no pooling configuration is found.
783
"""
859
- if is_remote_gguf(model):
860
- model, _ = split_remote_gguf(model)
861
-
784
modules_file_name = "modules.json"
785
786
modules_dict = None
@@ -1074,11 +996,6 @@ def get_hf_image_processor_config(
996
# ModelScope does not provide an interface for image_processor
997
if envs.VLLM_USE_MODELSCOPE:
998
return dict()
1077
- # Separate model folder from file path for GGUF models
1078
- if check_gguf_file(model):
1079
- model = Path(model).parent
1080
- elif is_remote_gguf(model):
1081
- model, _ = split_remote_gguf(model)
999
return get_image_processor_config(
1000
model, token=hf_token, revision=revision, **kwargs
1001
)
@@ -1108,13 +1025,6 @@ def try_get_generation_config(
1025
config_format: str | ConfigFormat = "auto",
1026
hf_token: bool | str | None = None,
1027
) -> GenerationConfig | None:
1111
- # GGUF files don't have generation_config.json - their config is embedded
1112
- # in the file header. Skip all filesystem lookups to avoid re-reading the
1113
- # memory-mapped file, which can hang in multi-process scenarios when the
1114
- # EngineCore process already has the file mapped.
1115
- if is_gguf(model):
1116
- return None
1117
-
1028
try:
1029
return GenerationConfig.from_pretrained(
1030
model,
vllm/transformers_utils/gguf_utils.py
deleted
-336
@@ -1,336 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""GGUF utility functions."""
4
-
5
-from functools import cache
6
-from os import PathLike
7
-from pathlib import Path
8
-
9
-import gguf
10
-import regex as re
11
-from gguf.constants import Keys, VisionProjectorType
12
-from gguf.quants import GGMLQuantizationType
13
-from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig
14
-
15
-from vllm.logger import init_logger
16
-
17
-from .repo_utils import list_filtered_repo_files
18
-
19
-logger = init_logger(__name__)
20
-
21
-
22
-@cache
23
-def check_gguf_file(model: str | PathLike) -> bool:
24
- """Check if the file is a GGUF model."""
25
- model = Path(model)
26
- if not model.is_file():
27
- return False
28
- elif model.suffix == ".gguf":
29
- return True
30
-
31
- try:
32
- with model.open("rb") as f:
33
- header = f.read(4)
34
-
35
- return header == b"GGUF"
36
- except Exception as e:
37
- logger.debug("Error reading file %s: %s", model, e)
38
- return False
39
-
40
-
41
-@cache
42
-def is_remote_gguf(model: str | Path) -> bool:
43
- """Check if the model is a remote GGUF model.
44
-
45
- Recognizes two forms:
46
- 1. Standard: ``repo_id:quant_type`` where *quant_type* is a known
47
- GGML quantization type (e.g. ``Q4_K_M``).
48
- 2. Non-standard: ``repo_id:quant_type`` where *quant_type* contains
49
- a known GGML type with extra prefixes (e.g. ``UD-Q4_K_XL``).
50
- A warning is logged and actual file existence is validated later
51
- during download.
52
- """
53
- pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*:[A-Za-z0-9_+-]+$"
54
- model = str(model)
55
- if re.fullmatch(pattern, model):
56
- _, quant_type = model.rsplit(":", 1)
57
- if is_valid_gguf_quant_type(quant_type):
58
- return True
59
- if is_nonstandard_gguf_quant_type(quant_type):
60
- logger.warning(
61
- "Non-standard GGUF quant type '%s' detected.",
62
- quant_type,
63
- )
64
- return True
65
- return False
66
-
67
-
68
-def is_nonstandard_gguf_quant_type(quant_type: str) -> bool:
69
- """Check if a non-standard quant type contains a known GGML type.
70
-
71
- Splits the quant type by the last ``-`` and checks whether the
72
- trailing part is a standard GGML type. For example::
73
-
74
- UD-Q4_K_XL → rsplit → ["UD", "Q4_K_XL"] → Q4_K_XL valid ✓
75
- UD-IQ4_NL → rsplit → ["UD", "IQ4_NL"] → IQ4_NL valid ✓
76
- Custom-UD-Q4_K → rsplit → ["Custom-UD", "Q4_K"] → Q4_K valid ✓
77
- RANDOM → no "-" → False
78
- """
79
- if "-" not in quant_type:
80
- return False
81
- _, remainder = quant_type.rsplit("-", 1)
82
- return is_valid_gguf_quant_type(remainder)
83
-
84
-
85
-# Common suffixes used in GGUF file naming conventions
86
-# e.g., Q4_K_M, Q3_K_S, Q5_K_L, Q2_K_XL
87
-_GGUF_QUANT_SUFFIXES = ("_M", "_S", "_L", "_XL", "_XS", "_XXS")
88
-
89
-
90
-def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
91
- """Check if the quant type is a valid GGUF quant type.
92
-
93
- Supports both exact GGML quant types (e.g., Q4_K, IQ1_S) and
94
- extended naming conventions (e.g., Q4_K_M, Q3_K_S, Q5_K_L).
95
- """
96
- # Check for exact match first
97
- if getattr(GGMLQuantizationType, gguf_quant_type, None) is not None:
98
- return True
99
-
100
- # Check for extended naming conventions (e.g., Q4_K_M -> Q4_K)
101
- for suffix in _GGUF_QUANT_SUFFIXES:
102
- if gguf_quant_type.endswith(suffix):
103
- base_type = gguf_quant_type[: -len(suffix)]
104
- if getattr(GGMLQuantizationType, base_type, None) is not None:
105
- return True
106
-
107
- return False
108
-
109
-
110
-def split_remote_gguf(model: str | Path) -> tuple[str, str]:
111
- """Split the model into repo_id and quant type."""
112
- model = str(model)
113
- if is_remote_gguf(model):
114
- parts = model.rsplit(":", 1)
115
- return (parts[0], parts[1])
116
- raise ValueError(
117
- f"Wrong GGUF model or invalid GGUF quant type: {model}.\n"
118
- "- It should be in repo_id:quant_type format.\n"
119
- f"- Valid base quant types: {GGMLQuantizationType._member_names_}\n"
120
- f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}\n"
121
- "- Non-standard GGUF quant types also supported: "
122
- "dash-separated prefixes (e.g. UD-Q4_K_XL, Custom-Q8_0)",
123
- )
124
-
125
-
126
-def is_gguf(model: str | Path) -> bool:
127
- """Check if the model is a GGUF model.
128
-
129
- Args:
130
- model: Model name, path, or Path object to check.
131
-
132
- Returns:
133
- True if the model is a GGUF model, False otherwise.
134
- """
135
- model = str(model)
136
-
137
- # Check if it's a local GGUF file
138
- if check_gguf_file(model):
139
- return True
140
-
141
- # Check if it's a remote GGUF model (repo_id:quant_type format)
142
- return is_remote_gguf(model)
143
-
144
-
145
-def detect_gguf_multimodal(model: str) -> Path | None:
146
- """Check if GGUF model has multimodal projector file.
147
-
148
- Args:
149
- model: Model path string
150
-
151
- Returns:
152
- Path to mmproj file if found, None otherwise
153
- """
154
- if not model.endswith(".gguf"):
155
- return None
156
-
157
- try:
158
- model_path = Path(model)
159
- if not model_path.is_file():
160
- return None
161
-
162
- model_dir = model_path.parent
163
- mmproj_patterns = ["mmproj.gguf", "mmproj-*.gguf", "*mmproj*.gguf"]
164
- for pattern in mmproj_patterns:
165
- mmproj_files = list(model_dir.glob(pattern))
166
- if mmproj_files:
167
- return mmproj_files[0]
168
- return None
169
- except Exception:
170
- return None
171
-
172
-
173
-def extract_vision_config_from_gguf(mmproj_path: str) -> "SiglipVisionConfig | None":
174
- """Extract vision config parameters from mmproj.gguf metadata.
175
-
176
- Reads vision encoder configuration from GGUF metadata fields using
177
- standardized GGUF constants. Automatically detects the projector type
178
- (e.g., gemma3, llama4) and applies model-specific parameters accordingly.
179
-
180
- The function extracts standard CLIP vision parameters from GGUF metadata
181
- and applies projector-type-specific customizations. For unknown projector
182
- types, it uses safe defaults from SiglipVisionConfig.
183
-
184
- Args:
185
- mmproj_path: Path to mmproj.gguf file (str or Path)
186
-
187
- Returns:
188
- SiglipVisionConfig if extraction succeeds, None if any required
189
- field is missing from the GGUF metadata
190
-
191
- Raises:
192
- Exception: Exceptions from GGUF reading (file not found, corrupted
193
- file, etc.) propagate directly from gguf.GGUFReader
194
- """
195
- reader = gguf.GGUFReader(str(mmproj_path))
196
-
197
- # Detect projector type to apply model-specific parameters
198
- projector_type = None
199
- projector_type_field = reader.get_field(Keys.Clip.PROJECTOR_TYPE)
200
- if projector_type_field:
201
- try:
202
- projector_type = bytes(projector_type_field.parts[-1]).decode("utf-8")
203
- except (AttributeError, UnicodeDecodeError) as e:
204
- logger.warning("Failed to decode projector type from GGUF: %s", e)
205
-
206
- # Map GGUF field constants to SiglipVisionConfig parameters.
207
- # Uses official GGUF constants from gguf-py for standardization.
208
- # Format: {gguf_constant: (param_name, dtype)}
209
- VISION_CONFIG_FIELDS = {
210
- Keys.ClipVision.EMBEDDING_LENGTH: ("hidden_size", int),
211
- Keys.ClipVision.FEED_FORWARD_LENGTH: ("intermediate_size", int),
212
- Keys.ClipVision.BLOCK_COUNT: ("num_hidden_layers", int),
213
- Keys.ClipVision.Attention.HEAD_COUNT: ("num_attention_heads", int),
214
- Keys.ClipVision.IMAGE_SIZE: ("image_size", int),
215
- Keys.ClipVision.PATCH_SIZE: ("patch_size", int),
216
- Keys.ClipVision.Attention.LAYERNORM_EPS: ("layer_norm_eps", float),
217
- }
218
-
219
- # Extract and validate all required fields
220
- config_params = {}
221
- for gguf_key, (param_name, dtype) in VISION_CONFIG_FIELDS.items():
222
- field = reader.get_field(gguf_key)
223
- if field is None:
224
- logger.warning(
225
- "Missing required vision config field '%s' in mmproj.gguf",
226
- gguf_key,
227
- )
228
- return None
229
- # Extract scalar value from GGUF field and convert to target type
230
- config_params[param_name] = dtype(field.parts[-1])
231
-
232
- # Apply model-specific parameters based on projector type
233
- if projector_type == VisionProjectorType.GEMMA3:
234
- # Gemma3 doesn't use the vision pooling head (multihead attention)
235
- # This is a vLLM-specific parameter used in SiglipVisionTransformer
236
- config_params["vision_use_head"] = False
237
- logger.info("Detected Gemma3 projector, disabling vision pooling head")
238
- # Add other projector-type-specific customizations here as needed
239
- # elif projector_type == VisionProjectorType.LLAMA4:
240
- # config_params["vision_use_head"] = ...
241
-
242
- # Create config with extracted parameters
243
- # Note: num_channels and attention_dropout use SiglipVisionConfig defaults
244
- # (3 and 0.0 respectively) which are correct for all models
245
- config = SiglipVisionConfig(**config_params)
246
-
247
- if projector_type:
248
- logger.info(
249
- "Extracted vision config from mmproj.gguf (projector_type: %s)",
250
- projector_type,
251
- )
252
- else:
253
- logger.info("Extracted vision config from mmproj.gguf metadata")
254
-
255
- return config
256
-
257
-
258
-def maybe_patch_hf_config_from_gguf(
259
- model: str,
260
- hf_config: PretrainedConfig,
261
-) -> PretrainedConfig:
262
- """Patch HF config for GGUF models.
263
-
264
- Applies GGUF-specific patches to HuggingFace config:
265
- 1. For multimodal models: patches architecture and vision config
266
- 2. For all GGUF models: overrides vocab_size from embedding tensor
267
-
268
- This ensures compatibility with GGUF models that have extended
269
- vocabularies (e.g., Unsloth) where the GGUF file contains more
270
- tokens than the HuggingFace tokenizer config specifies.
271
-
272
- Args:
273
- model: Model path string
274
- hf_config: HuggingFace config to patch in-place
275
-
276
- Returns:
277
- Updated HuggingFace config
278
- """
279
- # Patch multimodal config if mmproj.gguf exists
280
- mmproj_path = detect_gguf_multimodal(model)
281
- if mmproj_path is not None:
282
- vision_config = extract_vision_config_from_gguf(str(mmproj_path))
283
-
284
- # Create HF config for Gemma3 multimodal
285
- text_config = hf_config.get_text_config()
286
- is_gemma3 = hf_config.model_type in ("gemma3", "gemma3_text")
287
- if vision_config is not None and is_gemma3:
288
- new_hf_config = Gemma3Config(
289
- text_config=text_config,
290
- vision_config=vision_config,
291
- architectures=["Gemma3ForConditionalGeneration"],
292
- )
293
- hf_config = new_hf_config
294
-
295
- return hf_config
296
-
297
-
298
-def get_gguf_file_path_from_hf(
299
- repo_id: str | Path,
300
- quant_type: str,
301
- revision: str | None = None,
302
-) -> str:
303
- """Get the GGUF file path from HuggingFace Hub based on repo_id and quant_type.
304
-
305
- Args:
306
- repo_id: The HuggingFace repository ID (e.g., "Qwen/Qwen3-0.6B")
307
- quant_type: The quantization type (e.g., "Q4_K_M", "F16")
308
- revision: Optional revision/branch name
309
-
310
- Returns:
311
- The path to the GGUF file on HuggingFace Hub (e.g., "filename.gguf"),
312
- """
313
- repo_id = str(repo_id)
314
- gguf_patterns = [
315
- f"*-{quant_type}.gguf",
316
- f"*-{quant_type}-*.gguf",
317
- f"*/*-{quant_type}.gguf",
318
- f"*/*-{quant_type}-*.gguf",
319
- ]
320
- matching_files = list_filtered_repo_files(
321
- repo_id,
322
- allow_patterns=gguf_patterns,
323
- revision=revision,
324
- )
325
-
326
- if len(matching_files) == 0:
327
- raise ValueError(
328
- "Could not find GGUF file for repo %s with quantization %s.",
329
- repo_id,
330
- quant_type,
331
- )
332
-
333
- # Sort to ensure consistent ordering (prefer non-sharded files)
334
- matching_files.sort(key=lambda x: (x.count("-"), x))
335
- gguf_filename = matching_files[0]
336
- return gguf_filename
vllm/transformers_utils/processor.py
+6
-37
@@ -25,7 +25,6 @@ from typing_extensions import TypeVar
25
26
from vllm.logger import init_logger
27
from vllm.transformers_utils import processors
28
-from vllm.transformers_utils.gguf_utils import is_gguf
28
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
29
from vllm.transformers_utils.utils import convert_model_repo_to_path
30
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -181,17 +180,8 @@ _cached_get_video_processor_cls_name = lru_cache(
180
def get_video_processor_cls_name(
181
model_config: "ModelConfig",
182
) -> str | None:
184
- if is_gguf(model_config.model):
185
- assert not is_gguf(model_config.tokenizer), (
186
- "For multimodal GGUF models, the original tokenizer "
187
- "should be used to correctly load video processor metadata."
188
- )
189
- model = model_config.tokenizer
190
- revision = model_config.tokenizer_revision
191
- else:
192
- model = model_config.model
193
- revision = model_config.revision
194
-
183
+ model = model_config.model
184
+ revision = model_config.revision
185
return _cached_get_video_processor_cls_name(model, revision=revision)
186
187
@@ -375,20 +365,9 @@ def cached_processor_from_config(
365
processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
366
**kwargs: Any,
367
) -> _P:
378
- if is_gguf(model_config.model):
379
- assert not is_gguf(model_config.tokenizer), (
380
- "For multimodal GGUF models, the original tokenizer "
381
- "should be used to correctly load processor."
382
- )
383
- model = model_config.tokenizer
384
- revision = model_config.tokenizer_revision
385
- else:
386
- model = model_config.model
387
- revision = model_config.revision
388
-
368
return cached_get_processor_without_dynamic_kwargs(
390
- model,
391
- revision=revision,
369
+ model_config.model,
370
+ revision=model_config.revision,
371
trust_remote_code=model_config.trust_remote_code,
372
processor_cls=processor_cls, # type: ignore[arg-type]
373
**_merge_mm_kwargs(model_config, processor_cls, **kwargs),
@@ -489,19 +468,9 @@ def cached_image_processor_from_config(
468
model_config: "ModelConfig",
469
**kwargs: Any,
470
):
492
- if is_gguf(model_config.model):
493
- assert not is_gguf(model_config.tokenizer), (
494
- "For multimodal GGUF models, the original tokenizer "
495
- "should be used to correctly load image processor."
496
- )
497
- model = model_config.tokenizer
498
- revision = model_config.tokenizer_revision
499
- else:
500
- model = model_config.model
501
- revision = model_config.revision
471
return cached_get_image_processor(
503
- model,
504
- revision=revision,
472
+ model_config.model,
473
+ revision=model_config.revision,
474
trust_remote_code=model_config.trust_remote_code,
475
**_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
476
)
vllm/v1/metrics/perf.py
-1
@@ -66,7 +66,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {
66
"bitsandbytes": 0.5,
67
"modelopt_fp4": 0.5,
68
"petit_nvfp4": 0.5,
69
- "gguf": 0.5,
69
"compressed-tensors": 0.5,
70
"torchao": 0.5,
71
"quark": 0.5,