A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[AMD] Add support for GGUF quantization on ROCm (#10254)
kliuae committed
Nov 23, 2024 at 13:14 UTC
7c25fe45a6ef4fb5be148217cc7110e88e186446
11 files changed
+234
-211
.buildkite/run-amd-test.sh
-1
@@ -85,7 +85,6 @@ if [[ $commands == *" kernels "* ]]; then
85
--ignore=kernels/test_encoder_decoder_attn.py \
86
--ignore=kernels/test_flash_attn.py \
87
--ignore=kernels/test_flashinfer.py \
88
- --ignore=kernels/test_gguf.py \
88
--ignore=kernels/test_int8_quant.py \
89
--ignore=kernels/test_machete_gemm.py \
90
--ignore=kernels/test_mamba_ssm.py \
CMakeLists.txt
+1
-1
@@ -196,6 +196,7 @@ set(VLLM_EXT_SRC
196
"csrc/quantization/gptq/q_gemm.cu"
197
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
198
"csrc/quantization/fp8/common.cu"
199
+ "csrc/quantization/gguf/gguf_kernel.cu"
200
"csrc/cuda_utils_kernels.cu"
201
"csrc/prepare_inputs/advance_step.cu"
202
"csrc/torch_bindings.cpp")
@@ -237,7 +238,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
238
"csrc/mamba/causal_conv1d/causal_conv1d.cu"
239
"csrc/quantization/aqlm/gemm_kernels.cu"
240
"csrc/quantization/awq/gemm_kernels.cu"
240
- "csrc/quantization/gguf/gguf_kernel.cu"
241
"csrc/custom_all_reduce.cu"
242
"csrc/permute_cols.cu"
243
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu")
csrc/ops.h
+2
@@ -128,6 +128,7 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
128
int64_t thx, int64_t thy);
129
130
torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
131
+#endif
132
133
torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
134
int64_t n);
@@ -138,6 +139,7 @@ torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
139
torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
140
int64_t row);
141
142
+#ifndef USE_ROCM
143
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
144
145
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
csrc/quantization/gguf/ggml-common.h
+16
-1
@@ -1,7 +1,7 @@
1
// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
2
#define QK_K 256
3
#define K_QUANTS_PER_ITERATION 2
4
-#define WARP_SIZE 32
4
+#define WARP_SIZE_GGUF 32
5
#define K_SCALE_SIZE 12
6
#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
7
#define CUDA_QUANTIZE_BLOCK_SIZE 256
@@ -1112,4 +1112,19 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
1112
#endif
1113
return c;
1114
}
1115
+
1116
+static __device__ __forceinline__ uint32_t __vcmpeq4(const uint32_t a, const uint32_t b) {
1117
+ uint32_t neq = a^b;
1118
+ return !(neq & 0xff000000) * 0xff000000 |
1119
+ !(neq & 0x00ff0000) * 0x00ff0000 |
1120
+ !(neq & 0x0000ff00) * 0x0000ff00 |
1121
+ !(neq & 0x000000ff) * 0x000000ff;
1122
+}
1123
+
1124
+static __device__ __forceinline__ uint32_t __vsub4(const uint32_t a, const uint32_t b) {
1125
+ return (static_cast<uint8_t>(((a & 0xff000000) >> 24) - ((b & 0xff000000) >> 24)) << 24) +
1126
+ (static_cast<uint8_t>(((a & 0x00ff0000) >> 16) - ((b & 0x00ff0000) >> 16)) << 16) +
1127
+ (static_cast<uint8_t>(((a & 0x0000ff00) >> 8) - ((b & 0x0000ff00) >> 8)) << 8) +
1128
+ (static_cast<uint8_t>(((a & 0x000000ff) >> 0) - ((b & 0x000000ff) >> 0)) << 0);
1129
+}
1130
#endif // defined(USE_ROCM)
csrc/quantization/gguf/gguf_kernel.cu
+4
-2
@@ -4,6 +4,8 @@
4
#include <torch/all.h>
5
#include <c10/cuda/CUDAGuard.h>
6
7
+#include "cuda_compat.h"
8
+
9
#include "ggml-common.h"
10
#include "vecdotq.cuh"
11
#include "dequantize.cuh"
@@ -32,8 +34,8 @@ static __global__ void quantize_q8_1(const half* __restrict__ x,
34
35
#pragma unroll
36
for (int mask = 16; mask > 0; mask >>= 1) {
35
- amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, mask, 32));
36
- sum += __shfl_xor_sync(0xffffffff, sum, mask, 32);
37
+ amax = fmaxf(amax, VLLM_SHFL_XOR_SYNC_WIDTH(amax, mask, 32));
38
+ sum += VLLM_SHFL_XOR_SYNC_WIDTH(sum, mask, 32);
39
}
40
41
const float d = amax / 127;
csrc/quantization/gguf/mmq.cuh
+35
-35
@@ -10,7 +10,7 @@ static __device__ __forceinline__ void mul_mat_q(
10
11
const int blocks_per_row_x = ncols_x / qk;
12
const int blocks_per_col_y = nrows_y / QK8_1;
13
- const int blocks_per_warp = WARP_SIZE / qi;
13
+ const int blocks_per_warp = WARP_SIZE_GGUF / qi;
14
15
const int & ncols_dst = ncols_y;
16
@@ -27,10 +27,10 @@ static __device__ __forceinline__ void mul_mat_q(
27
28
allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
29
30
- __shared__ int tile_y_qs[mmq_x * WARP_SIZE];
31
- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];
30
+ __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
31
+ __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF/QI8_1];
32
33
- float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};
33
+ float sum[mmq_y/WARP_SIZE_GGUF][mmq_x/nwarps] = {{0.0f}};
34
35
for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
36
@@ -39,26 +39,26 @@ static __device__ __forceinline__ void mul_mat_q(
39
40
#pragma unroll
41
for (int ir = 0; ir < qr; ++ir) {
42
- const int kqs = ir*WARP_SIZE + threadIdx.x;
42
+ const int kqs = ir*WARP_SIZE_GGUF + threadIdx.x;
43
const int kbxd = kqs / QI8_1;
44
45
#pragma unroll
46
for (int i = 0; i < mmq_x; i += nwarps) {
47
const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses
48
const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];
49
- const int index_y = (threadIdx.y + i) * WARP_SIZE + kqs % WARP_SIZE;
49
+ const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
50
tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
51
}
52
53
#pragma unroll
54
for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
55
- const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE/QI8_1)) % mmq_x;
56
- const int kby = threadIdx.x % (WARP_SIZE/QI8_1);
55
+ const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF/QI8_1)) % mmq_x;
56
+ const int kby = threadIdx.x % (WARP_SIZE_GGUF/QI8_1);
57
const int col_y_eff = min(col_y_0 + ids, ncols_y-1);
58
59
// if the sum is not needed it's faster to transform the scale to f32 ahead of time
60
- const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE/QI8_1) + kby].ds;
61
- half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE/QI8_1) + kby];
60
+ const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE_GGUF/QI8_1) + kby].ds;
61
+ half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF/QI8_1) + kby];
62
if (need_sum) {
63
*dsi_dst = *dsi_src;
64
} else {
@@ -70,12 +70,12 @@ static __device__ __forceinline__ void mul_mat_q(
70
__syncthreads();
71
72
// #pragma unroll // unrolling this loop causes too much register pressure
73
- for (int k = ir*WARP_SIZE/qr; k < (ir+1)*WARP_SIZE/qr; k += vdr) {
73
+ for (int k = ir*WARP_SIZE_GGUF/qr; k < (ir+1)*WARP_SIZE_GGUF/qr; k += vdr) {
74
#pragma unroll
75
for (int j = 0; j < mmq_x; j += nwarps) {
76
#pragma unroll
77
- for (int i = 0; i < mmq_y; i += WARP_SIZE) {
78
- sum[i/WARP_SIZE][j/nwarps] += vec_dot(
77
+ for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
78
+ sum[i/WARP_SIZE_GGUF][j/nwarps] += vec_dot(
79
tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,
80
threadIdx.x + i, threadIdx.y + j, k);
81
}
@@ -93,12 +93,12 @@ static __device__ __forceinline__ void mul_mat_q(
93
}
94
95
#pragma unroll
96
- for (int i = 0; i < mmq_y; i += WARP_SIZE) {
96
+ for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
97
const int row_dst = row_dst_0 + threadIdx.x + i;
98
if (row_dst >= nrows_dst) {
99
continue;
100
}
101
- dst[col_dst*nrows_dst + row_dst] = __float2half(sum[i/WARP_SIZE][j/nwarps]);
101
+ dst[col_dst*nrows_dst + row_dst] = __float2half(sum[i/WARP_SIZE_GGUF][j/nwarps]);
102
}
103
}
104
}
@@ -115,7 +115,7 @@ static __device__ __forceinline__ void mul_mat_q(
115
116
template <bool need_check> static __global__ void
117
#if defined(USE_ROCM)
118
-__launch_bounds__(WARP_SIZE*NWARPS_Q4_0, 2)
118
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_0, 2)
119
#endif
120
mul_mat_q4_0(
121
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -140,7 +140,7 @@ static void ggml_mul_mat_q4_0_q8_1_cuda(
140
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
141
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
142
const dim3 block_nums(block_num_x, block_num_y, 1);
143
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
143
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
144
145
if (nrows_x % mmq_y == 0) {
146
const bool need_check = false;
@@ -165,7 +165,7 @@ static void ggml_mul_mat_q4_0_q8_1_cuda(
165
166
template <bool need_check> static __global__ void
167
#if defined(USE_ROCM)
168
-__launch_bounds__(WARP_SIZE*NWARPS_Q4_1, 2)
168
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_1, 2)
169
#endif
170
mul_mat_q4_1(
171
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -190,7 +190,7 @@ static void ggml_mul_mat_q4_1_q8_1_cuda(
190
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
191
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
192
const dim3 block_nums(block_num_x, block_num_y, 1);
193
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
193
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
194
195
if (nrows_x % mmq_y == 0) {
196
const bool need_check = false;
@@ -215,7 +215,7 @@ static void ggml_mul_mat_q4_1_q8_1_cuda(
215
216
template <bool need_check> static __global__ void
217
#if defined(USE_ROCM)
218
-__launch_bounds__(WARP_SIZE*NWARPS_Q5_0, 2)
218
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_0, 2)
219
#endif
220
mul_mat_q5_0(
221
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -240,7 +240,7 @@ static void ggml_mul_mat_q5_0_q8_1_cuda(
240
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
241
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
242
const dim3 block_nums(block_num_x, block_num_y, 1);
243
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
243
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
244
245
if (nrows_x % mmq_y == 0) {
246
const bool need_check = false;
@@ -265,7 +265,7 @@ static void ggml_mul_mat_q5_0_q8_1_cuda(
265
266
template <bool need_check> static __global__ void
267
#if defined(USE_ROCM)
268
-__launch_bounds__(WARP_SIZE*NWARPS_Q5_1, 2)
268
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_1, 2)
269
#endif
270
mul_mat_q5_1(
271
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -289,7 +289,7 @@ static void ggml_mul_mat_q5_1_q8_1_cuda(
289
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
290
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
291
const dim3 block_nums(block_num_x, block_num_y, 1);
292
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
292
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
293
294
if (nrows_x % mmq_y == 0) {
295
const bool need_check = false;
@@ -314,7 +314,7 @@ static void ggml_mul_mat_q5_1_q8_1_cuda(
314
315
template <bool need_check> static __global__ void
316
#if defined(USE_ROCM)
317
-__launch_bounds__(WARP_SIZE*NWARPS_Q8_0, 2)
317
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q8_0, 2)
318
#endif
319
mul_mat_q8_0(
320
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -338,7 +338,7 @@ static void ggml_mul_mat_q8_0_q8_1_cuda(
338
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
339
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
340
const dim3 block_nums(block_num_x, block_num_y, 1);
341
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
341
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
342
343
if (nrows_x % mmq_y == 0) {
344
const bool need_check = false;
@@ -363,7 +363,7 @@ static void ggml_mul_mat_q8_0_q8_1_cuda(
363
364
template <bool need_check> static __global__ void
365
#if defined(USE_ROCM)
366
-__launch_bounds__(WARP_SIZE*NWARPS_Q2_K, 2)
366
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q2_K, 2)
367
#endif
368
mul_mat_q2_K(
369
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -387,7 +387,7 @@ static void ggml_mul_mat_q2_K_q8_1_cuda(
387
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
388
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
389
const dim3 block_nums(block_num_x, block_num_y, 1);
390
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
390
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
391
392
if (nrows_x % mmq_y == 0) {
393
const bool need_check = false;
@@ -412,7 +412,7 @@ static void ggml_mul_mat_q2_K_q8_1_cuda(
412
413
template <bool need_check> static __global__ void
414
#if defined(USE_ROCM)
415
-__launch_bounds__(WARP_SIZE*NWARPS_Q3_K, 2)
415
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q3_K, 2)
416
#endif
417
mul_mat_q3_K(
418
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -438,7 +438,7 @@ static void ggml_mul_mat_q3_K_q8_1_cuda(
438
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
439
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
440
const dim3 block_nums(block_num_x, block_num_y, 1);
441
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
441
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
442
443
if (nrows_x % mmq_y == 0) {
444
const bool need_check = false;
@@ -463,7 +463,7 @@ static void ggml_mul_mat_q3_K_q8_1_cuda(
463
464
template <bool need_check> static __global__ void
465
#if defined(USE_ROCM)
466
-__launch_bounds__(WARP_SIZE*NWARPS_Q4_K, 2)
466
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_K, 2)
467
#endif
468
mul_mat_q4_K(
469
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -487,7 +487,7 @@ static void ggml_mul_mat_q4_K_q8_1_cuda(
487
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
488
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
489
const dim3 block_nums(block_num_x, block_num_y, 1);
490
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
490
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
491
492
if (nrows_x % mmq_y == 0) {
493
const bool need_check = false;
@@ -512,7 +512,7 @@ static void ggml_mul_mat_q4_K_q8_1_cuda(
512
513
template <bool need_check> static __global__ void
514
#if defined(USE_ROCM)
515
-__launch_bounds__(WARP_SIZE*NWARPS_Q5_K, 2)
515
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_K, 2)
516
#endif
517
mul_mat_q5_K(
518
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -537,7 +537,7 @@ static void ggml_mul_mat_q5_K_q8_1_cuda(
537
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
538
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
539
const dim3 block_nums(block_num_x, block_num_y, 1);
540
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
540
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
541
542
if (nrows_x % mmq_y == 0) {
543
const bool need_check = false;
@@ -562,7 +562,7 @@ static void ggml_mul_mat_q5_K_q8_1_cuda(
562
563
template <bool need_check> static __global__ void
564
#if defined(USE_ROCM)
565
-__launch_bounds__(WARP_SIZE*NWARPS_Q6_K, 2)
565
+__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q6_K, 2)
566
#endif
567
mul_mat_q6_K(
568
const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -586,7 +586,7 @@ static void ggml_mul_mat_q6_K_q8_1_cuda(
586
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
587
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
588
const dim3 block_nums(block_num_x, block_num_y, 1);
589
- const dim3 block_dims(WARP_SIZE, nwarps, 1);
589
+ const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
590
591
if (nrows_x % mmq_y == 0) {
592
const bool need_check = false;
csrc/quantization/gguf/mmvq.cuh
+2
-2
@@ -28,8 +28,8 @@ static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void *
28
29
// sum up partial sums and write back result
30
#pragma unroll
31
- for (int mask = 16; mask > 0; mask >>= 1) {
32
- tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32);
31
+ for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1) {
32
+ tmp += VLLM_SHFL_XOR_SYNC(tmp, mask);
33
}
34
35
if (threadIdx.x == 0) {
csrc/quantization/gguf/vecdotq.cuh
+143
-143
@@ -43,7 +43,7 @@ static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t *
43
44
template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
45
const int * v, const int * u, const float & d4, const half2 & ds8) {
46
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
46
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
47
int sumi = 0;
48
49
#pragma unroll
@@ -68,7 +68,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_imp
68
69
template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
70
const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
71
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
71
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
72
int sumi = 0;
73
74
#pragma unroll
@@ -95,7 +95,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_imp
95
96
template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
97
const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
98
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
98
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
99
int sumi = 0;
100
101
#pragma unroll
@@ -128,7 +128,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_imp
128
129
template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
130
const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
131
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
131
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
132
int sumi = 0;
133
134
#pragma unroll
@@ -162,7 +162,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_imp
162
163
template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(
164
const int * v, const int * u, const float & d8_0, const float & d8_1) {
165
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
165
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
166
int sumi = 0;
167
168
#pragma unroll
@@ -176,7 +176,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_imp
176
177
template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
178
const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
179
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
179
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
180
181
int sumi = 0;
182
@@ -202,7 +202,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_imp
202
static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
203
const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
204
const half2 & dm2, const float * __restrict__ d8) {
205
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
205
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
206
float sumf_d = 0.0f;
207
float sumf_m = 0.0f;
208
@@ -230,7 +230,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
230
static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
231
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
232
const half2 & dm2, const float & d8) {
233
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
233
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
234
int sumi_d = 0;
235
int sumi_m = 0;
236
@@ -267,7 +267,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
267
static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
268
const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
269
const int & scale_offset, const float & d3, const float * __restrict__ d8) {
270
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
270
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
271
272
float sumf = 0.0f;
273
@@ -301,7 +301,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
301
static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
302
const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
303
const float & d3, const float & d8) {
304
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
304
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
305
int sumi = 0;
306
307
#pragma unroll
@@ -326,7 +326,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
326
static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
327
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
328
const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
329
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
329
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
330
331
float sumf_d = 0.0f;
332
float sumf_m = 0.0f;
@@ -351,7 +351,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
351
static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
352
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
353
const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
354
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
354
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
355
float sumf_d = 0.0f;
356
float sumf_m = 0.0f;
357
@@ -382,7 +382,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
382
static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
383
const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
384
const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
385
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
385
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
386
387
float sumf_d = 0.0f;
388
float sumf_m = 0.0f;
@@ -413,7 +413,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
413
static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
414
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
415
const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
416
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
416
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
417
float sumf_d = 0.0f;
418
float sumf_m = 0.0f;
419
@@ -445,7 +445,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
445
static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
446
const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
447
const float & d, const float * __restrict__ d8) {
448
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
448
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
449
float sumf = 0.0f;
450
451
#pragma unroll
@@ -465,7 +465,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
465
static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
466
const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
467
const float & d6, const float * __restrict__ d8) {
468
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
468
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
469
float sumf_d = 0.0f;
470
471
#pragma unroll
@@ -507,8 +507,8 @@ static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
507
}
508
509
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
510
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
511
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];
510
+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
511
+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI4_0) + mmq_y/QI4_0];
512
*x_ql = tile_x_qs;
513
*x_dm = (half2 *) tile_x_d;
514
}
@@ -529,11 +529,11 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
529
i = min(i, i_max);
530
}
531
const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;
532
- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
533
- // x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbx] = bxi->d;
532
+ x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
533
+ // x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbx] = bxi->d;
534
}
535
536
- const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;
536
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_0;
537
const int kbxd = k % blocks_per_tile_x_row;
538
539
#pragma unroll
@@ -543,7 +543,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
543
i = min(i, i_max);
544
}
545
const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;
546
- x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
546
+ x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
547
}
548
}
549
@@ -559,13 +559,13 @@ static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(
559
560
#pragma unroll
561
for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
562
- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
563
- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_0) % WARP_SIZE];
562
+ u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
563
+ u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_0) % WARP_SIZE_GGUF];
564
}
565
566
return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
567
- (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dmf[i * (WARP_SIZE/QI4_0) + i/QI4_0 + k/QI4_0],
568
- y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
567
+ (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], u, x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i/QI4_0 + k/QI4_0],
568
+ y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
569
}
570
571
static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
@@ -587,8 +587,8 @@ static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
587
}
588
589
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
590
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];
591
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];
590
+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + + mmq_y];
591
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI4_1) + mmq_y/QI4_1];
592
*x_ql = tile_x_qs;
593
*x_dm = tile_x_dm;
594
}
@@ -608,10 +608,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
608
i = min(i, i_max);
609
}
610
const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;
611
- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
611
+ x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
612
}
613
614
- const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;
614
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_1;
615
const int kbxd = k % blocks_per_tile_x_row;
616
617
#pragma unroll
@@ -621,7 +621,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
621
i = min(i, i_max);
622
}
623
const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;
624
- x_dm[i * (WARP_SIZE/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
624
+ x_dm[i * (WARP_SIZE_GGUF/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
625
}
626
}
627
@@ -634,13 +634,13 @@ static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(
634
635
#pragma unroll
636
for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
637
- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
638
- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_1) % WARP_SIZE];
637
+ u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
638
+ u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_1) % WARP_SIZE_GGUF];
639
}
640
641
return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
642
- (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dm[i * (WARP_SIZE/QI4_1) + i/QI4_1 + k/QI4_1],
643
- y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
642
+ (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], u, x_dm[i * (WARP_SIZE_GGUF/QI4_1) + i/QI4_1 + k/QI4_1],
643
+ y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
644
}
645
646
static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
@@ -664,8 +664,8 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
664
}
665
666
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
667
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
668
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];
667
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
668
+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI5_0) + mmq_y/QI5_0];
669
670
*x_ql = tile_x_ql;
671
*x_dm = (half2 *) tile_x_d;
@@ -697,7 +697,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
697
qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
698
qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
699
700
- x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
700
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+0] = qs0;
701
702
int qs1 = (ql >> 4) & 0x0F0F0F0F;
703
qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
@@ -706,10 +706,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
706
qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
707
qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
708
709
- x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
709
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+1] = qs1;
710
}
711
712
- const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;
712
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_0;
713
const int kbxd = k % blocks_per_tile_x_row;
714
float * x_dmf = (float *) x_dm;
715
@@ -722,7 +722,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
722
}
723
724
const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;
725
- x_dmf[i * (WARP_SIZE/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
725
+ x_dmf[i * (WARP_SIZE_GGUF/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
726
}
727
}
728
@@ -730,7 +730,7 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
730
const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
731
const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
732
const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
733
- const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;
733
+ const int index_bx = i * (WARP_SIZE_GGUF/QI5_0) + i/QI5_0 + k/QI5_0;
734
const float * x_dmf = (const float *) x_dm;
735
const float * y_df = (const float *) y_ds;
736
@@ -738,12 +738,12 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
738
739
#pragma unroll
740
for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {
741
- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
742
- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_0) % WARP_SIZE];
741
+ u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
742
+ u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_0) % WARP_SIZE_GGUF];
743
}
744
745
return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>
746
- (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
746
+ (&x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
747
}
748
749
static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
@@ -767,8 +767,8 @@ static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
767
}
768
769
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
770
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
771
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];
770
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
771
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI5_1) + mmq_y/QI5_1];
772
773
*x_ql = tile_x_ql;
774
*x_dm = tile_x_dm;
@@ -801,7 +801,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
801
qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
802
qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
803
804
- x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
804
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+0] = qs0;
805
806
int qs1 = (ql >> 4) & 0x0F0F0F0F;
807
qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
@@ -809,10 +809,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
809
qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
810
qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
811
812
- x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
812
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+1] = qs1;
813
}
814
815
- const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;
815
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_1;
816
const int kbxd = k % blocks_per_tile_x_row;
817
818
#pragma unroll
@@ -825,7 +825,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
825
826
const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;
827
828
- x_dm[i * (WARP_SIZE/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
828
+ x_dm[i * (WARP_SIZE_GGUF/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
829
}
830
}
831
@@ -833,18 +833,18 @@ static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(
833
const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
834
const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
835
const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
836
- const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;
836
+ const int index_bx = i * (WARP_SIZE_GGUF/QI5_1) + + i/QI5_1 + k/QI5_1;
837
838
int u[2*VDR_Q5_1_Q8_1_MMQ];
839
840
#pragma unroll
841
for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {
842
- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
843
- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_1) % WARP_SIZE];
842
+ u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
843
+ u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_1) % WARP_SIZE_GGUF];
844
}
845
846
return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
847
- (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
847
+ (&x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
848
}
849
850
static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
@@ -865,8 +865,8 @@ static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
865
}
866
867
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
868
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
869
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];
868
+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
869
+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI8_0) + mmq_y/QI8_0];
870
871
*x_ql = tile_x_qs;
872
*x_dm = (half2 *) tile_x_d;
@@ -889,10 +889,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
889
i = min(i, i_max);
890
}
891
const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;
892
- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
892
+ x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
893
}
894
895
- const int blocks_per_tile_x_row = WARP_SIZE / QI8_0;
895
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI8_0;
896
const int kbxd = k % blocks_per_tile_x_row;
897
898
#pragma unroll
@@ -903,7 +903,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
903
i = min(i, i_max);
904
}
905
const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;
906
- x_dmf[i * (WARP_SIZE/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
906
+ x_dmf[i * (WARP_SIZE_GGUF/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
907
}
908
}
909
@@ -914,8 +914,8 @@ static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(
914
const float * y_df = (const float *) y_ds;
915
916
return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>
917
- (&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[j * WARP_SIZE + k], x_dmf[i * (WARP_SIZE/QI8_0) + i/QI8_0 + k/QI8_0],
918
- y_df[j * (WARP_SIZE/QI8_1) + k/QI8_1]);
917
+ (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], &y_qs[j * WARP_SIZE_GGUF + k], x_dmf[i * (WARP_SIZE_GGUF/QI8_0) + i/QI8_0 + k/QI8_0],
918
+ y_df[j * (WARP_SIZE_GGUF/QI8_1) + k/QI8_1]);
919
}
920
921
static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
@@ -942,9 +942,9 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
942
}
943
944
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
945
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
946
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];
947
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
945
+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
946
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI2_K) + mmq_y/QI2_K];
947
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/4) + mmq_y/4];
948
949
*x_ql = tile_x_ql;
950
*x_dm = tile_x_dm;
@@ -967,10 +967,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
967
i = min(i, i_max);
968
}
969
const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;
970
- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
970
+ x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
971
}
972
973
- const int blocks_per_tile_x_row = WARP_SIZE / QI2_K;
973
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI2_K;
974
const int kbxd = k % blocks_per_tile_x_row;
975
976
#pragma unroll
@@ -981,18 +981,18 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
981
i = min(i, i_max);
982
}
983
const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;
984
- x_dm[i * (WARP_SIZE/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
984
+ x_dm[i * (WARP_SIZE_GGUF/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
985
}
986
987
#pragma unroll
988
for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
989
- int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
989
+ int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF/4);
990
991
if (need_check) {
992
i = min(i, i_max);
993
}
994
- const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI2_K/4);
995
- x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
994
+ const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/4)) / (QI2_K/4);
995
+ x_sc[i * (WARP_SIZE_GGUF/4) + i / 4 + k % (WARP_SIZE_GGUF/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
996
}
997
}
998
@@ -1005,7 +1005,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
1005
1006
int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];
1007
1008
- const int kqsx = i * (WARP_SIZE + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
1008
+ const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
1009
const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));
1010
1011
#pragma unroll
@@ -1013,10 +1013,10 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
1013
v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;
1014
}
1015
1016
- const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE/4) + i/4 + kbx*4]) + ky/4;
1016
+ const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/4) + i/4 + kbx*4]) + ky/4;
1017
1018
- const int index_y = j * WARP_SIZE + (QR2_K*k) % WARP_SIZE;
1019
- return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1018
+ const int index_y = j * WARP_SIZE_GGUF + (QR2_K*k) % WARP_SIZE_GGUF;
1019
+ return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE_GGUF/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1020
}
1021
1022
static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
@@ -1047,10 +1047,10 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
1047
}
1048
1049
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1050
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1051
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI3_K) + mmq_y/QI3_K];
1052
- __shared__ int tile_x_qh[mmq_y * (WARP_SIZE/2) + mmq_y/2];
1053
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
1050
+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
1051
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI3_K) + mmq_y/QI3_K];
1052
+ __shared__ int tile_x_qh[mmq_y * (WARP_SIZE_GGUF/2) + mmq_y/2];
1053
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/4) + mmq_y/4];
1054
1055
*x_ql = tile_x_ql;
1056
*x_dm = tile_x_dm;
@@ -1073,10 +1073,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1073
i = min(i, i_max);
1074
}
1075
const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;
1076
- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1076
+ x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1077
}
1078
1079
- const int blocks_per_tile_x_row = WARP_SIZE / QI3_K;
1079
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI3_K;
1080
const int kbxd = k % blocks_per_tile_x_row;
1081
float * x_dmf = (float *) x_dm;
1082
@@ -1087,27 +1087,27 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1087
i = min(i, i_max);
1088
}
1089
const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;
1090
- x_dmf[i * (WARP_SIZE/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1090
+ x_dmf[i * (WARP_SIZE_GGUF/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1091
}
1092
1093
#pragma unroll
1094
for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {
1095
- int i = i0 + i_offset * 2 + k / (WARP_SIZE/2);
1095
+ int i = i0 + i_offset * 2 + k / (WARP_SIZE_GGUF/2);
1096
if (need_check) {
1097
i = min(i, i_max);
1098
}
1099
- const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/2)) / (QI3_K/2);
1099
+ const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/2)) / (QI3_K/2);
1100
// invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1101
- x_qh[i * (WARP_SIZE/2) + i / 2 + k % (WARP_SIZE/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1101
+ x_qh[i * (WARP_SIZE_GGUF/2) + i / 2 + k % (WARP_SIZE_GGUF/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1102
}
1103
1104
#pragma unroll
1105
for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
1106
- int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
1106
+ int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF/4);
1107
if (need_check) {
1108
i = min(i, i_max);
1109
}
1110
- const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI3_K/4);
1110
+ const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/4)) / (QI3_K/4);
1111
1112
const int ksc = k % (QI3_K/4);
1113
@@ -1121,7 +1121,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1121
1122
const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
1123
1124
- x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = sc;
1124
+ x_sc[i * (WARP_SIZE_GGUF/4) + i / 4 + k % (WARP_SIZE_GGUF/4)] = sc;
1125
}
1126
}
1127
@@ -1134,24 +1134,24 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(
1134
const float * x_dmf = (const float *) x_dm;
1135
const float * y_df = (const float *) y_ds;
1136
1137
- const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;
1137
+ const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE_GGUF/4) + i/4 + kbx*4)) + ky/4;
1138
1139
int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];
1140
1141
#pragma unroll
1142
for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {
1143
- const int kqsx = i * (WARP_SIZE + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1143
+ const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1144
const int shift = 2 * ((ky % 32) / 8);
1145
const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;
1146
1147
- const int vh = x_qh[i * (WARP_SIZE/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1147
+ const int vh = x_qh[i * (WARP_SIZE_GGUF/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1148
const int vlh = (vh << 2) & 0x04040404;
1149
1150
v[l] = __vsubss4(vll, vlh);
1151
}
1152
1153
- const int index_y = j * WARP_SIZE + (k*QR3_K) % WARP_SIZE;
1154
- return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1153
+ const int index_y = j * WARP_SIZE_GGUF + (k*QR3_K) % WARP_SIZE_GGUF;
1154
+ return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE_GGUF/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1155
}
1156
1157
static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
@@ -1200,9 +1200,9 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
1200
}
1201
1202
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1203
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1204
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];
1205
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1203
+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
1204
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI4_K) + mmq_y/QI4_K];
1205
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1206
1207
*x_ql = tile_x_ql;
1208
*x_dm = tile_x_dm;
@@ -1225,10 +1225,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1225
i = min(i, i_max);
1226
}
1227
const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;
1228
- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1228
+ x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1229
}
1230
1231
- const int blocks_per_tile_x_row = WARP_SIZE / QI4_K; // == 1 if QK_K == 256
1231
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_K; // == 1 if QK_K == 256
1232
const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1233
1234
#pragma unroll
@@ -1238,27 +1238,27 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1238
i = min(i, i_max);
1239
}
1240
const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;
1241
- x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1241
+ x_dm[i * (WARP_SIZE_GGUF/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1242
}
1243
1244
#pragma unroll
1245
for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1246
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1246
+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1247
1248
if (need_check) {
1249
i = min(i, i_max);
1250
}
1251
1252
- const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);
1252
+ const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / (QI4_K/8);
1253
1254
const int * scales = (const int *) bxi->scales;
1255
1256
- const int ksc = k % (WARP_SIZE/8);
1256
+ const int ksc = k % (WARP_SIZE_GGUF/8);
1257
// scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1258
int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1259
scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1260
1261
- x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1261
+ x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + ksc] = scales8;
1262
}
1263
}
1264
@@ -1267,11 +1267,11 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(
1267
const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1268
(void)x_qh;
1269
1270
- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1270
+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1271
1272
- const int index_y = j * WARP_SIZE + (QR4_K*k) % WARP_SIZE;
1273
- return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[index_y], sc, sc+8,
1274
- x_dm[i * (WARP_SIZE/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1272
+ const int index_y = j * WARP_SIZE_GGUF + (QR4_K*k) % WARP_SIZE_GGUF;
1273
+ return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE_GGUF + 1) + k], &y_qs[index_y], sc, sc+8,
1274
+ x_dm[i * (WARP_SIZE_GGUF/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1275
}
1276
1277
static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
@@ -1321,9 +1321,9 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
1321
}
1322
1323
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1324
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1325
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];
1326
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1324
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
1325
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI5_K) + mmq_y/QI5_K];
1326
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1327
1328
*x_ql = tile_x_ql;
1329
*x_dm = tile_x_dm;
@@ -1360,11 +1360,11 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1360
const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;
1361
const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);
1362
1363
- x_ql[i * (2*WARP_SIZE + 1) + kq0] = ql0 | qh0;
1364
- x_ql[i * (2*WARP_SIZE + 1) + kq1] = ql1 | qh1;
1363
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq0] = ql0 | qh0;
1364
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq1] = ql1 | qh1;
1365
}
1366
1367
- const int blocks_per_tile_x_row = WARP_SIZE / QI5_K; // == 1 if QK_K == 256
1367
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_K; // == 1 if QK_K == 256
1368
const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1369
1370
#pragma unroll
@@ -1376,40 +1376,40 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1376
}
1377
1378
const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;
1379
- x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1379
+ x_dm[i * (WARP_SIZE_GGUF/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1380
}
1381
1382
#pragma unroll
1383
for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1384
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1384
+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1385
1386
if (need_check) {
1387
i = min(i, i_max);
1388
}
1389
1390
- const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);
1390
+ const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / (QI5_K/8);
1391
1392
const int * scales = (const int *) bxi->scales;
1393
1394
- const int ksc = k % (WARP_SIZE/8);
1394
+ const int ksc = k % (WARP_SIZE_GGUF/8);
1395
1396
// scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1397
int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1398
scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1399
1400
- x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1400
+ x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + ksc] = scales8;
1401
}
1402
}
1403
1404
static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(
1405
const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1406
const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1407
- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1407
+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1408
1409
- const int index_x = i * (QR5_K*WARP_SIZE + 1) + QR5_K*k;
1410
- const int index_y = j * WARP_SIZE + (QR5_K*k) % WARP_SIZE;
1409
+ const int index_x = i * (QR5_K*WARP_SIZE_GGUF + 1) + QR5_K*k;
1410
+ const int index_y = j * WARP_SIZE_GGUF + (QR5_K*k) % WARP_SIZE_GGUF;
1411
return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,
1412
- x_dm[i * (WARP_SIZE/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1412
+ x_dm[i * (WARP_SIZE_GGUF/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1413
}
1414
1415
static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
@@ -1439,9 +1439,9 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
1439
}
1440
1441
template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1442
- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1443
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];
1444
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1442
+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
1443
+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI6_K) + mmq_y/QI6_K];
1444
+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1445
1446
*x_ql = tile_x_ql;
1447
*x_dm = tile_x_dm;
@@ -1478,11 +1478,11 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1478
const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;
1479
const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);
1480
1481
- x_ql[i * (2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1482
- x_ql[i * (2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1481
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1482
+ x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1483
}
1484
1485
- const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256
1485
+ const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI6_K; // == 1 if QK_K == 256
1486
const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1487
float * x_dmf = (float *) x_dm;
1488
@@ -1496,20 +1496,20 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1496
1497
const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;
1498
1499
- x_dmf[i * (WARP_SIZE/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1499
+ x_dmf[i * (WARP_SIZE_GGUF/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1500
}
1501
1502
#pragma unroll
1503
for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1504
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1504
+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1505
1506
if (need_check) {
1507
i = min(i, i_max);
1508
}
1509
1510
- const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / 4;
1510
+ const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / 4;
1511
1512
- x_sc[i * (WARP_SIZE/8) + i / 8 + k % (WARP_SIZE/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1512
+ x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + k % (WARP_SIZE_GGUF/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1513
}
1514
}
1515
@@ -1519,11 +1519,11 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(
1519
const float * x_dmf = (const float *) x_dm;
1520
const float * y_df = (const float *) y_ds;
1521
1522
- const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/8]);
1522
+ const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/8]);
1523
1524
- const int index_x = i * (QR6_K*WARP_SIZE + 1) + QR6_K*k;
1525
- const int index_y = j * WARP_SIZE + (QR6_K*k) % WARP_SIZE;
1526
- return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1524
+ const int index_x = i * (QR6_K*WARP_SIZE_GGUF + 1) + QR6_K*k;
1525
+ const int index_y = j * WARP_SIZE_GGUF + (QR6_K*k) % WARP_SIZE_GGUF;
1526
+ return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE_GGUF/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1527
}
1528
1529
static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
@@ -1582,7 +1582,7 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
1582
1583
static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1584
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1585
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1585
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1586
const block_iq2_s * bq2 = (const block_iq2_s *) vbq;
1587
1588
const int ib32 = iqs;
@@ -1619,7 +1619,7 @@ static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1619
1620
static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1621
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1622
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1622
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1623
const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;
1624
1625
const int ib32 = iqs;
@@ -1646,7 +1646,7 @@ static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1646
1647
static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1648
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1649
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1649
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1650
const block_iq3_s * bq2 = (const block_iq3_s *) vbq;
1651
1652
const int ib32 = iqs;
@@ -1671,7 +1671,7 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1671
1672
static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1673
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1674
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1674
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1675
const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1676
1677
const int qs_packed = get_int_b2(bq1->qs, iqs);
@@ -1703,7 +1703,7 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1703
1704
static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
1705
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1706
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1706
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1707
1708
const block_iq1_m * bq1 = (const block_iq1_m *) vbq;
1709
@@ -1763,7 +1763,7 @@ static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4
1763
1764
static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1765
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1766
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1766
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1767
1768
const block_iq4_nl * bq = (const block_iq4_nl *) vbq;
1769
@@ -1788,7 +1788,7 @@ static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1788
1789
static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
1790
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1791
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1791
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1792
const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;
1793
const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1794
csrc/torch_bindings.cpp
+2
@@ -258,6 +258,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
258
"awq_marlin_repack(Tensor b_q_weight, SymInt size_k, "
259
"SymInt size_n, int num_bits) -> Tensor");
260
// conditionally compiled so impl registrations are in source file
261
+#endif
262
263
// Dequantization for GGML.
264
ops.def("ggml_dequantize(Tensor W, int type, SymInt m, SymInt n) -> Tensor");
@@ -274,6 +275,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
275
"ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor");
276
ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
277
278
+#ifndef USE_ROCM
279
// fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
280
ops.def(
281
"fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
vllm/_custom_ops.py
+28
-25
@@ -344,31 +344,6 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
344
is_zp_float: bool = False) -> torch.Tensor:
345
return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
346
347
- @register_fake("_C::ggml_dequantize")
348
- def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int,
349
- m: torch.SymInt,
350
- n: torch.SymInt) -> torch.Tensor:
351
- return torch.empty((m, n), dtype=torch.float16, device=W.device)
352
-
353
- @register_fake("_C::ggml_mul_mat_vec_a8")
354
- def _ggml_mul_mat_vec_a8_fake(
355
- W: torch.Tensor,
356
- X: torch.Tensor,
357
- quant_type: int,
358
- row: torch.SymInt,
359
- ) -> torch.Tensor:
360
- return torch.empty((1, row), dtype=torch.float16, device=W.device)
361
-
362
- @register_fake("_C::ggml_mul_mat_a8")
363
- def _ggml_mul_mat_a8_fake(
364
- W: torch.Tensor,
365
- X: torch.Tensor,
366
- quant_type: int,
367
- row: torch.SymInt,
368
- ) -> torch.Tensor:
369
- batch = X.size(0)
370
- return torch.empty((batch, row), dtype=torch.float16, device=W.device)
371
-
347
@register_fake("_C::marlin_qqq_gemm")
348
def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
349
s_tok: torch.Tensor, s_ch: torch.Tensor,
@@ -468,6 +443,34 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
443
memory_format=torch.contiguous_format)
444
445
446
+if hasattr(torch.ops._C, "ggml_dequantize"):
447
+
448
+ @register_fake("_C::ggml_dequantize")
449
+ def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int,
450
+ m: torch.SymInt,
451
+ n: torch.SymInt) -> torch.Tensor:
452
+ return torch.empty((m, n), dtype=torch.float16, device=W.device)
453
+
454
+ @register_fake("_C::ggml_mul_mat_vec_a8")
455
+ def _ggml_mul_mat_vec_a8_fake(
456
+ W: torch.Tensor,
457
+ X: torch.Tensor,
458
+ quant_type: int,
459
+ row: torch.SymInt,
460
+ ) -> torch.Tensor:
461
+ return torch.empty((1, row), dtype=torch.float16, device=W.device)
462
+
463
+ @register_fake("_C::ggml_mul_mat_a8")
464
+ def _ggml_mul_mat_a8_fake(
465
+ W: torch.Tensor,
466
+ X: torch.Tensor,
467
+ quant_type: int,
468
+ row: torch.SymInt,
469
+ ) -> torch.Tensor:
470
+ batch = X.size(0)
471
+ return torch.empty((batch, row), dtype=torch.float16, device=W.device)
472
+
473
+
474
# cutlass
475
def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
476
return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
vllm/config.py
+1
-1
@@ -387,7 +387,7 @@ class ModelConfig:
387
supported_quantization = QUANTIZATION_METHODS
388
rocm_supported_quantization = [
389
"awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
390
- "fbgemm_fp8"
390
+ "fbgemm_fp8", "gguf"
391
]
392
optimized_quantization_methods = [
393
"fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",