@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c25fe45a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[AMD] Add support for GGUF quantization on ROCm (#10254)

kliuae committed Nov 23, 2024 at 13:14 UTC 7c25fe45a6ef4fb5be148217cc7110e88e186446
11 files changed +234 -211
.buildkite/run-amd-test.sh
-1
@@ -85,7 +85,6 @@ if [[ $commands == *" kernels "* ]]; then
85 --ignore=kernels/test_encoder_decoder_attn.py \
86 --ignore=kernels/test_flash_attn.py \
87 --ignore=kernels/test_flashinfer.py \
88 - --ignore=kernels/test_gguf.py \
88 --ignore=kernels/test_int8_quant.py \
89 --ignore=kernels/test_machete_gemm.py \
90 --ignore=kernels/test_mamba_ssm.py \
CMakeLists.txt
+1 -1
@@ -196,6 +196,7 @@ set(VLLM_EXT_SRC
196 "csrc/quantization/gptq/q_gemm.cu"
197 "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
198 "csrc/quantization/fp8/common.cu"
199 + "csrc/quantization/gguf/gguf_kernel.cu"
200 "csrc/cuda_utils_kernels.cu"
201 "csrc/prepare_inputs/advance_step.cu"
202 "csrc/torch_bindings.cpp")
@@ -237,7 +238,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
238 "csrc/mamba/causal_conv1d/causal_conv1d.cu"
239 "csrc/quantization/aqlm/gemm_kernels.cu"
240 "csrc/quantization/awq/gemm_kernels.cu"
240 - "csrc/quantization/gguf/gguf_kernel.cu"
241 "csrc/custom_all_reduce.cu"
242 "csrc/permute_cols.cu"
243 "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu")
csrc/ops.h
+2
@@ -128,6 +128,7 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
128 int64_t thx, int64_t thy);
129
130 torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
131 +#endif
132
133 torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
134 int64_t n);
@@ -138,6 +139,7 @@ torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
139 torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
140 int64_t row);
141
142 +#ifndef USE_ROCM
143 bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
144
145 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
csrc/quantization/gguf/ggml-common.h
+16 -1
@@ -1,7 +1,7 @@
1 // copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
2 #define QK_K 256
3 #define K_QUANTS_PER_ITERATION 2
4 -#define WARP_SIZE 32
4 +#define WARP_SIZE_GGUF 32
5 #define K_SCALE_SIZE 12
6 #define CUDA_DEQUANTIZE_BLOCK_SIZE 256
7 #define CUDA_QUANTIZE_BLOCK_SIZE 256
@@ -1112,4 +1112,19 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
1112 #endif
1113 return c;
1114 }
1115 +
1116 +static __device__ __forceinline__ uint32_t __vcmpeq4(const uint32_t a, const uint32_t b) {
1117 + uint32_t neq = a^b;
1118 + return !(neq & 0xff000000) * 0xff000000 |
1119 + !(neq & 0x00ff0000) * 0x00ff0000 |
1120 + !(neq & 0x0000ff00) * 0x0000ff00 |
1121 + !(neq & 0x000000ff) * 0x000000ff;
1122 +}
1123 +
1124 +static __device__ __forceinline__ uint32_t __vsub4(const uint32_t a, const uint32_t b) {
1125 + return (static_cast<uint8_t>(((a & 0xff000000) >> 24) - ((b & 0xff000000) >> 24)) << 24) +
1126 + (static_cast<uint8_t>(((a & 0x00ff0000) >> 16) - ((b & 0x00ff0000) >> 16)) << 16) +
1127 + (static_cast<uint8_t>(((a & 0x0000ff00) >> 8) - ((b & 0x0000ff00) >> 8)) << 8) +
1128 + (static_cast<uint8_t>(((a & 0x000000ff) >> 0) - ((b & 0x000000ff) >> 0)) << 0);
1129 +}
1130 #endif // defined(USE_ROCM)
csrc/quantization/gguf/gguf_kernel.cu
+4 -2
@@ -4,6 +4,8 @@
4 #include <torch/all.h>
5 #include <c10/cuda/CUDAGuard.h>
6
7 +#include "cuda_compat.h"
8 +
9 #include "ggml-common.h"
10 #include "vecdotq.cuh"
11 #include "dequantize.cuh"
@@ -32,8 +34,8 @@ static __global__ void quantize_q8_1(const half* __restrict__ x,
34
35 #pragma unroll
36 for (int mask = 16; mask > 0; mask >>= 1) {
35 - amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, mask, 32));
36 - sum += __shfl_xor_sync(0xffffffff, sum, mask, 32);
37 + amax = fmaxf(amax, VLLM_SHFL_XOR_SYNC_WIDTH(amax, mask, 32));
38 + sum += VLLM_SHFL_XOR_SYNC_WIDTH(sum, mask, 32);
39 }
40
41 const float d = amax / 127;
csrc/quantization/gguf/mmq.cuh
+35 -35
@@ -10,7 +10,7 @@ static __device__ __forceinline__ void mul_mat_q(
10
11 const int blocks_per_row_x = ncols_x / qk;
12 const int blocks_per_col_y = nrows_y / QK8_1;
13 - const int blocks_per_warp = WARP_SIZE / qi;
13 + const int blocks_per_warp = WARP_SIZE_GGUF / qi;
14
15 const int & ncols_dst = ncols_y;
16
@@ -27,10 +27,10 @@ static __device__ __forceinline__ void mul_mat_q(
27
28 allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
29
30 - __shared__ int tile_y_qs[mmq_x * WARP_SIZE];
31 - __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];
30 + __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
31 + __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF/QI8_1];
32
33 - float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};
33 + float sum[mmq_y/WARP_SIZE_GGUF][mmq_x/nwarps] = {{0.0f}};
34
35 for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
36
@@ -39,26 +39,26 @@ static __device__ __forceinline__ void mul_mat_q(
39
40 #pragma unroll
41 for (int ir = 0; ir < qr; ++ir) {
42 - const int kqs = ir*WARP_SIZE + threadIdx.x;
42 + const int kqs = ir*WARP_SIZE_GGUF + threadIdx.x;
43 const int kbxd = kqs / QI8_1;
44
45 #pragma unroll
46 for (int i = 0; i < mmq_x; i += nwarps) {
47 const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses
48 const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];
49 - const int index_y = (threadIdx.y + i) * WARP_SIZE + kqs % WARP_SIZE;
49 + const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
50 tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
51 }
52
53 #pragma unroll
54 for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
55 - const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE/QI8_1)) % mmq_x;
56 - const int kby = threadIdx.x % (WARP_SIZE/QI8_1);
55 + const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF/QI8_1)) % mmq_x;
56 + const int kby = threadIdx.x % (WARP_SIZE_GGUF/QI8_1);
57 const int col_y_eff = min(col_y_0 + ids, ncols_y-1);
58
59 // if the sum is not needed it's faster to transform the scale to f32 ahead of time
60 - const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE/QI8_1) + kby].ds;
61 - half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE/QI8_1) + kby];
60 + const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE_GGUF/QI8_1) + kby].ds;
61 + half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF/QI8_1) + kby];
62 if (need_sum) {
63 *dsi_dst = *dsi_src;
64 } else {
@@ -70,12 +70,12 @@ static __device__ __forceinline__ void mul_mat_q(
70 __syncthreads();
71
72 // #pragma unroll // unrolling this loop causes too much register pressure
73 - for (int k = ir*WARP_SIZE/qr; k < (ir+1)*WARP_SIZE/qr; k += vdr) {
73 + for (int k = ir*WARP_SIZE_GGUF/qr; k < (ir+1)*WARP_SIZE_GGUF/qr; k += vdr) {
74 #pragma unroll
75 for (int j = 0; j < mmq_x; j += nwarps) {
76 #pragma unroll
77 - for (int i = 0; i < mmq_y; i += WARP_SIZE) {
78 - sum[i/WARP_SIZE][j/nwarps] += vec_dot(
77 + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
78 + sum[i/WARP_SIZE_GGUF][j/nwarps] += vec_dot(
79 tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,
80 threadIdx.x + i, threadIdx.y + j, k);
81 }
@@ -93,12 +93,12 @@ static __device__ __forceinline__ void mul_mat_q(
93 }
94
95 #pragma unroll
96 - for (int i = 0; i < mmq_y; i += WARP_SIZE) {
96 + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
97 const int row_dst = row_dst_0 + threadIdx.x + i;
98 if (row_dst >= nrows_dst) {
99 continue;
100 }
101 - dst[col_dst*nrows_dst + row_dst] = __float2half(sum[i/WARP_SIZE][j/nwarps]);
101 + dst[col_dst*nrows_dst + row_dst] = __float2half(sum[i/WARP_SIZE_GGUF][j/nwarps]);
102 }
103 }
104 }
@@ -115,7 +115,7 @@ static __device__ __forceinline__ void mul_mat_q(
115
116 template <bool need_check> static __global__ void
117 #if defined(USE_ROCM)
118 -__launch_bounds__(WARP_SIZE*NWARPS_Q4_0, 2)
118 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_0, 2)
119 #endif
120 mul_mat_q4_0(
121 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -140,7 +140,7 @@ static void ggml_mul_mat_q4_0_q8_1_cuda(
140 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
141 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
142 const dim3 block_nums(block_num_x, block_num_y, 1);
143 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
143 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
144
145 if (nrows_x % mmq_y == 0) {
146 const bool need_check = false;
@@ -165,7 +165,7 @@ static void ggml_mul_mat_q4_0_q8_1_cuda(
165
166 template <bool need_check> static __global__ void
167 #if defined(USE_ROCM)
168 -__launch_bounds__(WARP_SIZE*NWARPS_Q4_1, 2)
168 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_1, 2)
169 #endif
170 mul_mat_q4_1(
171 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -190,7 +190,7 @@ static void ggml_mul_mat_q4_1_q8_1_cuda(
190 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
191 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
192 const dim3 block_nums(block_num_x, block_num_y, 1);
193 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
193 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
194
195 if (nrows_x % mmq_y == 0) {
196 const bool need_check = false;
@@ -215,7 +215,7 @@ static void ggml_mul_mat_q4_1_q8_1_cuda(
215
216 template <bool need_check> static __global__ void
217 #if defined(USE_ROCM)
218 -__launch_bounds__(WARP_SIZE*NWARPS_Q5_0, 2)
218 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_0, 2)
219 #endif
220 mul_mat_q5_0(
221 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -240,7 +240,7 @@ static void ggml_mul_mat_q5_0_q8_1_cuda(
240 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
241 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
242 const dim3 block_nums(block_num_x, block_num_y, 1);
243 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
243 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
244
245 if (nrows_x % mmq_y == 0) {
246 const bool need_check = false;
@@ -265,7 +265,7 @@ static void ggml_mul_mat_q5_0_q8_1_cuda(
265
266 template <bool need_check> static __global__ void
267 #if defined(USE_ROCM)
268 -__launch_bounds__(WARP_SIZE*NWARPS_Q5_1, 2)
268 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_1, 2)
269 #endif
270 mul_mat_q5_1(
271 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -289,7 +289,7 @@ static void ggml_mul_mat_q5_1_q8_1_cuda(
289 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
290 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
291 const dim3 block_nums(block_num_x, block_num_y, 1);
292 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
292 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
293
294 if (nrows_x % mmq_y == 0) {
295 const bool need_check = false;
@@ -314,7 +314,7 @@ static void ggml_mul_mat_q5_1_q8_1_cuda(
314
315 template <bool need_check> static __global__ void
316 #if defined(USE_ROCM)
317 -__launch_bounds__(WARP_SIZE*NWARPS_Q8_0, 2)
317 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q8_0, 2)
318 #endif
319 mul_mat_q8_0(
320 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -338,7 +338,7 @@ static void ggml_mul_mat_q8_0_q8_1_cuda(
338 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
339 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
340 const dim3 block_nums(block_num_x, block_num_y, 1);
341 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
341 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
342
343 if (nrows_x % mmq_y == 0) {
344 const bool need_check = false;
@@ -363,7 +363,7 @@ static void ggml_mul_mat_q8_0_q8_1_cuda(
363
364 template <bool need_check> static __global__ void
365 #if defined(USE_ROCM)
366 -__launch_bounds__(WARP_SIZE*NWARPS_Q2_K, 2)
366 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q2_K, 2)
367 #endif
368 mul_mat_q2_K(
369 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -387,7 +387,7 @@ static void ggml_mul_mat_q2_K_q8_1_cuda(
387 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
388 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
389 const dim3 block_nums(block_num_x, block_num_y, 1);
390 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
390 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
391
392 if (nrows_x % mmq_y == 0) {
393 const bool need_check = false;
@@ -412,7 +412,7 @@ static void ggml_mul_mat_q2_K_q8_1_cuda(
412
413 template <bool need_check> static __global__ void
414 #if defined(USE_ROCM)
415 -__launch_bounds__(WARP_SIZE*NWARPS_Q3_K, 2)
415 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q3_K, 2)
416 #endif
417 mul_mat_q3_K(
418 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -438,7 +438,7 @@ static void ggml_mul_mat_q3_K_q8_1_cuda(
438 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
439 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
440 const dim3 block_nums(block_num_x, block_num_y, 1);
441 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
441 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
442
443 if (nrows_x % mmq_y == 0) {
444 const bool need_check = false;
@@ -463,7 +463,7 @@ static void ggml_mul_mat_q3_K_q8_1_cuda(
463
464 template <bool need_check> static __global__ void
465 #if defined(USE_ROCM)
466 -__launch_bounds__(WARP_SIZE*NWARPS_Q4_K, 2)
466 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_K, 2)
467 #endif
468 mul_mat_q4_K(
469 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -487,7 +487,7 @@ static void ggml_mul_mat_q4_K_q8_1_cuda(
487 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
488 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
489 const dim3 block_nums(block_num_x, block_num_y, 1);
490 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
490 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
491
492 if (nrows_x % mmq_y == 0) {
493 const bool need_check = false;
@@ -512,7 +512,7 @@ static void ggml_mul_mat_q4_K_q8_1_cuda(
512
513 template <bool need_check> static __global__ void
514 #if defined(USE_ROCM)
515 -__launch_bounds__(WARP_SIZE*NWARPS_Q5_K, 2)
515 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_K, 2)
516 #endif
517 mul_mat_q5_K(
518 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -537,7 +537,7 @@ static void ggml_mul_mat_q5_K_q8_1_cuda(
537 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
538 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
539 const dim3 block_nums(block_num_x, block_num_y, 1);
540 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
540 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
541
542 if (nrows_x % mmq_y == 0) {
543 const bool need_check = false;
@@ -562,7 +562,7 @@ static void ggml_mul_mat_q5_K_q8_1_cuda(
562
563 template <bool need_check> static __global__ void
564 #if defined(USE_ROCM)
565 -__launch_bounds__(WARP_SIZE*NWARPS_Q6_K, 2)
565 +__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q6_K, 2)
566 #endif
567 mul_mat_q6_K(
568 const void * __restrict__ vx, const void * __restrict__ vy, half * __restrict__ dst,
@@ -586,7 +586,7 @@ static void ggml_mul_mat_q6_K_q8_1_cuda(
586 const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
587 const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
588 const dim3 block_nums(block_num_x, block_num_y, 1);
589 - const dim3 block_dims(WARP_SIZE, nwarps, 1);
589 + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
590
591 if (nrows_x % mmq_y == 0) {
592 const bool need_check = false;
csrc/quantization/gguf/mmvq.cuh
+2 -2
@@ -28,8 +28,8 @@ static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void *
28
29 // sum up partial sums and write back result
30 #pragma unroll
31 - for (int mask = 16; mask > 0; mask >>= 1) {
32 - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32);
31 + for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1) {
32 + tmp += VLLM_SHFL_XOR_SYNC(tmp, mask);
33 }
34
35 if (threadIdx.x == 0) {
csrc/quantization/gguf/vecdotq.cuh
+143 -143
@@ -43,7 +43,7 @@ static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t *
43
44 template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
45 const int * v, const int * u, const float & d4, const half2 & ds8) {
46 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
46 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
47 int sumi = 0;
48
49 #pragma unroll
@@ -68,7 +68,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_imp
68
69 template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
70 const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
71 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
71 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
72 int sumi = 0;
73
74 #pragma unroll
@@ -95,7 +95,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_imp
95
96 template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
97 const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
98 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
98 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
99 int sumi = 0;
100
101 #pragma unroll
@@ -128,7 +128,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_imp
128
129 template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
130 const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
131 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
131 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
132 int sumi = 0;
133
134 #pragma unroll
@@ -162,7 +162,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_imp
162
163 template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(
164 const int * v, const int * u, const float & d8_0, const float & d8_1) {
165 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
165 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
166 int sumi = 0;
167
168 #pragma unroll
@@ -176,7 +176,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_imp
176
177 template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
178 const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
179 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
179 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
180
181 int sumi = 0;
182
@@ -202,7 +202,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_imp
202 static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
203 const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
204 const half2 & dm2, const float * __restrict__ d8) {
205 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
205 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
206 float sumf_d = 0.0f;
207 float sumf_m = 0.0f;
208
@@ -230,7 +230,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
230 static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
231 const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
232 const half2 & dm2, const float & d8) {
233 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
233 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
234 int sumi_d = 0;
235 int sumi_m = 0;
236
@@ -267,7 +267,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
267 static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
268 const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
269 const int & scale_offset, const float & d3, const float * __restrict__ d8) {
270 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
270 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
271
272 float sumf = 0.0f;
273
@@ -301,7 +301,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
301 static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
302 const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
303 const float & d3, const float & d8) {
304 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
304 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
305 int sumi = 0;
306
307 #pragma unroll
@@ -326,7 +326,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
326 static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
327 const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
328 const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
329 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
329 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
330
331 float sumf_d = 0.0f;
332 float sumf_m = 0.0f;
@@ -351,7 +351,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
351 static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
352 const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
353 const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
354 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
354 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
355 float sumf_d = 0.0f;
356 float sumf_m = 0.0f;
357
@@ -382,7 +382,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
382 static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
383 const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
384 const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
385 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
385 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
386
387 float sumf_d = 0.0f;
388 float sumf_m = 0.0f;
@@ -413,7 +413,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
413 static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
414 const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
415 const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
416 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
416 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
417 float sumf_d = 0.0f;
418 float sumf_m = 0.0f;
419
@@ -445,7 +445,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
445 static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
446 const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
447 const float & d, const float * __restrict__ d8) {
448 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
448 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
449 float sumf = 0.0f;
450
451 #pragma unroll
@@ -465,7 +465,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
465 static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
466 const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
467 const float & d6, const float * __restrict__ d8) {
468 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
468 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
469 float sumf_d = 0.0f;
470
471 #pragma unroll
@@ -507,8 +507,8 @@ static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
507 }
508
509 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
510 - __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
511 - __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];
510 + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
511 + __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI4_0) + mmq_y/QI4_0];
512 *x_ql = tile_x_qs;
513 *x_dm = (half2 *) tile_x_d;
514 }
@@ -529,11 +529,11 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
529 i = min(i, i_max);
530 }
531 const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;
532 - x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
533 - // x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbx] = bxi->d;
532 + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
533 + // x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbx] = bxi->d;
534 }
535
536 - const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;
536 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_0;
537 const int kbxd = k % blocks_per_tile_x_row;
538
539 #pragma unroll
@@ -543,7 +543,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
543 i = min(i, i_max);
544 }
545 const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;
546 - x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
546 + x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
547 }
548 }
549
@@ -559,13 +559,13 @@ static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(
559
560 #pragma unroll
561 for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
562 - u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
563 - u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_0) % WARP_SIZE];
562 + u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
563 + u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_0) % WARP_SIZE_GGUF];
564 }
565
566 return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
567 - (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dmf[i * (WARP_SIZE/QI4_0) + i/QI4_0 + k/QI4_0],
568 - y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
567 + (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], u, x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i/QI4_0 + k/QI4_0],
568 + y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
569 }
570
571 static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
@@ -587,8 +587,8 @@ static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
587 }
588
589 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
590 - __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];
591 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];
590 + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + + mmq_y];
591 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI4_1) + mmq_y/QI4_1];
592 *x_ql = tile_x_qs;
593 *x_dm = tile_x_dm;
594 }
@@ -608,10 +608,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
608 i = min(i, i_max);
609 }
610 const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;
611 - x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
611 + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
612 }
613
614 - const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;
614 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_1;
615 const int kbxd = k % blocks_per_tile_x_row;
616
617 #pragma unroll
@@ -621,7 +621,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
621 i = min(i, i_max);
622 }
623 const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;
624 - x_dm[i * (WARP_SIZE/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
624 + x_dm[i * (WARP_SIZE_GGUF/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
625 }
626 }
627
@@ -634,13 +634,13 @@ static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(
634
635 #pragma unroll
636 for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
637 - u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
638 - u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_1) % WARP_SIZE];
637 + u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
638 + u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_1) % WARP_SIZE_GGUF];
639 }
640
641 return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
642 - (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dm[i * (WARP_SIZE/QI4_1) + i/QI4_1 + k/QI4_1],
643 - y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
642 + (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], u, x_dm[i * (WARP_SIZE_GGUF/QI4_1) + i/QI4_1 + k/QI4_1],
643 + y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
644 }
645
646 static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
@@ -664,8 +664,8 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
664 }
665
666 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
667 - __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
668 - __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];
667 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
668 + __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI5_0) + mmq_y/QI5_0];
669
670 *x_ql = tile_x_ql;
671 *x_dm = (half2 *) tile_x_d;
@@ -697,7 +697,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
697 qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
698 qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
699
700 - x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
700 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+0] = qs0;
701
702 int qs1 = (ql >> 4) & 0x0F0F0F0F;
703 qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
@@ -706,10 +706,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
706 qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
707 qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
708
709 - x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
709 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+1] = qs1;
710 }
711
712 - const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;
712 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_0;
713 const int kbxd = k % blocks_per_tile_x_row;
714 float * x_dmf = (float *) x_dm;
715
@@ -722,7 +722,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
722 }
723
724 const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;
725 - x_dmf[i * (WARP_SIZE/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
725 + x_dmf[i * (WARP_SIZE_GGUF/QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
726 }
727 }
728
@@ -730,7 +730,7 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
730 const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
731 const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
732 const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
733 - const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;
733 + const int index_bx = i * (WARP_SIZE_GGUF/QI5_0) + i/QI5_0 + k/QI5_0;
734 const float * x_dmf = (const float *) x_dm;
735 const float * y_df = (const float *) y_ds;
736
@@ -738,12 +738,12 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
738
739 #pragma unroll
740 for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {
741 - u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
742 - u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_0) % WARP_SIZE];
741 + u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
742 + u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_0) % WARP_SIZE_GGUF];
743 }
744
745 return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>
746 - (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
746 + (&x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
747 }
748
749 static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
@@ -767,8 +767,8 @@ static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
767 }
768
769 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
770 - __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
771 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];
770 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
771 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI5_1) + mmq_y/QI5_1];
772
773 *x_ql = tile_x_ql;
774 *x_dm = tile_x_dm;
@@ -801,7 +801,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
801 qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
802 qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
803
804 - x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;
804 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+0] = qs0;
805
806 int qs1 = (ql >> 4) & 0x0F0F0F0F;
807 qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
@@ -809,10 +809,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
809 qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
810 qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
811
812 - x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;
812 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2*k+1] = qs1;
813 }
814
815 - const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;
815 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_1;
816 const int kbxd = k % blocks_per_tile_x_row;
817
818 #pragma unroll
@@ -825,7 +825,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
825
826 const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;
827
828 - x_dm[i * (WARP_SIZE/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
828 + x_dm[i * (WARP_SIZE_GGUF/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
829 }
830 }
831
@@ -833,18 +833,18 @@ static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(
833 const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
834 const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
835 const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));
836 - const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;
836 + const int index_bx = i * (WARP_SIZE_GGUF/QI5_1) + + i/QI5_1 + k/QI5_1;
837
838 int u[2*VDR_Q5_1_Q8_1_MMQ];
839
840 #pragma unroll
841 for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {
842 - u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];
843 - u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_1) % WARP_SIZE];
842 + u[2*l+0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
843 + u[2*l+1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_1) % WARP_SIZE_GGUF];
844 }
845
846 return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
847 - (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);
847 + (&x_ql[i * (2*WARP_SIZE_GGUF + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE_GGUF/QI8_1) + (2*k/QI8_1) % (WARP_SIZE_GGUF/QI8_1)]);
848 }
849
850 static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
@@ -865,8 +865,8 @@ static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
865 }
866
867 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
868 - __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];
869 - __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];
868 + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
869 + __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF/QI8_0) + mmq_y/QI8_0];
870
871 *x_ql = tile_x_qs;
872 *x_dm = (half2 *) tile_x_d;
@@ -889,10 +889,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
889 i = min(i, i_max);
890 }
891 const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;
892 - x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
892 + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
893 }
894
895 - const int blocks_per_tile_x_row = WARP_SIZE / QI8_0;
895 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI8_0;
896 const int kbxd = k % blocks_per_tile_x_row;
897
898 #pragma unroll
@@ -903,7 +903,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
903 i = min(i, i_max);
904 }
905 const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;
906 - x_dmf[i * (WARP_SIZE/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
906 + x_dmf[i * (WARP_SIZE_GGUF/QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
907 }
908 }
909
@@ -914,8 +914,8 @@ static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(
914 const float * y_df = (const float *) y_ds;
915
916 return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>
917 - (&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[j * WARP_SIZE + k], x_dmf[i * (WARP_SIZE/QI8_0) + i/QI8_0 + k/QI8_0],
918 - y_df[j * (WARP_SIZE/QI8_1) + k/QI8_1]);
917 + (&x_ql[i * (WARP_SIZE_GGUF + 1) + k], &y_qs[j * WARP_SIZE_GGUF + k], x_dmf[i * (WARP_SIZE_GGUF/QI8_0) + i/QI8_0 + k/QI8_0],
918 + y_df[j * (WARP_SIZE_GGUF/QI8_1) + k/QI8_1]);
919 }
920
921 static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
@@ -942,9 +942,9 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
942 }
943
944 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
945 - __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
946 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];
947 - __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
945 + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
946 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI2_K) + mmq_y/QI2_K];
947 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/4) + mmq_y/4];
948
949 *x_ql = tile_x_ql;
950 *x_dm = tile_x_dm;
@@ -967,10 +967,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
967 i = min(i, i_max);
968 }
969 const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;
970 - x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
970 + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
971 }
972
973 - const int blocks_per_tile_x_row = WARP_SIZE / QI2_K;
973 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI2_K;
974 const int kbxd = k % blocks_per_tile_x_row;
975
976 #pragma unroll
@@ -981,18 +981,18 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
981 i = min(i, i_max);
982 }
983 const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;
984 - x_dm[i * (WARP_SIZE/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
984 + x_dm[i * (WARP_SIZE_GGUF/QI2_K) + i / QI2_K + kbxd] = bxi->dm;
985 }
986
987 #pragma unroll
988 for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
989 - int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
989 + int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF/4);
990
991 if (need_check) {
992 i = min(i, i_max);
993 }
994 - const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI2_K/4);
995 - x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
994 + const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/4)) / (QI2_K/4);
995 + x_sc[i * (WARP_SIZE_GGUF/4) + i / 4 + k % (WARP_SIZE_GGUF/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));
996 }
997 }
998
@@ -1005,7 +1005,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
1005
1006 int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];
1007
1008 - const int kqsx = i * (WARP_SIZE + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
1008 + const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);
1009 const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));
1010
1011 #pragma unroll
@@ -1013,10 +1013,10 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
1013 v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;
1014 }
1015
1016 - const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE/4) + i/4 + kbx*4]) + ky/4;
1016 + const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/4) + i/4 + kbx*4]) + ky/4;
1017
1018 - const int index_y = j * WARP_SIZE + (QR2_K*k) % WARP_SIZE;
1019 - return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1018 + const int index_y = j * WARP_SIZE_GGUF + (QR2_K*k) % WARP_SIZE_GGUF;
1019 + return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE_GGUF/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);
1020 }
1021
1022 static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
@@ -1047,10 +1047,10 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
1047 }
1048
1049 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1050 - __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1051 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI3_K) + mmq_y/QI3_K];
1052 - __shared__ int tile_x_qh[mmq_y * (WARP_SIZE/2) + mmq_y/2];
1053 - __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];
1050 + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
1051 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI3_K) + mmq_y/QI3_K];
1052 + __shared__ int tile_x_qh[mmq_y * (WARP_SIZE_GGUF/2) + mmq_y/2];
1053 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/4) + mmq_y/4];
1054
1055 *x_ql = tile_x_ql;
1056 *x_dm = tile_x_dm;
@@ -1073,10 +1073,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1073 i = min(i, i_max);
1074 }
1075 const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;
1076 - x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1076 + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
1077 }
1078
1079 - const int blocks_per_tile_x_row = WARP_SIZE / QI3_K;
1079 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI3_K;
1080 const int kbxd = k % blocks_per_tile_x_row;
1081 float * x_dmf = (float *) x_dm;
1082
@@ -1087,27 +1087,27 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1087 i = min(i, i_max);
1088 }
1089 const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;
1090 - x_dmf[i * (WARP_SIZE/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1090 + x_dmf[i * (WARP_SIZE_GGUF/QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
1091 }
1092
1093 #pragma unroll
1094 for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {
1095 - int i = i0 + i_offset * 2 + k / (WARP_SIZE/2);
1095 + int i = i0 + i_offset * 2 + k / (WARP_SIZE_GGUF/2);
1096 if (need_check) {
1097 i = min(i, i_max);
1098 }
1099 - const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/2)) / (QI3_K/2);
1099 + const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/2)) / (QI3_K/2);
1100 // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1101 - x_qh[i * (WARP_SIZE/2) + i / 2 + k % (WARP_SIZE/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1101 + x_qh[i * (WARP_SIZE_GGUF/2) + i / 2 + k % (WARP_SIZE_GGUF/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));
1102 }
1103
1104 #pragma unroll
1105 for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
1106 - int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);
1106 + int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF/4);
1107 if (need_check) {
1108 i = min(i, i_max);
1109 }
1110 - const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI3_K/4);
1110 + const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/4)) / (QI3_K/4);
1111
1112 const int ksc = k % (QI3_K/4);
1113
@@ -1121,7 +1121,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1121
1122 const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
1123
1124 - x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = sc;
1124 + x_sc[i * (WARP_SIZE_GGUF/4) + i / 4 + k % (WARP_SIZE_GGUF/4)] = sc;
1125 }
1126 }
1127
@@ -1134,24 +1134,24 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(
1134 const float * x_dmf = (const float *) x_dm;
1135 const float * y_df = (const float *) y_ds;
1136
1137 - const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;
1137 + const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE_GGUF/4) + i/4 + kbx*4)) + ky/4;
1138
1139 int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];
1140
1141 #pragma unroll
1142 for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {
1143 - const int kqsx = i * (WARP_SIZE + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1143 + const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);
1144 const int shift = 2 * ((ky % 32) / 8);
1145 const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;
1146
1147 - const int vh = x_qh[i * (WARP_SIZE/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1147 + const int vh = x_qh[i * (WARP_SIZE_GGUF/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);
1148 const int vlh = (vh << 2) & 0x04040404;
1149
1150 v[l] = __vsubss4(vll, vlh);
1151 }
1152
1153 - const int index_y = j * WARP_SIZE + (k*QR3_K) % WARP_SIZE;
1154 - return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1153 + const int index_y = j * WARP_SIZE_GGUF + (k*QR3_K) % WARP_SIZE_GGUF;
1154 + return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE_GGUF/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);
1155 }
1156
1157 static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
@@ -1200,9 +1200,9 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
1200 }
1201
1202 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1203 - __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];
1204 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];
1205 - __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1203 + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
1204 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI4_K) + mmq_y/QI4_K];
1205 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1206
1207 *x_ql = tile_x_ql;
1208 *x_dm = tile_x_dm;
@@ -1225,10 +1225,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1225 i = min(i, i_max);
1226 }
1227 const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;
1228 - x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1228 + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
1229 }
1230
1231 - const int blocks_per_tile_x_row = WARP_SIZE / QI4_K; // == 1 if QK_K == 256
1231 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_K; // == 1 if QK_K == 256
1232 const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1233
1234 #pragma unroll
@@ -1238,27 +1238,27 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1238 i = min(i, i_max);
1239 }
1240 const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;
1241 - x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1241 + x_dm[i * (WARP_SIZE_GGUF/QI4_K) + i / QI4_K + kbxd] = bxi->dm;
1242 }
1243
1244 #pragma unroll
1245 for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1246 - int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1246 + int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1247
1248 if (need_check) {
1249 i = min(i, i_max);
1250 }
1251
1252 - const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);
1252 + const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / (QI4_K/8);
1253
1254 const int * scales = (const int *) bxi->scales;
1255
1256 - const int ksc = k % (WARP_SIZE/8);
1256 + const int ksc = k % (WARP_SIZE_GGUF/8);
1257 // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1258 int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1259 scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1260
1261 - x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1261 + x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + ksc] = scales8;
1262 }
1263 }
1264
@@ -1267,11 +1267,11 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(
1267 const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1268 (void)x_qh;
1269
1270 - const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1270 + const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/16]) + 2*((k % 16) / 8);
1271
1272 - const int index_y = j * WARP_SIZE + (QR4_K*k) % WARP_SIZE;
1273 - return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[index_y], sc, sc+8,
1274 - x_dm[i * (WARP_SIZE/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1272 + const int index_y = j * WARP_SIZE_GGUF + (QR4_K*k) % WARP_SIZE_GGUF;
1273 + return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE_GGUF + 1) + k], &y_qs[index_y], sc, sc+8,
1274 + x_dm[i * (WARP_SIZE_GGUF/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);
1275 }
1276
1277 static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
@@ -1321,9 +1321,9 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
1321 }
1322
1323 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1324 - __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1325 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];
1326 - __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1324 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
1325 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI5_K) + mmq_y/QI5_K];
1326 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1327
1328 *x_ql = tile_x_ql;
1329 *x_dm = tile_x_dm;
@@ -1360,11 +1360,11 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1360 const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;
1361 const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);
1362
1363 - x_ql[i * (2*WARP_SIZE + 1) + kq0] = ql0 | qh0;
1364 - x_ql[i * (2*WARP_SIZE + 1) + kq1] = ql1 | qh1;
1363 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq0] = ql0 | qh0;
1364 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq1] = ql1 | qh1;
1365 }
1366
1367 - const int blocks_per_tile_x_row = WARP_SIZE / QI5_K; // == 1 if QK_K == 256
1367 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_K; // == 1 if QK_K == 256
1368 const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1369
1370 #pragma unroll
@@ -1376,40 +1376,40 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1376 }
1377
1378 const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;
1379 - x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1379 + x_dm[i * (WARP_SIZE_GGUF/QI5_K) + i / QI5_K + kbxd] = bxi->dm;
1380 }
1381
1382 #pragma unroll
1383 for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1384 - int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1384 + int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1385
1386 if (need_check) {
1387 i = min(i, i_max);
1388 }
1389
1390 - const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);
1390 + const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / (QI5_K/8);
1391
1392 const int * scales = (const int *) bxi->scales;
1393
1394 - const int ksc = k % (WARP_SIZE/8);
1394 + const int ksc = k % (WARP_SIZE_GGUF/8);
1395
1396 // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
1397 int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits
1398 scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
1399
1400 - x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;
1400 + x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + ksc] = scales8;
1401 }
1402 }
1403
1404 static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(
1405 const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,
1406 const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {
1407 - const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1407 + const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);
1408
1409 - const int index_x = i * (QR5_K*WARP_SIZE + 1) + QR5_K*k;
1410 - const int index_y = j * WARP_SIZE + (QR5_K*k) % WARP_SIZE;
1409 + const int index_x = i * (QR5_K*WARP_SIZE_GGUF + 1) + QR5_K*k;
1410 + const int index_y = j * WARP_SIZE_GGUF + (QR5_K*k) % WARP_SIZE_GGUF;
1411 return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,
1412 - x_dm[i * (WARP_SIZE/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1412 + x_dm[i * (WARP_SIZE_GGUF/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);
1413 }
1414
1415 static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
@@ -1439,9 +1439,9 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
1439 }
1440
1441 template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {
1442 - __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];
1443 - __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];
1444 - __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];
1442 + __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE_GGUF) + mmq_y];
1443 + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF/QI6_K) + mmq_y/QI6_K];
1444 + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF/8) + mmq_y/8];
1445
1446 *x_ql = tile_x_ql;
1447 *x_dm = tile_x_dm;
@@ -1478,11 +1478,11 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1478 const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;
1479 const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);
1480
1481 - x_ql[i * (2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1482 - x_ql[i * (2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1481 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
1482 + x_ql[i * (2*WARP_SIZE_GGUF + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
1483 }
1484
1485 - const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256
1485 + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI6_K; // == 1 if QK_K == 256
1486 const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
1487 float * x_dmf = (float *) x_dm;
1488
@@ -1496,20 +1496,20 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlin
1496
1497 const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;
1498
1499 - x_dmf[i * (WARP_SIZE/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1499 + x_dmf[i * (WARP_SIZE_GGUF/QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
1500 }
1501
1502 #pragma unroll
1503 for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
1504 - int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;
1504 + int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF/8)) % mmq_y;
1505
1506 if (need_check) {
1507 i = min(i, i_max);
1508 }
1509
1510 - const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / 4;
1510 + const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE_GGUF/8)) / 4;
1511
1512 - x_sc[i * (WARP_SIZE/8) + i / 8 + k % (WARP_SIZE/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1512 + x_sc[i * (WARP_SIZE_GGUF/8) + i / 8 + k % (WARP_SIZE_GGUF/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));
1513 }
1514 }
1515
@@ -1519,11 +1519,11 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(
1519 const float * x_dmf = (const float *) x_dm;
1520 const float * y_df = (const float *) y_ds;
1521
1522 - const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/8]);
1522 + const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE_GGUF/8) + i/8 + k/8]);
1523
1524 - const int index_x = i * (QR6_K*WARP_SIZE + 1) + QR6_K*k;
1525 - const int index_y = j * WARP_SIZE + (QR6_K*k) % WARP_SIZE;
1526 - return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1524 + const int index_x = i * (QR6_K*WARP_SIZE_GGUF + 1) + QR6_K*k;
1525 + const int index_y = j * WARP_SIZE_GGUF + (QR6_K*k) % WARP_SIZE_GGUF;
1526 + return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE_GGUF/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);
1527 }
1528
1529 static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
@@ -1582,7 +1582,7 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
1582
1583 static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1584 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1585 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1585 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1586 const block_iq2_s * bq2 = (const block_iq2_s *) vbq;
1587
1588 const int ib32 = iqs;
@@ -1619,7 +1619,7 @@ static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
1619
1620 static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1621 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1622 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1622 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1623 const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;
1624
1625 const int ib32 = iqs;
@@ -1646,7 +1646,7 @@ static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
1646
1647 static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1648 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1649 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1649 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1650 const block_iq3_s * bq2 = (const block_iq3_s *) vbq;
1651
1652 const int ib32 = iqs;
@@ -1671,7 +1671,7 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1671
1672 static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1673 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1674 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1674 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1675 const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1676
1677 const int qs_packed = get_int_b2(bq1->qs, iqs);
@@ -1703,7 +1703,7 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1703
1704 static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
1705 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1706 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1706 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1707
1708 const block_iq1_m * bq1 = (const block_iq1_m *) vbq;
1709
@@ -1763,7 +1763,7 @@ static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4
1763
1764 static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1765 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1766 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1766 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1767
1768 const block_iq4_nl * bq = (const block_iq4_nl *) vbq;
1769
@@ -1788,7 +1788,7 @@ static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1788
1789 static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
1790 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1791 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1791 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM
1792 const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;
1793 const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1794
csrc/torch_bindings.cpp
+2
@@ -258,6 +258,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
258 "awq_marlin_repack(Tensor b_q_weight, SymInt size_k, "
259 "SymInt size_n, int num_bits) -> Tensor");
260 // conditionally compiled so impl registrations are in source file
261 +#endif
262
263 // Dequantization for GGML.
264 ops.def("ggml_dequantize(Tensor W, int type, SymInt m, SymInt n) -> Tensor");
@@ -274,6 +275,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
275 "ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor");
276 ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
277
278 +#ifndef USE_ROCM
279 // fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
280 ops.def(
281 "fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
vllm/_custom_ops.py
+28 -25
@@ -344,31 +344,6 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
344 is_zp_float: bool = False) -> torch.Tensor:
345 return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
346
347 - @register_fake("_C::ggml_dequantize")
348 - def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int,
349 - m: torch.SymInt,
350 - n: torch.SymInt) -> torch.Tensor:
351 - return torch.empty((m, n), dtype=torch.float16, device=W.device)
352 -
353 - @register_fake("_C::ggml_mul_mat_vec_a8")
354 - def _ggml_mul_mat_vec_a8_fake(
355 - W: torch.Tensor,
356 - X: torch.Tensor,
357 - quant_type: int,
358 - row: torch.SymInt,
359 - ) -> torch.Tensor:
360 - return torch.empty((1, row), dtype=torch.float16, device=W.device)
361 -
362 - @register_fake("_C::ggml_mul_mat_a8")
363 - def _ggml_mul_mat_a8_fake(
364 - W: torch.Tensor,
365 - X: torch.Tensor,
366 - quant_type: int,
367 - row: torch.SymInt,
368 - ) -> torch.Tensor:
369 - batch = X.size(0)
370 - return torch.empty((batch, row), dtype=torch.float16, device=W.device)
371 -
347 @register_fake("_C::marlin_qqq_gemm")
348 def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
349 s_tok: torch.Tensor, s_ch: torch.Tensor,
@@ -468,6 +443,34 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
443 memory_format=torch.contiguous_format)
444
445
446 +if hasattr(torch.ops._C, "ggml_dequantize"):
447 +
448 + @register_fake("_C::ggml_dequantize")
449 + def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int,
450 + m: torch.SymInt,
451 + n: torch.SymInt) -> torch.Tensor:
452 + return torch.empty((m, n), dtype=torch.float16, device=W.device)
453 +
454 + @register_fake("_C::ggml_mul_mat_vec_a8")
455 + def _ggml_mul_mat_vec_a8_fake(
456 + W: torch.Tensor,
457 + X: torch.Tensor,
458 + quant_type: int,
459 + row: torch.SymInt,
460 + ) -> torch.Tensor:
461 + return torch.empty((1, row), dtype=torch.float16, device=W.device)
462 +
463 + @register_fake("_C::ggml_mul_mat_a8")
464 + def _ggml_mul_mat_a8_fake(
465 + W: torch.Tensor,
466 + X: torch.Tensor,
467 + quant_type: int,
468 + row: torch.SymInt,
469 + ) -> torch.Tensor:
470 + batch = X.size(0)
471 + return torch.empty((batch, row), dtype=torch.float16, device=W.device)
472 +
473 +
474 # cutlass
475 def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
476 return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
vllm/config.py
+1 -1
@@ -387,7 +387,7 @@ class ModelConfig:
387 supported_quantization = QUANTIZATION_METHODS
388 rocm_supported_quantization = [
389 "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
390 - "fbgemm_fp8"
390 + "fbgemm_fp8", "gguf"
391 ]
392 optimized_quantization_methods = [
393 "fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",