A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
Support W4A8 quantization for vllm (#5218)
HandH1998 committed
Jul 31, 2024 at 21:55 UTC
6512937de1d7b4738938e0bb3004be86b6883729
15 files changed
+1963
-84
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m HandH1998/QQQ-Llama-3-8b-g128 -b 32 -l 1000 -f 5 -t 1
2
+model_name: "HandH1998/QQQ-Llama-3-8b-g128"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.409
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.406
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -7,3 +7,4 @@ Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7
Minitron-4B-Base.yaml
8
Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
9
Qwen2-1.5B-Instruct-FP8W8.yaml
10
+Meta-Llama-3-8B-QQQ.yaml
CMakeLists.txt
+1
@@ -170,6 +170,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
170
"csrc/quantization/awq/gemm_kernels.cu"
171
"csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
172
"csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
173
+ "csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
174
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
175
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
176
"csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
csrc/ops.h
+7
@@ -115,6 +115,13 @@ void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
115
torch::Tensor const& b_scales,
116
c10::optional<torch::Tensor> const& bias);
117
118
+torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
119
+ torch::Tensor const& b_q_weight,
120
+ torch::Tensor const& s_tok,
121
+ torch::Tensor const& s_ch,
122
+ torch::Tensor const& s_group,
123
+ torch::Tensor& workspace, int64_t size_m,
124
+ int64_t size_n, int64_t size_k);
125
#endif
126
127
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/marlin/dense/common/base.h
new
+32
@@ -0,0 +1,32 @@
1
+/*
2
+ * Modified by HandH1998
3
+ * Modified by Neural Magic
4
+ * Copyright (C) Marlin.2024 Elias Frantar
5
+ *
6
+ * Licensed under the Apache License, Version 2.0 (the "License");
7
+ * you may not use this file except in compliance with the License.
8
+ * You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing, software
13
+ * distributed under the License is distributed on an "AS IS" BASIS,
14
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ * See the License for the specific language governing permissions and
16
+ * limitations under the License.
17
+ */
18
+
19
+#pragma once
20
+
21
+constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
22
+
23
+// Instances of `Vec` are used to organize groups of >>registers<<, as needed
24
+// for instance as inputs to tensor core operations. Consequently, all
25
+// corresponding index accesses must be compile-time constants, which is why we
26
+// extensively use `#pragma unroll` throughout the kernel code to guarantee
27
+// this.
28
+template <typename T, int n>
29
+struct Vec {
30
+ T elems[n];
31
+ __device__ T& operator[](int i) { return elems[i]; }
32
+};
csrc/quantization/marlin/dense/common/mem.h
new
+89
@@ -0,0 +1,89 @@
1
+/*
2
+ * Modified by HandH1998
3
+ * Modified by Neural Magic
4
+ * Copyright (C) Marlin.2024 Elias Frantar
5
+ *
6
+ * Licensed under the Apache License, Version 2.0 (the "License");
7
+ * you may not use this file except in compliance with the License.
8
+ * You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing, software
13
+ * distributed under the License is distributed on an "AS IS" BASIS,
14
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ * See the License for the specific language governing permissions and
16
+ * limitations under the License.
17
+ */
18
+
19
+#pragma once
20
+
21
+// Predicated asynchronous global->shared copy; used for inputs A where we apply
22
+// predication to handle batchsizes that are not multiples of 16.
23
+__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
24
+ bool pred = true) {
25
+ const int BYTES = 16;
26
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
27
+ asm volatile(
28
+ "{\n"
29
+ " .reg .pred p;\n"
30
+ " setp.ne.b32 p, %0, 0;\n"
31
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
32
+ "}\n" ::"r"((int)pred),
33
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
34
+}
35
+
36
+// Asynchronous global->shared copy
37
+__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
38
+ const int BYTES = 16;
39
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
40
+ asm volatile(
41
+ "{\n"
42
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
43
+ "}\n" ::"r"(smem),
44
+ "l"(glob_ptr), "n"(BYTES));
45
+}
46
+
47
+// Async copy fence.
48
+__device__ inline void cp_async_fence() {
49
+ asm volatile("cp.async.commit_group;\n" ::);
50
+}
51
+
52
+// Wait until at most `n` async copy stages are still pending.
53
+template <int n>
54
+__device__ inline void cp_async_wait() {
55
+ asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
56
+}
57
+
58
+// Wait until barrier reaches `count`, then lock for current threadblock.
59
+__device__ inline void barrier_acquire(int* lock, int count) {
60
+ if (threadIdx.x == 0) {
61
+ int state = -1;
62
+ do
63
+ // Guarantee that subsequent writes by this threadblock will be visible
64
+ // globally.
65
+ asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
66
+ : "=r"(state)
67
+ : "l"(lock));
68
+ while (state != count);
69
+ }
70
+ __syncthreads();
71
+}
72
+
73
+// Release barrier and increment visitation count.
74
+__device__ inline void barrier_release(int* lock, bool reset = false) {
75
+ __syncthreads();
76
+ if (threadIdx.x == 0) {
77
+ if (reset) {
78
+ lock[0] = 0;
79
+ return;
80
+ }
81
+ int val = 1;
82
+ // Make sure that all writes since acquiring this barrier are visible
83
+ // globally, while releasing the barrier.
84
+ asm volatile("fence.acq_rel.gpu;\n");
85
+ asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
86
+ :
87
+ : "l"(lock), "r"(val));
88
+ }
89
+}
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+6
-84
@@ -25,6 +25,12 @@
25
26
#include <iostream>
27
28
+#include "common/base.h"
29
+
30
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
31
+ #include "common/mem.h"
32
+#endif
33
+
34
template <typename T>
35
inline std::string str(T x) {
36
return std::to_string(x);
@@ -32,23 +38,9 @@ inline std::string str(T x) {
38
39
namespace marlin_dense {
40
35
-constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
36
-
41
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
42
39
-// Instances of `Vec` are used to organize groups of >>registers<<, as needed
40
-// for instance as inputs to tensor core operations. Consequently, all
41
-// corresponding index accesses must be compile-time constants, which is why we
42
-// extensively use `#pragma unroll` throughout the kernel code to guarantee
43
-// this.
44
-template <typename T, int n>
45
-struct Vec {
46
- T elems[n];
47
- __device__ T& operator[](int i) { return elems[i]; }
48
-};
49
-
43
using I4 = Vec<int, 4>;
51
-
44
// Matrix fragments for tensor core instructions; their precise layout is
45
// documented here:
46
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
@@ -57,43 +49,6 @@ using FragB = Vec<half2, 2>;
49
using FragC = Vec<float, 4>;
50
using FragS = Vec<half2, 1>; // quantization scales
51
60
-// Predicated asynchronous global->shared copy; used for inputs A where we apply
61
-// predication to handle batchsizes that are not multiples of 16.
62
-__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
63
- bool pred = true) {
64
- const int BYTES = 16;
65
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
66
- asm volatile(
67
- "{\n"
68
- " .reg .pred p;\n"
69
- " setp.ne.b32 p, %0, 0;\n"
70
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
71
- "}\n" ::"r"((int)pred),
72
- "r"(smem), "l"(glob_ptr), "n"(BYTES));
73
-}
74
-
75
-// Asynchronous global->shared copy
76
-__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
77
- const int BYTES = 16;
78
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
79
- asm volatile(
80
- "{\n"
81
- " cp.async.cg.shared.global [%0], [%1], %2;\n"
82
- "}\n" ::"r"(smem),
83
- "l"(glob_ptr), "n"(BYTES));
84
-}
85
-
86
-// Async copy fence.
87
-__device__ inline void cp_async_fence() {
88
- asm volatile("cp.async.commit_group;\n" ::);
89
-}
90
-
91
-// Wait until at most `n` async copy stages are still pending.
92
-template <int n>
93
-__device__ inline void cp_async_wait() {
94
- asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
95
-}
96
-
52
// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
53
// output/accumulation.
54
__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
@@ -164,39 +119,6 @@ __device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
119
frag_b[1] = __hmul2(frag_b[1], s);
120
}
121
167
-// Wait until barrier reaches `count`, then lock for current threadblock.
168
-__device__ inline void barrier_acquire(int* lock, int count) {
169
- if (threadIdx.x == 0) {
170
- int state = -1;
171
- do
172
- // Guarantee that subsequent writes by this threadblock will be visible
173
- // globally.
174
- asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
175
- : "=r"(state)
176
- : "l"(lock));
177
- while (state != count);
178
- }
179
- __syncthreads();
180
-}
181
-
182
-// Release barrier and increment visitation count.
183
-__device__ inline void barrier_release(int* lock, bool reset = false) {
184
- __syncthreads();
185
- if (threadIdx.x == 0) {
186
- if (reset) {
187
- lock[0] = 0;
188
- return;
189
- }
190
- int val = 1;
191
- // Make sure that all writes since acquiring this barrier are visible
192
- // globally, while releasing the barrier.
193
- asm volatile("fence.acq_rel.gpu;\n");
194
- asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
195
- :
196
- : "l"(lock), "r"(val));
197
- }
198
-}
199
-
122
template <const int threads, // number of threads in a threadblock
123
const int thread_m_blocks, // number of 16x16 blocks in the m
124
// dimension (batchsize) of the
csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu
new
+1243
@@ -0,0 +1,1243 @@
1
+/*
2
+ * Adapted from
3
+ * https://github.com/IST-DASLab/marlin/blob/master/marlin/marlin_cuda_kernel.cu
4
+ * https://github.com/IST-DASLab/marlin/blob/master/marlin/marlin_cuda.cpp
5
+ * Modified by HandH1998
6
+ * Copyright (C) 2024 HandH1998
7
+ * Copyright (C) Marlin.2024 Elias Frantar
8
+ *
9
+ * Licensed under the Apache License, Version 2.0 (the "License");
10
+ * you may not use this file except in compliance with the License.
11
+ * You may obtain a copy of the License at
12
+ *
13
+ * http://www.apache.org/licenses/LICENSE-2.0
14
+ *
15
+ * Unless required by applicable law or agreed to in writing, software
16
+ * distributed under the License is distributed on an "AS IS" BASIS,
17
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ * See the License for the specific language governing permissions and
19
+ * limitations under the License.
20
+ */
21
+
22
+#include <torch/all.h>
23
+
24
+#include <ATen/cuda/CUDAContext.h>
25
+#include <c10/cuda/CUDAGuard.h>
26
+#include <cuda.h>
27
+#include <cuda_fp16.h>
28
+#include <cuda_runtime.h>
29
+
30
+#include <iostream>
31
+
32
+#include "../dense/common/base.h"
33
+
34
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
35
+ #include "../dense/common/mem.h"
36
+#endif
37
+
38
+template <typename T>
39
+inline std::string str(T x) {
40
+ return std::to_string(x);
41
+}
42
+
43
+namespace {
44
+
45
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
46
+
47
+using I4 = Vec<int, 4>;
48
+// Matrix fragments for tensor core instructions; their precise layout is
49
+// documented here:
50
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-integer-type
51
+using FragA = Vec<uint32_t, 2>;
52
+using FragB = Vec<uint32_t, 1>;
53
+using FragC = Vec<int, 4>;
54
+using FragS_GROUP = Vec<half2, 1>; // weight per-group quantization scales
55
+using FragS_CHANNEL =
56
+ Vec<float, 2>; // weight per-channel quantization scales or activaton
57
+ // per-token quantization scales
58
+
59
+// NOTE(HandH1998): cp.async.cg only support BYTES = 16, however,
60
+// cp.async.ca can support BYTES = 4, 8, 16;
61
+// as s_tok's shape is equal to prob_m, we need set s_tok to float type,
62
+// and cp_size = 1 float, i.e., 4 BYTES
63
+// Asynchronous global->shared copy for activation quantizaton scales s_tok
64
+__device__ inline void cp_async1(void* smem_ptr, const void* glob_ptr) {
65
+ const int BYTES = 4;
66
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
67
+ asm volatile(
68
+ "{\n"
69
+ " cp.async.ca.shared.global [%0], [%1], %2;\n"
70
+ "}\n" ::"r"(smem),
71
+ "l"(glob_ptr), "n"(BYTES));
72
+}
73
+
74
+// m16n8k16 tensor core mma instruction with int8 inputs and int32
75
+// output/accumulation.
76
+__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
77
+ FragC& frag_c) {
78
+ const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
79
+ const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
80
+ int* c = reinterpret_cast<int*>(&frag_c);
81
+ asm volatile(
82
+ "mma.sync.aligned.m16n8k16.row.col.satfinite.s32.s8.s8.s32 "
83
+ "{%0,%1,%2,%3}, {%4,%5}, {%6}, {%7,%8,%9,%10};\n"
84
+ : "=r"(c[0]), "=r"(c[1]), "=r"(c[2]), "=r"(c[3])
85
+ : "r"(a[0]), "r"(a[1]), "r"(b[0]), "r"(c[0]), "r"(c[1]), "r"(c[2]),
86
+ "r"(c[3]));
87
+}
88
+
89
+// Instruction for loading a full 16x16 matrix fragment of operand A from shared
90
+// memory, directly in int8 tensor core layout.
91
+__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
92
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
93
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
94
+ asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n"
95
+ : "=r"(a[0]), "=r"(a[1])
96
+ : "r"(smem));
97
+}
98
+
99
+inline __device__ half2 float2_to_half2(float2 f) {
100
+ uint32_t res;
101
+ // NOTE(HandH1998): h0,h1 should be uint16_t, not half
102
+ uint16_t h0, h1;
103
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(h0) : "f"(f.x));
104
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(h1) : "f"(f.y));
105
+ asm volatile("mov.b32 %0, {%1, %2};\n" : "=r"(res) : "h"(h0), "h"(h1));
106
+ return reinterpret_cast<half2&>(res);
107
+}
108
+
109
+inline __device__ float int32_to_float(int h) {
110
+ float res;
111
+ asm volatile("cvt.rn.f32.s32 %0, %1;\n" : "=f"(res) : "r"(h));
112
+ return res;
113
+}
114
+
115
+// Lookup-table based 3-input logical operation; explicitly used for
116
+// dequantization as the compiler does not seem to automatically recognize it in
117
+// all cases.
118
+template <int lut>
119
+__device__ inline int lop3(int a, int b, int c) {
120
+ int res;
121
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
122
+ : "=r"(res)
123
+ : "r"(a), "r"(b), "r"(c), "n"(lut));
124
+ return res;
125
+}
126
+
127
+// Efficiently dequantize an int32 value into a full B-fragment of 4 int8 values
128
+// for weight per channel dequant.
129
+__device__ inline FragB dequant_per_channel(int q) {
130
+ static constexpr int MASK = 0xf0f0f0f0;
131
+ FragB frag_b;
132
+ frag_b[0] = (q & MASK);
133
+ return frag_b;
134
+}
135
+
136
+// Efficiently dequantize an int32 value into a full B-fragment of 4 int8 values
137
+// for weight per group dequant.
138
+__device__ inline FragB dequant_per_group(int q, FragS_GROUP& frag_s, int i) {
139
+ static constexpr uint32_t LO = 0x000f000f;
140
+ static constexpr uint32_t HI = 0x00f000f0;
141
+ static constexpr uint32_t EX = 0x64006400;
142
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
143
+ uint32_t t0 = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
144
+ uint32_t t1 = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
145
+ // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
146
+ // directly into `SUB` and `ADD`.
147
+ static constexpr uint32_t SUB = 0x64086408;
148
+ static constexpr uint32_t MUL = 0x2c002c00;
149
+ static constexpr uint32_t ADD = 0xd480d480;
150
+ *reinterpret_cast<half2*>(&t0) = __hsub2(
151
+ *reinterpret_cast<half2*>(&t0), *reinterpret_cast<const half2*>(&SUB));
152
+ *reinterpret_cast<half2*>(&t1) = __hfma2(
153
+ *reinterpret_cast<half2*>(&t1), *reinterpret_cast<const half2*>(&MUL),
154
+ *reinterpret_cast<const half2*>(&ADD));
155
+
156
+ uint16_t s = reinterpret_cast<uint16_t*>(&frag_s)[i];
157
+ uint32_t double_s;
158
+ // pack 2xfp16 to half2
159
+ asm volatile("mov.b32 %0, {%1, %2};\n" : "=r"(double_s) : "h"(s), "h"(s));
160
+ // dequant and convert 4 half to 4 uint8 (be placed at the low 8 bits of 4
161
+ // half, respectively)
162
+ static constexpr uint32_t MAGIC_NUM = 0x64806480;
163
+ *reinterpret_cast<half2*>(&t0) = __hfma2(
164
+ *reinterpret_cast<half2*>(&t0), *reinterpret_cast<half2*>(&double_s),
165
+ *reinterpret_cast<const half2*>(&MAGIC_NUM));
166
+ *reinterpret_cast<half2*>(&t1) = __hfma2(
167
+ *reinterpret_cast<half2*>(&t1), *reinterpret_cast<half2*>(&double_s),
168
+ *reinterpret_cast<const half2*>(&MAGIC_NUM));
169
+ // take out the 4 uint8 from 4 half, then convert them to 4 int8 and pack 4
170
+ // int8 into 1 uint32
171
+ FragB frag_b;
172
+ uint32_t uint8s;
173
+ static constexpr uint32_t MASK_0246 = 0x6420;
174
+ static constexpr uint32_t UINT8s_TO_INT8s_MASK = 0x80808080;
175
+ asm volatile("prmt.b32 %0,%1,%2,%3;\n"
176
+ : "=r"(uint8s)
177
+ : "r"(t0), "r"(t1), "n"(MASK_0246));
178
+ frag_b[0] = (uint8s ^ UINT8s_TO_INT8s_MASK);
179
+ return frag_b;
180
+}
181
+
182
+template <const int threads, // number of threads in a threadblock
183
+ const int thread_m_blocks, // number of 16x16 blocks in the m
184
+ // dimension (batchsize) of the
185
+ // threadblock
186
+ const int thread_n_blocks, // same for n dimension (output)
187
+ const int thread_k_blocks, // same for k dimension (reduction)
188
+ const int stages, // number of stages for the async global->shared
189
+ // fetch pipeline
190
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
191
+ // with a separate quantization scale
192
+ >
193
+__global__ void Marlin(
194
+ const int4* __restrict__ A, // int8 input matrix of shape mxk
195
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
196
+ int4* __restrict__ C, // int32 global_reduce buffer of shape
197
+ // (max_par*16*4)xn, as int8 tensor core's output is
198
+ // int32 dtype
199
+ int4* __restrict__ D, // fp16 output buffer of shape mxn
200
+ const float* __restrict__ s_tok, // fp32 activation per-token quantization
201
+ // scales of shape mx1
202
+ const int4* __restrict__ s_ch, // fp32 weight per-channel quantization
203
+ // scales of shape 1xn
204
+ const int4* __restrict__ s_group, // fp16 weight per-group quantization
205
+ // scales of shape (k/groupsize)xn, when
206
+ // group_blocks=-1, it should be nullptr
207
+ int prob_m, // batch dimension m
208
+ int prob_n, // output dimension n
209
+ int prob_k, // reduction dimension k
210
+ int* locks // extra global storage for barrier synchronization
211
+) {
212
+ // Each threadblock processes one "stripe" of the B matrix with (roughly) the
213
+ // same size, which might involve multiple column "slices" (of width 16 *
214
+ // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM
215
+ // example:
216
+ // 0 1 3
217
+ // 0 2 3
218
+ // 1 2 4
219
+ // While this kind of partitioning makes things somewhat more complicated, it
220
+ // ensures good utilization of all SMs for many kinds of shape and GPU
221
+ // configurations, while requiring as few slow global cross-threadblock
222
+ // reductions as possible.
223
+
224
+ // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
225
+ // better partitioning with less reductions
226
+ int parallel = 1;
227
+ if (prob_m > 16 * thread_m_blocks) {
228
+ parallel = prob_m / (16 * thread_m_blocks);
229
+ prob_m = 16 * thread_m_blocks;
230
+ }
231
+
232
+ int k_tiles = prob_k / 16 / thread_k_blocks;
233
+ int n_tiles = prob_n / 16 / thread_n_blocks;
234
+ int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
235
+ // Ensure that the number of tiles in each stripe is a multiple of the
236
+ // groupsize; this avoids an annoying special case where a stripe starts in
237
+ // the middle of group.
238
+ if constexpr (group_blocks != -1)
239
+ iters = (group_blocks / thread_k_blocks) *
240
+ ceildiv(iters, (group_blocks / thread_k_blocks));
241
+
242
+ int slice_row = (iters * blockIdx.x) % k_tiles;
243
+ int slice_col_par = (iters * blockIdx.x) / k_tiles;
244
+ int slice_col = slice_col_par;
245
+ int slice_iters; // number of threadblock tiles in the current slice
246
+ int slice_count =
247
+ 0; // total number of active threadblocks in the current slice
248
+ int slice_idx; // index of threadblock in current slice; numbered bottom to
249
+ // top
250
+
251
+ // We can easily implement parallel problem execution by just remapping
252
+ // indices and advancing global pointers
253
+ if (slice_col_par >= n_tiles) {
254
+ A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_k / 16;
255
+ C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 4;
256
+ D += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
257
+ s_tok += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
258
+ locks += (slice_col_par / n_tiles) * n_tiles;
259
+ slice_col = slice_col_par % n_tiles;
260
+ }
261
+
262
+ // Compute all information about the current slice which is required for
263
+ // synchronization.
264
+ auto init_slice = [&]() {
265
+ slice_iters =
266
+ iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
267
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
268
+ if (slice_iters == 0) return;
269
+ if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
270
+ slice_count = 1;
271
+ slice_idx = 0;
272
+ int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
273
+ if (col_first <= k_tiles * (slice_col_par + 1)) {
274
+ int col_off = col_first - k_tiles * slice_col_par;
275
+ slice_count = ceildiv(k_tiles - col_off, iters);
276
+ if (col_off > 0) slice_count++;
277
+ int delta_first = iters * blockIdx.x - col_first;
278
+ if (delta_first < 0 || (col_off == 0 && delta_first == 0))
279
+ slice_idx = slice_count - 1;
280
+ else {
281
+ slice_idx = slice_count - 1 - delta_first / iters;
282
+ if (col_off > 0) slice_idx--;
283
+ }
284
+ }
285
+ if (slice_col == n_tiles) {
286
+ A += 16 * thread_m_blocks * prob_k / 16;
287
+ C += 16 * thread_m_blocks * prob_n / 4;
288
+ D += 16 * thread_m_blocks * prob_n / 8;
289
+ s_tok += 16 * thread_m_blocks;
290
+ locks += n_tiles;
291
+ slice_col = 0;
292
+ }
293
+ };
294
+ init_slice();
295
+
296
+ int a_gl_stride = prob_k / 16; // stride of the A matrix in global memory
297
+ // We typically use `constexpr` to indicate that this value is a compile-time
298
+ // constant
299
+ constexpr int a_sh_stride =
300
+ 16 * thread_k_blocks / 16; // stride of an A matrix tile in shared memory
301
+ constexpr int a_gl_rd_delta_o =
302
+ 16 * thread_k_blocks /
303
+ 16; // delta between subsequent A tiles in global memory
304
+ int a_gl_rd_delta_i =
305
+ a_gl_stride *
306
+ (threads / a_gl_rd_delta_o); // between subsequent accesses within a tile
307
+ constexpr int a_sh_wr_delta =
308
+ a_sh_stride *
309
+ (threads / a_gl_rd_delta_o); // between shared memory writes
310
+ constexpr int a_sh_rd_delta_o =
311
+ 1 * ((threads / 32) /
312
+ (thread_n_blocks / 4)); // between shared memory tile reads
313
+ constexpr int a_sh_rd_delta_i =
314
+ a_sh_stride * 16; // within a shared memory tile
315
+ constexpr int a_sh_stage =
316
+ a_sh_stride * (16 * thread_m_blocks); // overall size of a tile
317
+ constexpr int a_sh_wr_iters =
318
+ ceildiv(a_sh_stage,
319
+ a_sh_wr_delta); // number of shared write iterations for a tile
320
+
321
+ int b_gl_stride = 16 * prob_n / 32;
322
+ constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
323
+ int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
324
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
325
+ constexpr int b_sh_wr_delta = threads;
326
+ constexpr int b_sh_rd_delta = threads;
327
+ constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
328
+ constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
329
+
330
+ constexpr int s_tok_sh_stride = 16 * thread_m_blocks;
331
+
332
+ constexpr int s_ch_sh_stride = 16 * thread_n_blocks / 4;
333
+
334
+ int s_group_gl_stride = prob_n / 8;
335
+ constexpr int s_group_sh_stride = 16 * thread_n_blocks / 8;
336
+ constexpr int s_group_sh_stage = s_group_sh_stride;
337
+ int s_group_gl_rd_delta = s_group_gl_stride;
338
+
339
+ // Global A read index of current thread.
340
+ int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
341
+ (threadIdx.x % a_gl_rd_delta_o);
342
+ a_gl_rd += a_gl_rd_delta_o * slice_row;
343
+ // Shared write index of current thread.
344
+ int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
345
+ (threadIdx.x % a_gl_rd_delta_o);
346
+ // Shared read index.
347
+ // NOTE(HandH1998): int8 input a only need 16 threads to load 16x16 matrix
348
+ int a_sh_rd = a_sh_stride * ((threadIdx.x % 32) % 16);
349
+ a_sh_rd += 1 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
350
+
351
+ int b_gl_rd =
352
+ b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
353
+ b_gl_rd += b_sh_stride * slice_col;
354
+ b_gl_rd += b_gl_rd_delta_o * slice_row;
355
+ int b_sh_wr = threadIdx.x;
356
+ int b_sh_rd = threadIdx.x;
357
+
358
+ int s_tok_gl_rd = threadIdx.x;
359
+ // NOTE(HandH1998): activation scale s_tok need shuffle to [0, 8, 1, 9, 2, 10,
360
+ // 3, 11, 4, 12, 5, 13, 6, 14, 7, 15] for example, 0, 8 row scales serve for
361
+ // thread 0, 1, 2, 3. For more details, refer to mma operand A layout as
362
+ // s_tok's size is not fixed, we can not shuffle before inference we shuffle
363
+ // it when fetching s_tok from global memory to shared memory, that's why
364
+ // s_tok_sh_wr is like this
365
+ int s_tok_sh_wr =
366
+ (threadIdx.x / 16) * 16 + (threadIdx.x % 8) * 2 + (threadIdx.x % 16) / 8;
367
+ int s_tok_sh_rd = (threadIdx.x % 32) / 4;
368
+ bool s_tok_sh_wr_pred = threadIdx.x < prob_m;
369
+
370
+ int s_ch_gl_rd = s_ch_sh_stride * slice_col + threadIdx.x;
371
+ int s_ch_sh_wr = threadIdx.x;
372
+ int s_ch_sh_rd = 16 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
373
+ 2 * ((threadIdx.x % 32) % 4);
374
+ bool s_ch_sh_wr_pred = threadIdx.x < s_ch_sh_stride;
375
+
376
+ int s_group_gl_rd, s_group_sh_wr, s_group_sh_rd;
377
+ bool s_group_sh_wr_pred;
378
+ if constexpr (group_blocks != -1) {
379
+ s_group_gl_rd =
380
+ s_group_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
381
+ s_group_sh_stride * slice_col + threadIdx.x;
382
+ s_group_sh_wr = threadIdx.x;
383
+ // NOTE(HandH1998): s_group_sh_rd is related to mma output C
384
+ s_group_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
385
+ (threadIdx.x % 32) / 4;
386
+ s_group_sh_wr_pred = threadIdx.x < s_group_sh_stride;
387
+ }
388
+
389
+ // Precompute which thread should not read memory in which iterations; this is
390
+ // needed if there are more threads than required for a certain tilesize or
391
+ // when the batchsize is not a multiple of 16.
392
+ bool a_sh_wr_pred[a_sh_wr_iters];
393
+ #pragma unroll
394
+ for (int i = 0; i < a_sh_wr_iters; i++)
395
+ a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
396
+
397
+ // To ensure that writing and reading A tiles to/from shared memory, the
398
+ // latter in fragment format, is fully bank conflict free, we need to use a
399
+ // rather fancy XOR-based layout. The key here is that neither reads nor
400
+ // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
401
+ // same shared memory banks. Further, it seems (based on NSight-Compute) that
402
+ // each warp must also write a consecutive memory segment?
403
+ auto transform_a = [&](int i) {
404
+ int row = i / a_gl_rd_delta_o;
405
+ return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
406
+ };
407
+ // Since the computation of this remapping is non-trivial and, due to our main
408
+ // loop unrolls, all shared memory accesses are static, we simply precompute
409
+ // both transformed reads and writes.
410
+ int a_sh_wr_trans[a_sh_wr_iters];
411
+ #pragma unroll
412
+ for (int i = 0; i < a_sh_wr_iters; i++)
413
+ a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
414
+ int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
415
+ #pragma unroll
416
+ for (int i = 0; i < b_sh_wr_iters; i++) {
417
+ #pragma unroll
418
+ for (int j = 0; j < thread_m_blocks; j++)
419
+ a_sh_rd_trans[i][j] =
420
+ transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
421
+ }
422
+
423
+ // Since B-accesses have non-constant stride they have to be computed at
424
+ // runtime; we break dependencies between subsequent accesses with a tile by
425
+ // maintining multiple pointers (we have enough registers), a tiny
426
+ // optimization.
427
+ const int4* B_ptr[b_sh_wr_iters];
428
+ #pragma unroll
429
+ for (int i = 0; i < b_sh_wr_iters; i++)
430
+ B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
431
+
432
+ extern __shared__ int4 sh[];
433
+ // Shared memory storage for global fetch pipelines.
434
+ // NOTE(HandH1998): stages need >= 4, otherwise, sh_s_tok = sh + max(stages *
435
+ // a_sh_stage + stages * b_sh_stage, 4 * stages * a_sh_stage)
436
+ int4* sh_a = sh;
437
+ int4* sh_b = sh_a + (stages * a_sh_stage);
438
+ int4* sh_s_tok = sh_b + (stages * b_sh_stage);
439
+ int4* sh_s_ch = sh_s_tok + s_tok_sh_stride;
440
+ int4* sh_s_group = sh_s_ch + s_ch_sh_stride;
441
+
442
+ // Register storage for double buffer of shared memory reads.
443
+ FragA frag_a[2][thread_m_blocks];
444
+ I4 frag_b_quant[2];
445
+ FragC frag_c[thread_m_blocks][4][2];
446
+ FragS_GROUP frag_s_group[2][4];
447
+ FragS_CHANNEL frag_s_tok[thread_m_blocks];
448
+ FragS_CHANNEL frag_s_ch[2][4];
449
+
450
+ // Zero accumulators.
451
+ auto zero_accums = [&]() {
452
+ #pragma unroll
453
+ for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
454
+ reinterpret_cast<int*>(frag_c)[i] = 0;
455
+ };
456
+
457
+ // Asynchronously fetch the next A, B and s tile from global to the next
458
+ // shared memory pipeline location.
459
+ auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
460
+ if (pred) {
461
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
462
+ #pragma unroll
463
+ for (int i = 0; i < a_sh_wr_iters; i++) {
464
+ cp_async4_pred(
465
+ &sh_a_stage[a_sh_wr_trans[i]],
466
+ &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
467
+ a_sh_wr_pred[i]);
468
+ }
469
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
470
+ #pragma unroll
471
+ for (int i = 0; i < b_sh_wr_iters; i++) {
472
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
473
+ B_ptr[i] += b_gl_rd_delta_o;
474
+ }
475
+ // Only fetch scales if this tile starts a new group
476
+ if constexpr (group_blocks != -1) {
477
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
478
+ int4* sh_s_group_stage = sh_s_group + s_group_sh_stage * pipe;
479
+ if (s_group_sh_wr_pred)
480
+ cp_async4(&sh_s_group_stage[s_group_sh_wr],
481
+ &s_group[s_group_gl_rd]);
482
+ s_group_gl_rd += s_group_gl_rd_delta;
483
+ }
484
+ }
485
+ }
486
+ // Insert a fence even when we are winding down the pipeline to ensure that
487
+ // waiting is also correct at this point.
488
+ cp_async_fence();
489
+ };
490
+
491
+ // Wait until the next thread tile has been loaded to shared memory.
492
+ auto wait_for_stage = [&]() {
493
+ // We only have `stages - 2` active fetches since we are double buffering
494
+ // and can only issue the next fetch when it is guaranteed that the previous
495
+ // shared memory load is fully complete (as it may otherwise be
496
+ // overwritten).
497
+ cp_async_wait<stages - 2>();
498
+ __syncthreads();
499
+ };
500
+
501
+ // Load the next sub-tile from the current location in the shared memory pipe
502
+ // into the current register buffer.
503
+ auto fetch_to_registers = [&](int k, int pipe) {
504
+ // It may seem inefficient that we reload the groups for every sub-tile;
505
+ // however, this does not seem to be a significant bottleneck, while some
506
+ // theoretically better attempts have lead to bad instruction ordering by
507
+ // the compiler and correspondingly a noticeable drop in performance.
508
+ if constexpr (group_blocks != -1) {
509
+ int4* sh_s_group_stage =
510
+ sh_s_group +
511
+ s_group_sh_stage * ((group_blocks / thread_k_blocks) *
512
+ (pipe / (group_blocks / thread_k_blocks)));
513
+ reinterpret_cast<int4*>(&frag_s_group[k % 2])[0] =
514
+ sh_s_group_stage[s_group_sh_rd];
515
+ }
516
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
517
+ #pragma unroll
518
+ for (int i = 0; i < thread_m_blocks; i++)
519
+ ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
520
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
521
+ frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
522
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
523
+ };
524
+
525
+ // Execute the actual tensor core matmul of a sub-tile.
526
+ auto matmul = [&](int k) {
527
+ // We have the m dimension as the inner loop in order to encourage overlapping
528
+ // dequantization and matmul operations.
529
+ #pragma unroll
530
+ for (int j = 0; j < 4; j++) {
531
+ int b_quant = frag_b_quant[k % 2][j];
532
+ // int b_quant_shift = b_quant << 4;
533
+ FragB frag_b0, frag_b1;
534
+ // If there are no groups, we can just scale the final output once and can
535
+ // avoid doing so for each weight.
536
+ if constexpr (group_blocks != -1) {
537
+ int b_quant_shift = b_quant >> 8;
538
+ frag_b0 = dequant_per_group(b_quant, frag_s_group[k % 2][j], 0);
539
+ frag_b1 = dequant_per_group(b_quant_shift, frag_s_group[k % 2][j], 1);
540
+ } else {
541
+ int b_quant_shift = b_quant << 4;
542
+ frag_b0 = dequant_per_channel(b_quant);
543
+ frag_b1 = dequant_per_channel(b_quant_shift);
544
+ }
545
+ #pragma unroll
546
+ for (int i = 0; i < thread_m_blocks; i++) {
547
+ mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
548
+ mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
549
+ }
550
+ }
551
+ };
552
+
553
+ // Since we slice across the k dimension of a tile in order to increase the
554
+ // number of warps while keeping the n dimension of a tile reasonable, we have
555
+ // multiple warps that accumulate their partial sums of the same output
556
+ // location; which we have to reduce over in the end. We do in shared memory.
557
+ auto thread_block_reduce = [&]() {
558
+ constexpr int red_off = threads / b_sh_stride / 2;
559
+ if (red_off >= 1) {
560
+ int red_idx = threadIdx.x / b_sh_stride;
561
+ constexpr int red_sh_stride = b_sh_stride * 4 * 2;
562
+ constexpr int red_sh_delta = b_sh_stride;
563
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
564
+ (threadIdx.x % b_sh_stride);
565
+
566
+ // Parallel logarithmic shared memory reduction. We make sure to avoid any
567
+ // unnecessary read or write iterations, e.g., for two warps we write only
568
+ // once by warp 1 and read only once by warp 0.
569
+
570
+ #pragma unroll
571
+ for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
572
+ #pragma unroll
573
+ for (int i = red_off; i > 0; i /= 2) {
574
+ if (i <= red_idx && red_idx < 2 * i) {
575
+ #pragma unroll
576
+ for (int j = 0; j < 4 * 2; j++) {
577
+ int red_sh_wr =
578
+ red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
579
+ if (i < red_off) {
580
+ int* c_rd =
581
+ reinterpret_cast<int*>(&sh[red_sh_delta * j + red_sh_rd]);
582
+ int* c_wr = reinterpret_cast<int*>(&sh[red_sh_wr]);
583
+ #pragma unroll
584
+ for (int k = 0; k < 4; k++)
585
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
586
+ c_rd[k] + c_wr[k];
587
+ }
588
+ sh[red_sh_wr] =
589
+ reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
590
+ }
591
+ }
592
+ __syncthreads();
593
+ }
594
+ if (red_idx == 0) {
595
+ #pragma unroll
596
+ for (int i = 0; i < 4 * 2; i++) {
597
+ int* c_rd =
598
+ reinterpret_cast<int*>(&sh[red_sh_delta * i + red_sh_rd]);
599
+ #pragma unroll
600
+ for (int j = 0; j < 4; j++)
601
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
602
+ c_rd[j];
603
+ }
604
+ }
605
+ __syncthreads();
606
+ }
607
+ }
608
+ };
609
+
610
+ // Since multiple threadblocks may process parts of the same column slice, we
611
+ // finally have to globally reduce over the results. As the striped
612
+ // partitioning minimizes the number of such reductions and our outputs are
613
+ // usually rather small, we perform this reduction serially in L2 cache.
614
+ // global_reduce works on INT32 elements, which are the results of INT8 GEMM.
615
+ // This is why we need another INT32 maxtrix `C` to reduce instead of the
616
+ // original half matrix `D`.
617
+ auto global_reduce = [&](bool first = false, bool last = false) {
618
+ // We are very careful here to reduce directly in the output buffer to
619
+ // maximize L2 cache utilization in this step. To do this, we write out
620
+ // results in FP16 (but still reduce with FP32 compute).
621
+ constexpr int active_threads = 32 * thread_n_blocks / 4;
622
+ if (threadIdx.x < active_threads) {
623
+ int c_gl_stride = prob_n / 4;
624
+ int c_gl_wr_delta_o = 8 * c_gl_stride;
625
+ int c_gl_wr_delta_i = 8 * (active_threads / 32);
626
+ int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
627
+ 8 * (threadIdx.x / 32) + (threadIdx.x % 4) * 2;
628
+ c_gl_wr += (4 * thread_n_blocks) * slice_col;
629
+ constexpr int c_sh_wr_delta = active_threads * 2;
630
+ int c_sh_wr = 2 * threadIdx.x;
631
+
632
+ int row = (threadIdx.x % 32) / 4;
633
+
634
+ if (!first) {
635
+ // Interestingly, doing direct global accesses here really seems to mess up
636
+ // the compiler and lead to slowdowns, hence we also use async-copies even
637
+ // though these fetches are not actually asynchronous.
638
+ #pragma unroll
639
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
640
+ cp_async4_pred(
641
+ &sh[c_sh_wr + c_sh_wr_delta * i],
642
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
643
+ c_gl_wr_delta_i * (i % 2)],
644
+ i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
645
+ cp_async4_pred(
646
+ &sh[c_sh_wr + c_sh_wr_delta * i + 1],
647
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
648
+ c_gl_wr_delta_i * (i % 2) + 1],
649
+ i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
650
+ }
651
+ cp_async_fence();
652
+ cp_async_wait<0>();
653
+ }
654
+
655
+ #pragma unroll
656
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
657
+ if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
658
+ if (!first) {
659
+ int4 d_red1 = sh[c_sh_wr + i * c_sh_wr_delta];
660
+ int4 d_red2 = sh[c_sh_wr + i * c_sh_wr_delta + 1];
661
+ #pragma unroll
662
+ for (int j = 0; j < 4; j++) {
663
+ reinterpret_cast<int*>(
664
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
665
+ reinterpret_cast<int*>(&d_red1)[j];
666
+ }
667
+ #pragma unroll
668
+ for (int j = 0; j < 4; j++) {
669
+ reinterpret_cast<int*>(
670
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * (j + 4) + (i % 4)] +=
671
+ reinterpret_cast<int*>(&d_red2)[j];
672
+ }
673
+ }
674
+ if (!last) {
675
+ int4 d1, d2;
676
+ #pragma unroll
677
+ for (int j = 0; j < 4; j++) {
678
+ reinterpret_cast<int*>(&d1)[j] = reinterpret_cast<int*>(
679
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)];
680
+ }
681
+ #pragma unroll
682
+ for (int j = 0; j < 4; j++) {
683
+ reinterpret_cast<int*>(&d2)[j] = reinterpret_cast<int*>(
684
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * (j + 4) + (i % 4)];
685
+ }
686
+ C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
687
+ d1;
688
+ C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2) +
689
+ 1] = d2;
690
+ }
691
+ }
692
+ }
693
+ }
694
+ };
695
+
696
+ // Write out the reduce final result in the correct layout. We only actually
697
+ // reshuffle matrix fragments in this step, the reduction above is performed
698
+ // in fragment layout.
699
+ auto write_result = [&]() {
700
+ int d_gl_stride = prob_n / 8;
701
+ constexpr int d_sh_stride = 2 * thread_n_blocks + 1;
702
+ int d_gl_wr_delta = d_gl_stride * (threads / (2 * thread_n_blocks));
703
+ constexpr int d_sh_rd_delta =
704
+ d_sh_stride * (threads / (2 * thread_n_blocks));
705
+
706
+ int d_gl_wr = d_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
707
+ (threadIdx.x % (2 * thread_n_blocks));
708
+ d_gl_wr += (2 * thread_n_blocks) * slice_col;
709
+ int d_sh_wr =
710
+ (4 * d_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
711
+ d_sh_wr += 32 * (threadIdx.x / 32);
712
+ int d_sh_rd = d_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
713
+ (threadIdx.x % (2 * thread_n_blocks));
714
+
715
+ int d_gl_wr_end = d_gl_stride * prob_m;
716
+
717
+ // We first reorder in shared memory to guarantee the most efficient final
718
+ // global write patterns
719
+ auto write = [&](int idx, int c0, int c1, float a_s, FragS_CHANNEL& w_s) {
720
+ float2 deq_res;
721
+ deq_res.x = int32_to_float(c0) * w_s[0] * a_s;
722
+ deq_res.y = int32_to_float(c1) * w_s[1] * a_s;
723
+ ((half2*)sh)[idx] = float2_to_half2(deq_res);
724
+ };
725
+
726
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
727
+ #pragma unroll
728
+ for (int i = 0; i < thread_m_blocks; i++) {
729
+ #pragma unroll
730
+ for (int j = 0; j < 4; j++) {
731
+ int wr = d_sh_wr + 8 * j;
732
+ write(wr + (4 * d_sh_stride) * 0 + 0, frag_c[i][j][0][0],
733
+ frag_c[i][j][0][1], frag_s_tok[i][0],
734
+ frag_s_ch[j / 2][2 * (j % 2) + 0]);
735
+ write(wr + (4 * d_sh_stride) * 8 + 0, frag_c[i][j][0][2],
736
+ frag_c[i][j][0][3], frag_s_tok[i][1],
737
+ frag_s_ch[j / 2][2 * (j % 2) + 0]);
738
+ write(wr + (4 * d_sh_stride) * 0 + 4, frag_c[i][j][1][0],
739
+ frag_c[i][j][1][1], frag_s_tok[i][0],
740
+ frag_s_ch[j / 2][2 * (j % 2) + 1]);
741
+ write(wr + (4 * d_sh_stride) * 8 + 4, frag_c[i][j][1][2],
742
+ frag_c[i][j][1][3], frag_s_tok[i][1],
743
+ frag_s_ch[j / 2][2 * (j % 2) + 1]);
744
+ }
745
+ d_sh_wr += 16 * (4 * d_sh_stride);
746
+ }
747
+ }
748
+ __syncthreads();
749
+
750
+ #pragma unroll
751
+ for (int i = 0;
752
+ i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
753
+ i++) {
754
+ if (d_gl_wr < d_gl_wr_end) {
755
+ D[d_gl_wr] = sh[d_sh_rd];
756
+ d_gl_wr += d_gl_wr_delta;
757
+ d_sh_rd += d_sh_rd_delta;
758
+ }
759
+ }
760
+ };
761
+
762
+ // Start global fetch and register load pipelines.
763
+ auto start_pipes = [&]() {
764
+ #pragma unroll
765
+ for (int i = 0; i < stages - 1; i++) fetch_to_shared(i, i, i < slice_iters);
766
+ zero_accums();
767
+ wait_for_stage();
768
+ fetch_to_registers(0, 0);
769
+ a_gl_rd += a_gl_rd_delta_o * (stages - 1);
770
+ };
771
+ start_pipes();
772
+
773
+ // Main loop.
774
+ while (slice_iters) {
775
+ // We unroll over both the global fetch and the register load pipeline to
776
+ // ensure all shared memory accesses are static. Note that both pipelines have
777
+ // even length meaning that the next iteration will always start at index 0.
778
+ #pragma unroll
779
+ for (int pipe = 0; pipe < stages;) {
780
+ #pragma unroll
781
+ for (int k = 0; k < b_sh_wr_iters; k++) {
782
+ fetch_to_registers(k + 1, pipe % stages);
783
+ if (k == b_sh_wr_iters - 2) {
784
+ fetch_to_shared((pipe + stages - 1) % stages, pipe,
785
+ slice_iters >= stages);
786
+ pipe++;
787
+ wait_for_stage();
788
+ }
789
+ matmul(k);
790
+ }
791
+ slice_iters--;
792
+ if (slice_iters == 0) break;
793
+ }
794
+ a_gl_rd += a_gl_rd_delta_o * stages;
795
+
796
+ // Process results and, if necessary, proceed to the next column slice.
797
+ // While this pattern may not be the most readable, other ways of writing
798
+ // the loop seemed to noticeably worse performance after compilation.
799
+ if (slice_iters == 0) {
800
+ cp_async_wait<0>();
801
+ bool last = slice_idx == slice_count - 1;
802
+ // For per-column scales, we only fetch them here in the final step before
803
+ // write-out
804
+ if (last) {
805
+ if (s_tok_sh_wr_pred) {
806
+ cp_async1(&sh_s_tok[s_tok_sh_wr], &s_tok[s_tok_gl_rd]);
807
+ }
808
+ if (s_ch_sh_wr_pred) {
809
+ cp_async4(&sh_s_ch[s_ch_sh_wr], &s_ch[s_ch_gl_rd]);
810
+ }
811
+ cp_async_fence();
812
+ }
813
+ thread_block_reduce();
814
+ if (last) {
815
+ cp_async_wait<0>();
816
+ __syncthreads();
817
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
818
+ #pragma unroll
819
+ for (int i = 0; i < thread_m_blocks; i++) {
820
+ frag_s_tok[i][0] =
821
+ *reinterpret_cast<float*>(&sh_s_tok[16 * i + 2 * s_tok_sh_rd]);
822
+ frag_s_tok[i][1] = *reinterpret_cast<float*>(
823
+ &sh_s_tok[16 * i + 2 * s_tok_sh_rd + 1]);
824
+ }
825
+ reinterpret_cast<int4*>(&frag_s_ch)[0] = sh_s_ch[s_ch_sh_rd + 0];
826
+ reinterpret_cast<int4*>(&frag_s_ch)[1] = sh_s_ch[s_ch_sh_rd + 1];
827
+ reinterpret_cast<int4*>(&frag_s_ch)[2] = sh_s_ch[s_ch_sh_rd + 8];
828
+ reinterpret_cast<int4*>(&frag_s_ch)[3] = sh_s_ch[s_ch_sh_rd + 9];
829
+ }
830
+ }
831
+ if (slice_count > 1) { // only globally reduce if there is more than one
832
+ // block in a slice
833
+ barrier_acquire(&locks[slice_col], slice_idx);
834
+ global_reduce(slice_idx == 0, last);
835
+ barrier_release(&locks[slice_col], last);
836
+ }
837
+ if (last) // only the last block in a slice actually writes the result
838
+ write_result();
839
+ slice_row = 0;
840
+ slice_col_par++;
841
+ slice_col++;
842
+ init_slice();
843
+ if (slice_iters) {
844
+ a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
845
+ (threadIdx.x % a_gl_rd_delta_o);
846
+ #pragma unroll
847
+ for (int i = 0; i < b_sh_wr_iters; i++)
848
+ B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
849
+ if (slice_col == 0) {
850
+ #pragma unroll
851
+ for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
852
+ }
853
+ s_group_gl_rd = s_group_sh_stride * slice_col + threadIdx.x;
854
+ s_ch_gl_rd = s_ch_sh_stride * slice_col + threadIdx.x;
855
+ start_pipes();
856
+ }
857
+ }
858
+ }
859
+}
860
+
861
+#else
862
+
863
+template <const int threads, // number of threads in a threadblock
864
+ const int thread_m_blocks, // number of 16x16 blocks in the m
865
+ // dimension (batchsize) of the
866
+ // threadblock
867
+ const int thread_n_blocks, // same for n dimension (output)
868
+ const int thread_k_blocks, // same for k dimension (reduction)
869
+ const int stages, // number of stages for the async global->shared
870
+ // fetch pipeline
871
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
872
+ // with a separate quantization scale
873
+ >
874
+__global__ void Marlin(
875
+ const int4* __restrict__ A, // int8 input matrix of shape mxk
876
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
877
+ int4* __restrict__ C, // int32 global_reduce buffer of shape
878
+ // (max_par*16*4)xn, as int8 tensor core's output is
879
+ // int32 dtype
880
+ int4* __restrict__ D, // fp16 output buffer of shape mxn
881
+ const float* __restrict__ s_tok, // fp32 activation per-token quantization
882
+ // scales of shape mx1
883
+ const int4* __restrict__ s_ch, // fp32 weight per-channel quantization
884
+ // scales of shape 1xn
885
+ const int4* __restrict__ s_group, // fp16 weight per-group quantization
886
+ // scales of shape (k/groupsize)xn, when
887
+ // group_blocks=-1, it should be nullptr
888
+ int prob_m, // batch dimension m
889
+ int prob_n, // output dimension n
890
+ int prob_k, // reduction dimension k
891
+ int* locks // extra global storage for barrier synchronization
892
+) {
893
+ // Marlin is not implemented yet for SM < 8.0
894
+ assert(false);
895
+ return;
896
+}
897
+
898
+#endif
899
+
900
+// 8 warps are a good choice since every SM has 4 schedulers and having more
901
+// than 1 warp per schedule allows some more latency hiding. At the same time,
902
+// we want relatively few warps to have many registers per warp and small tiles.
903
+const int USER_THREADS =
904
+ 256; // Note: This is only used with user-provided thread_k/n
905
+const int STAGES = 4; // 4 pipeline stages fit into shared memory
906
+
907
+static constexpr int min_thread_n = 64;
908
+static constexpr int min_thread_k = 64;
909
+
910
+static constexpr int tile_size = 16;
911
+static constexpr int max_par = 16;
912
+
913
+static constexpr int pack_factor_4bit =
914
+ 8; // We have 8 4-bit vals inside a 32 bit
915
+
916
+#define __CALL_IF(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
917
+ GROUP_BLOCKS, NUM_THREADS) \
918
+ else if (thread_m_blocks == THREAD_M_BLOCKS && \
919
+ thread_n_blocks == THREAD_N_BLOCKS && \
920
+ thread_k_blocks == THREAD_K_BLOCKS && \
921
+ group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
922
+ cudaFuncSetAttribute(Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
923
+ THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
924
+ cudaFuncAttributeMaxDynamicSharedMemorySize, \
925
+ max_shared_mem); \
926
+ Marlin<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
927
+ STAGES, GROUP_BLOCKS> \
928
+ <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
929
+ A_ptr, B_ptr, C_ptr, D_ptr, s_tok_ptr, s_ch_ptr, s_group_ptr, \
930
+ prob_m, prob_n, prob_k, locks); \
931
+ }
932
+
933
+typedef struct {
934
+ int thread_k;
935
+ int thread_n;
936
+ int num_threads;
937
+} thread_config_t;
938
+
939
+thread_config_t small_batch_thread_configs[] = {
940
+ // Ordered by priority
941
+
942
+ // thread_k, thread_n, num_threads
943
+ {128, 128, 256}, // Default
944
+ {128, 64, 128}, // Reduce N 2X, same K
945
+ {64, 256, 256}, // Reduce K 2X, increase N 2X
946
+ {64, 128, 128}, // Reduce K 2X, same N
947
+};
948
+
949
+thread_config_t large_batch_thread_configs[] = {
950
+ // Ordered by priority
951
+
952
+ // thread_k, thread_n, num_threads
953
+ {64, 256, 256}, // Default
954
+ {128, 128, 256}, // Reduce N 2X, increase K 2X
955
+ {64, 128, 128}, // Reduce N 2X, same K
956
+ {128, 64, 128}, // Reduce N 4X, increase K 2X
957
+};
958
+
959
+bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
960
+ int prob_k) {
961
+ // Sanity
962
+ if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
963
+ th_config.num_threads == -1) {
964
+ return false;
965
+ }
966
+
967
+ // Verify K/N are divisible by thread K/N
968
+ if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) {
969
+ return false;
970
+ }
971
+
972
+ // thread_k can be only 128 or 64 (because it must be less than groupsize
973
+ // which is 128)
974
+ if (th_config.thread_k != 128 && th_config.thread_k != 64) {
975
+ return false;
976
+ }
977
+
978
+ // Verify min for thread K/N
979
+ if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) {
980
+ return false;
981
+ }
982
+
983
+ // num_threads must be at least 128 (= 4 warps)
984
+ if (th_config.num_threads < 128) {
985
+ return false;
986
+ }
987
+
988
+ return true;
989
+}
990
+
991
+thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
992
+ if (prob_m <= 16) {
993
+ for (auto th_config : small_batch_thread_configs) {
994
+ if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
995
+ return th_config;
996
+ }
997
+ }
998
+
999
+ } else {
1000
+ for (auto th_config : large_batch_thread_configs) {
1001
+ if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1002
+ return th_config;
1003
+ }
1004
+ }
1005
+ }
1006
+
1007
+ return thread_config_t{-1, -1, -1};
1008
+}
1009
+
1010
+#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1011
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1012
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
1013
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1014
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
1015
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1016
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
1017
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1018
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
1019
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1020
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS)
1021
+
1022
+void marlin_qqq_cuda(const void* A, const void* B, void* C, void* D,
1023
+ void* s_tok, void* s_ch, void* s_group, int prob_m,
1024
+ int prob_n, int prob_k, void* workspace,
1025
+ int groupsize = -1, int dev = 0, cudaStream_t stream = 0,
1026
+ int thread_k = -1, int thread_n = -1, int sms = -1,
1027
+ int max_par = 16) {
1028
+ int tot_m = prob_m;
1029
+ int tot_m_blocks = ceildiv(tot_m, 16);
1030
+ int pad = 16 * tot_m_blocks - tot_m;
1031
+
1032
+ if (sms == -1)
1033
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1034
+
1035
+ int max_shared_mem = 0;
1036
+ cudaDeviceGetAttribute(&max_shared_mem,
1037
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1038
+ TORCH_CHECK(max_shared_mem > 0);
1039
+
1040
+ // Set thread config
1041
+ thread_config_t th_config;
1042
+ if (thread_k != -1 && thread_n != -1) {
1043
+ // User-defined config
1044
+ th_config = thread_config_t{thread_k, thread_n, USER_THREADS};
1045
+ } else {
1046
+ // Auto config
1047
+ th_config = determine_thread_config(prob_m, prob_n, prob_k);
1048
+ }
1049
+
1050
+ if (!is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1051
+ throw std::runtime_error(
1052
+ "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1053
+ ", thread_n = " + str(th_config.thread_n) +
1054
+ ", num_threads = " + str(th_config.num_threads) + " for MKN = [" +
1055
+ str(prob_m) + ", " + str(prob_k) + ", " + str(prob_n) + "]");
1056
+ }
1057
+
1058
+ int num_threads = th_config.num_threads;
1059
+ thread_k = th_config.thread_k;
1060
+ thread_n = th_config.thread_n;
1061
+
1062
+ int thread_k_blocks = thread_k / 16;
1063
+ int thread_n_blocks = thread_n / 16;
1064
+ int group_blocks = (groupsize == -1) ? -1 : groupsize / 16;
1065
+ int blocks = sms;
1066
+
1067
+ if (prob_m == 0 || prob_n == 0 || prob_k == 0) {
1068
+ return;
1069
+ }
1070
+
1071
+ TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
1072
+ " is not divisible by thread_n = ", thread_n);
1073
+ TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
1074
+ " is not divisible by thread_k = ", thread_k);
1075
+ if (group_blocks != -1) {
1076
+ TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1077
+ " is not divisible by group_blocks = ", group_blocks);
1078
+ }
1079
+
1080
+ const int4* A_ptr = (const int4*)A;
1081
+ const int4* B_ptr = (const int4*)B;
1082
+ int4* C_ptr = (int4*)C;
1083
+ int4* D_ptr = (int4*)D;
1084
+ const float* s_tok_ptr = (const float*)s_tok;
1085
+ const int4* s_ch_ptr = (const int4*)s_ch;
1086
+ const int4* s_group_ptr = (const int4*)s_group;
1087
+
1088
+ int* locks = (int*)workspace;
1089
+
1090
+ for (int i = 0; i < tot_m_blocks; i += 4) {
1091
+ int thread_m_blocks = tot_m_blocks - i;
1092
+ prob_m = tot_m - 16 * i;
1093
+ int par = 1;
1094
+ if (thread_m_blocks > 4) {
1095
+ // Note that parallel > 1 currently only works for inputs without any
1096
+ // padding
1097
+ par = (16 * thread_m_blocks - pad) / 64;
1098
+ if (par > max_par) par = max_par;
1099
+ prob_m = 64 * par;
1100
+ i += 4 * (par - 1);
1101
+ thread_m_blocks = 4;
1102
+ }
1103
+
1104
+ // For compilation speed, we only define the kernel configurations that have
1105
+ // seemed useful (in terms of performance) in our testing, however many more
1106
+ // are, in principle, possible.
1107
+ if (false) {
1108
+ }
1109
+ CALL_IF(8, 8, 256)
1110
+ CALL_IF(16, 4, 256)
1111
+ CALL_IF(8, 4, 128)
1112
+ CALL_IF(4, 8, 128)
1113
+ else {
1114
+ throw std::runtime_error("Unsupported shapes: MKN = [" + str(prob_m) +
1115
+ ", " + str(prob_k) + ", " + str(prob_n) + "]" +
1116
+ ", groupsize = " + str(groupsize) +
1117
+ ", thread_m_blocks = " + str(thread_m_blocks) +
1118
+ ", thread_n_blocks = " + str(thread_n_blocks) +
1119
+ ", thread_k_blocks = " + str(thread_k_blocks));
1120
+ }
1121
+
1122
+ A_ptr += 16 * thread_m_blocks * (prob_k / 16) * par;
1123
+ D_ptr += 16 * thread_m_blocks * (prob_n / 8) * par;
1124
+ s_tok_ptr += 16 * thread_m_blocks * par;
1125
+ }
1126
+}
1127
+} // anonymous namespace
1128
+
1129
+torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
1130
+ torch::Tensor const& b_q_weight,
1131
+ torch::Tensor const& s_tok,
1132
+ torch::Tensor const& s_ch,
1133
+ torch::Tensor const& s_group,
1134
+ torch::Tensor& workspace, int64_t size_m,
1135
+ int64_t size_n, int64_t size_k) {
1136
+ // Verify M
1137
+ TORCH_CHECK(size_m == a.size(0),
1138
+ "Shape mismatch: a.size(0) = " + str(a.size(0)) +
1139
+ ", size_m = " + str(size_m));
1140
+ TORCH_CHECK(size_m == s_tok.numel(),
1141
+ "Shape mismatch: s_tok.numel() = " + str(s_tok.numel()) +
1142
+ ", size_m = " + str(size_m));
1143
+
1144
+ // Verify K
1145
+ TORCH_CHECK(size_k == a.size(1),
1146
+ "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1147
+ ", size_k = " + str(size_k));
1148
+ TORCH_CHECK(size_k % tile_size == 0,
1149
+ "size_k = " + str(size_k) +
1150
+ " is not divisible by tile_size = " + str(tile_size));
1151
+ TORCH_CHECK(
1152
+ (size_k / tile_size) == b_q_weight.size(0),
1153
+ "Shape mismatch: b_q_weight.size(0) = " + str(b_q_weight.size(0)) +
1154
+ ", size_k = " + str(size_k) + ", tile_size = " + str(tile_size));
1155
+
1156
+ int groupsize = (s_group.numel() == 0) ? -1 : size_k / s_group.size(0);
1157
+ // Verify groupsize
1158
+ TORCH_CHECK(groupsize == -1 || groupsize == 128,
1159
+ "Unexpected groupsize = " + str(groupsize));
1160
+
1161
+ // Verify N
1162
+ TORCH_CHECK(s_ch.numel() == size_n,
1163
+ "Shape mismatch: s_ch.numel() = " + str(s_ch.numel()) +
1164
+ ", size_n = " + str(size_n));
1165
+ TORCH_CHECK(b_q_weight.size(1) % tile_size == 0,
1166
+ "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1167
+ " is not divisible by tile_size = " + str(tile_size));
1168
+ if (groupsize != -1) {
1169
+ TORCH_CHECK(s_group.size(1) == size_n,
1170
+ "Shape mismatch: s_group.size(1) = " + str(s_group.size(1)) +
1171
+ ", size_n = " + str(size_n));
1172
+ TORCH_CHECK(
1173
+ size_k % s_group.size(0) == 0,
1174
+ "size_k = " + str(size_k) +
1175
+ ", is not divisible by s_group.size(0) = " + str(s_group.size(0)));
1176
+ }
1177
+
1178
+ int actual_size_n = (b_q_weight.size(1) / tile_size) * pack_factor_4bit;
1179
+ TORCH_CHECK(size_n == actual_size_n,
1180
+ "Shape mismatch: size_n = " + str(size_n) +
1181
+ ", actual_size_n = " + str(actual_size_n));
1182
+
1183
+ // Verify A device and strides
1184
+ TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1185
+ TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1186
+
1187
+ // Verify B device and strides
1188
+ TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
1189
+ TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
1190
+
1191
+ // Verify s_tok device, strides and dtype
1192
+ TORCH_CHECK(s_tok.device().is_cuda(), "s_tok is not on GPU");
1193
+ TORCH_CHECK(s_tok.is_contiguous(), "s_tok is not contiguous");
1194
+ TORCH_CHECK(s_tok.dtype() == torch::kFloat32, "s_tok's dtype is not float32");
1195
+
1196
+ // Verify s_ch device, strides and dtype
1197
+ TORCH_CHECK(s_ch.device().is_cuda(), "s_ch is not on GPU");
1198
+ TORCH_CHECK(s_ch.is_contiguous(), "s_ch is not contiguous");
1199
+ TORCH_CHECK(s_ch.dtype() == torch::kFloat32, "s_ch's dtype is not float32");
1200
+
1201
+ // Verify s_group device, strides and dtype
1202
+ TORCH_CHECK(s_group.device().is_cuda(), "s_group is not on GPU");
1203
+ TORCH_CHECK(s_group.is_contiguous(), "s_group is not contiguous");
1204
+ TORCH_CHECK(s_group.dtype() == torch::kFloat16,
1205
+ "s_group's dtype is not float16");
1206
+
1207
+ // Verify workspace size
1208
+ TORCH_CHECK(size_n % min_thread_n == 0,
1209
+ "size_n = " + str(size_n) +
1210
+ ", is not divisible by min_thread_n = " + str(min_thread_n));
1211
+ int min_workspace_size = (size_n / min_thread_n) * max_par;
1212
+ TORCH_CHECK(workspace.numel() >= min_workspace_size,
1213
+ "workspace.numel = " + str(workspace.numel()) +
1214
+ " is below min_workspace_size = " + str(min_workspace_size));
1215
+
1216
+ // Alloc C matrix
1217
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1218
+ auto options_c = torch::TensorOptions().dtype(torch::kInt).device(a.device());
1219
+ torch::Tensor c = torch::empty({max_par * 64, size_n}, options_c);
1220
+
1221
+ // Alloc D matrix
1222
+ auto options_d =
1223
+ torch::TensorOptions().dtype(torch::kFloat16).device(a.device());
1224
+ torch::Tensor d = torch::empty({size_m, size_n}, options_d);
1225
+
1226
+ // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
1227
+ // auto -1)
1228
+ int thread_k = -1;
1229
+ // thread_n: `n` size of a thread_tile in `weights` (can usually be left as
1230
+ // auto -1)
1231
+ int thread_n = -1;
1232
+ // sms: number of SMs to use for the kernel (can usually be left as auto -1)
1233
+ int sms = -1;
1234
+
1235
+ int dev = a.get_device();
1236
+ marlin_qqq_cuda(
1237
+ a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), d.data_ptr(),
1238
+ s_tok.data_ptr(), s_ch.data_ptr(), s_group.data_ptr(), size_m, size_n,
1239
+ size_k, workspace.data_ptr(), groupsize, dev,
1240
+ at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms, max_par);
1241
+
1242
+ return d;
1243
+}
csrc/torch_bindings.cpp
+4
@@ -149,6 +149,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
149
ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
150
ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
151
152
+ // marlin_qqq_gemm for QQQ.
153
+ ops.def("marlin_qqq_gemm", &marlin_qqq_gemm);
154
+ ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
155
+
156
// CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
157
// quantization.
158
ops.def(
tests/kernels/test_marlin_gemm.py
+66
@@ -10,6 +10,9 @@ from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
12
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
13
+from vllm.model_executor.layers.quantization.qqq import (
14
+ MARLIN_QQQ_MAX_PARALLEL, MARLIN_QQQ_MIN_THREAD_N,
15
+ MARLIN_QQQ_SUPPORTED_GROUP_SIZES, MARLIN_QQQ_SUPPORTED_NUM_BITS)
16
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
17
GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
18
MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS,
@@ -21,6 +24,8 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
24
marlin_weights)
25
from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
26
marlin_24_quantize)
27
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test_qqq import ( # noqa: E501
28
+ marlin_qqq_quantize)
29
from vllm.model_executor.layers.quantization.utils.quant_utils import (
30
awq_pack, gptq_pack, quantize_weights, quantize_weights_with_zp,
31
sort_weights)
@@ -425,3 +430,64 @@ def test_awq_marlin_gemm(
430
print("max_diff = {}".format(max_diff))
431
432
assert max_diff < 0.04
433
+
434
+
435
+@pytest.mark.skipif(not is_quant_method_supported("qqq"),
436
+ reason="Marlin is not supported on this GPU type.")
437
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
438
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
439
+@pytest.mark.parametrize("num_bits", MARLIN_QQQ_SUPPORTED_NUM_BITS)
440
+@pytest.mark.parametrize("group_size", MARLIN_QQQ_SUPPORTED_GROUP_SIZES)
441
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
442
+def test_marlin_qqq_gemm(
443
+ k_chunk,
444
+ n_chunk,
445
+ num_bits,
446
+ group_size,
447
+ mnk_factors,
448
+):
449
+ int8_traits = torch.iinfo(torch.int8)
450
+ m_factor, n_factor, k_factor = mnk_factors
451
+
452
+ size_m = m_factor
453
+ size_k = k_chunk * k_factor
454
+ size_n = n_chunk * n_factor
455
+
456
+ print(f"MNK = {size_m} {size_n} {size_k}")
457
+ print(f"groupsize = {group_size}")
458
+
459
+ a_input = rand_data((size_m, size_k))
460
+ b_weight = rand_data((size_k, size_n))
461
+
462
+ # Quantize activations
463
+ s_a = a_input.abs().max(dim=-1, keepdim=True)[0].div(int8_traits.max).to(
464
+ torch.float)
465
+ q_a = (a_input / s_a).round().clamp(int8_traits.min,
466
+ int8_traits.max).to(torch.int8)
467
+
468
+ # Quantize weights
469
+ w_ref, marlin_qqq_q_w, marlin_qqq_s_group, marlin_qqq_s_channel = \
470
+ marlin_qqq_quantize(b_weight, num_bits, group_size)
471
+
472
+ workspace = MarlinWorkspace(size_n, MARLIN_QQQ_MIN_THREAD_N,
473
+ MARLIN_QQQ_MAX_PARALLEL)
474
+
475
+ output = ops.marlin_qqq_gemm(
476
+ q_a,
477
+ marlin_qqq_q_w,
478
+ s_a,
479
+ marlin_qqq_s_channel,
480
+ marlin_qqq_s_group,
481
+ workspace.scratch,
482
+ a_input.shape[0],
483
+ b_weight.shape[1],
484
+ a_input.shape[1],
485
+ )
486
+ output_ref = torch.matmul(q_a.half() * s_a.half(), w_ref)
487
+
488
+ torch.cuda.synchronize()
489
+
490
+ max_diff = compute_max_diff(output, output_ref)
491
+ print("max_diff = {}".format(max_diff))
492
+
493
+ assert max_diff < 0.04
vllm/_custom_ops.py
+9
@@ -389,6 +389,15 @@ def scaled_int8_quant(
389
return output, input_scales
390
391
392
+# qqq ops
393
+def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
394
+ s_tok: torch.Tensor, s_ch: torch.Tensor,
395
+ s_group: torch.Tensor, workspace: torch.Tensor,
396
+ size_m: int, size_n: int, size_k: int) -> torch.Tensor:
397
+ return torch.ops._C.marlin_qqq_gemm(a, b_q_weight, s_tok, s_ch, s_group,
398
+ workspace, size_m, size_n, size_k)
399
+
400
+
401
# moe
402
def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
403
block_size: int, sorted_token_ids: torch.Tensor,
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
19
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
20
GPTQMarlin24Config)
21
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
22
+from vllm.model_executor.layers.quantization.qqq import QQQConfig
23
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
24
25
QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
@@ -37,6 +38,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
38
"squeezellm": SqueezeLLMConfig,
39
"compressed-tensors": CompressedTensorsConfig,
40
"bitsandbytes": BitsAndBytesConfig,
41
+ "qqq": QQQConfig,
42
}
43
44
vllm/model_executor/layers/quantization/qqq.py
new
+285
@@ -0,0 +1,285 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig)
11
+from vllm.model_executor.utils import set_weight_attrs
12
+
13
+logger = init_logger(__name__)
14
+
15
+MARLIN_QQQ_TILE = 16
16
+MARLIN_QQQ_MIN_THREAD_N = 64
17
+MARLIN_QQQ_MIN_THREAD_K = 128
18
+MARLIN_QQQ_MAX_PARALLEL = 16
19
+
20
+MARLIN_QQQ_SUPPORTED_NUM_BITS = [4]
21
+MARLIN_QQQ_SUPPORTED_GROUP_SIZES = [-1, 128]
22
+MARLIN_QQQ_SUPPORTED_SYM = [True]
23
+
24
+
25
+class QQQConfig(QuantizationConfig):
26
+ """Config class for QQQ
27
+
28
+ Reference: https://arxiv.org/pdf/2406.09904
29
+ """
30
+
31
+ def __init__(
32
+ self,
33
+ weight_bits: int,
34
+ group_size: int,
35
+ is_sym: bool = True,
36
+ ) -> None:
37
+ self.weight_bits = weight_bits
38
+ self.group_size = group_size
39
+ self.is_sym = is_sym
40
+
41
+ # Verify
42
+ if self.weight_bits not in MARLIN_QQQ_SUPPORTED_NUM_BITS:
43
+ raise ValueError(
44
+ f"QQQ does not support weight_bits = {self.weight_bits}. "
45
+ f"Only weight_bits = {MARLIN_QQQ_SUPPORTED_NUM_BITS} "
46
+ "are supported.")
47
+ if self.group_size not in MARLIN_QQQ_SUPPORTED_GROUP_SIZES:
48
+ raise ValueError(
49
+ f"QQQ does not support group_size = {self.group_size}. "
50
+ f"Only group_sizes = {MARLIN_QQQ_SUPPORTED_GROUP_SIZES} "
51
+ "are supported.")
52
+ if self.is_sym not in MARLIN_QQQ_SUPPORTED_SYM:
53
+ raise ValueError(
54
+ f"QQQ does not support is_sym = {self.is_sym}. "
55
+ f"Only sym = {MARLIN_QQQ_SUPPORTED_SYM} are supported.")
56
+
57
+ # 4 Bits packed into 32 bit datatype.
58
+ self.pack_factor = 32 // self.weight_bits
59
+
60
+ # Tile size used by QQQ kernels.
61
+ self.tile_size = MARLIN_QQQ_TILE
62
+
63
+ # Min out_features dim
64
+ self.min_n_threads = MARLIN_QQQ_MIN_THREAD_N
65
+
66
+ # Min in_features dim
67
+ self.min_k_threads = MARLIN_QQQ_MIN_THREAD_K
68
+
69
+ # Max parallel problems to solve at once (improves large
70
+ # batch performance)
71
+ self.max_parallel = MARLIN_QQQ_MAX_PARALLEL
72
+
73
+ # Permutation length used by the QQQ kernels.
74
+ self.perm_len = 1024
75
+
76
+ def __repr__(self) -> str:
77
+ return "QQQConfig(weight_bits={}, group_size={})".format(
78
+ self.weight_bits, self.group_size)
79
+
80
+ @classmethod
81
+ def get_name(cls) -> str:
82
+ return "qqq"
83
+
84
+ @classmethod
85
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
86
+ return [torch.half]
87
+
88
+ @classmethod
89
+ def get_min_capability(cls) -> int:
90
+ return 80
91
+
92
+ @classmethod
93
+ def get_config_filenames(cls) -> List[str]:
94
+ """List of filenames to search for in the model directory."""
95
+ return [
96
+ "quant_config.json",
97
+ "quantize_config.json",
98
+ ]
99
+
100
+ @classmethod
101
+ def from_config(cls, config: Dict[str, Any]) -> "QQQConfig":
102
+ weight_bits = cls.get_from_keys(config, ["wbits"])
103
+ group_size = cls.get_from_keys(config, ["group_size"])
104
+ return cls(weight_bits, group_size)
105
+
106
+ def get_quant_method(self, layer: torch.nn.Module,
107
+ prefix: str) -> Optional["QQQLinearMethod"]:
108
+ if isinstance(layer, LinearBase):
109
+ return QQQLinearMethod(self)
110
+ return None
111
+
112
+ def get_scaled_act_names(self) -> List[str]:
113
+ return []
114
+
115
+
116
+class QQQLinearMethod(LinearMethodBase):
117
+ """Linear method for QQQ.
118
+
119
+ Args:
120
+ quant_config: The QQQ quantization config.
121
+ """
122
+
123
+ def __init__(self, quant_config: QQQConfig):
124
+ self.quant_config = quant_config
125
+
126
+ def create_weights(
127
+ self,
128
+ layer: torch.nn.Module,
129
+ input_size_per_partition: int,
130
+ output_partition_sizes: List[int],
131
+ input_size: int,
132
+ output_size: int,
133
+ params_dtype: torch.dtype,
134
+ **extra_weight_attrs,
135
+ ):
136
+ if params_dtype != torch.float16:
137
+ raise ValueError(
138
+ f"The params dtype must be float16, but got {params_dtype}")
139
+
140
+ # Validate output_size_per_partition
141
+ output_size_per_partition = sum(output_partition_sizes)
142
+ if output_size_per_partition % self.quant_config.min_n_threads != 0:
143
+ raise ValueError(
144
+ f"Weight output_size_per_partition = "
145
+ f"{output_size_per_partition} is not divisible by "
146
+ f"min_n_threads = {self.quant_config.min_n_threads}.")
147
+ if output_size_per_partition % self.quant_config.pack_factor != 0:
148
+ raise ValueError(
149
+ f"Weight output_size_per_partition = "
150
+ f"{output_size_per_partition} is not divisible by "
151
+ f"pack_factor = {self.quant_config.pack_factor}.")
152
+
153
+ # Validate input_size_per_partition
154
+ if input_size_per_partition % self.quant_config.min_k_threads != 0:
155
+ raise ValueError(
156
+ f"Weight input_size_per_partition = "
157
+ f"{input_size_per_partition} is not divisible by "
158
+ f"min_k_threads = {self.quant_config.min_k_threads}.")
159
+ if (self.quant_config.group_size != -1 and
160
+ input_size_per_partition % self.quant_config.group_size != 0):
161
+ raise ValueError(f"Weight input_size_per_partition = "
162
+ f"{input_size_per_partition} is not divisible by "
163
+ f"group_size = {self.quant_config.group_size}.")
164
+
165
+ # Check that we have at least 4 tiles horizontally in the shard
166
+ num_tiles_per_perm = self.quant_config.perm_len // (
167
+ self.quant_config.tile_size**2)
168
+ if output_size_per_partition % num_tiles_per_perm != 0:
169
+ raise ValueError(
170
+ "Each permutation group must reside on the same gpu")
171
+
172
+ # Quantized 4Bit weights packed into Int32.
173
+ qweight = Parameter(
174
+ torch.empty(
175
+ input_size_per_partition // self.quant_config.tile_size,
176
+ output_size_per_partition * self.quant_config.tile_size //
177
+ self.quant_config.pack_factor,
178
+ device="cuda",
179
+ dtype=torch.int32,
180
+ ),
181
+ requires_grad=False,
182
+ )
183
+ set_weight_attrs(
184
+ qweight,
185
+ {
186
+ "input_dim": 0,
187
+ "output_dim": 1,
188
+ "packed_dim": 1,
189
+ "pack_factor": self.quant_config.pack_factor,
190
+ "marlin_tile_size": self.quant_config.tile_size,
191
+ },
192
+ )
193
+
194
+ s_channel = Parameter(
195
+ torch.empty(
196
+ 1,
197
+ output_size_per_partition,
198
+ device="cuda",
199
+ dtype=torch.float,
200
+ ),
201
+ requires_grad=False,
202
+ )
203
+ set_weight_attrs(
204
+ s_channel,
205
+ {
206
+ "input_dim": None,
207
+ "output_dim": 1,
208
+ },
209
+ )
210
+
211
+ if self.quant_config.group_size == -1:
212
+ s_group = Parameter(
213
+ torch.tensor(
214
+ [],
215
+ device="cuda",
216
+ dtype=torch.half,
217
+ ),
218
+ requires_grad=False,
219
+ )
220
+ else:
221
+ s_group = Parameter(
222
+ torch.empty(
223
+ input_size_per_partition // self.quant_config.group_size,
224
+ output_size_per_partition,
225
+ device="cuda",
226
+ dtype=torch.half,
227
+ ),
228
+ requires_grad=False,
229
+ )
230
+
231
+ set_weight_attrs(
232
+ s_group,
233
+ {
234
+ "input_dim": None if self.quant_config.group_size == -1 else 0,
235
+ "output_dim":
236
+ None if self.quant_config.group_size == -1 else 1,
237
+ },
238
+ )
239
+
240
+ # Allocate workspace (Used for internal locking mechanism)
241
+ max_workspace_size = (
242
+ output_size_per_partition //
243
+ self.quant_config.min_n_threads) * self.quant_config.max_parallel
244
+ workspace = Parameter(torch.zeros(max_workspace_size,
245
+ device="cuda",
246
+ dtype=torch.int),
247
+ requires_grad=False)
248
+
249
+ layer.register_parameter("B", qweight)
250
+ set_weight_attrs(qweight, extra_weight_attrs)
251
+ layer.register_parameter("s_channel", s_channel)
252
+ set_weight_attrs(s_channel, extra_weight_attrs)
253
+ layer.register_parameter("s_group", s_group)
254
+ set_weight_attrs(s_group, extra_weight_attrs)
255
+ layer.register_parameter("workspace", workspace)
256
+ set_weight_attrs(workspace, extra_weight_attrs)
257
+
258
+ def apply(
259
+ self,
260
+ layer: torch.nn.Module,
261
+ x: torch.Tensor,
262
+ bias: Optional[torch.Tensor] = None,
263
+ ) -> torch.Tensor:
264
+ qweight = layer.B
265
+ s_ch = layer.s_channel
266
+ s_group = layer.s_group
267
+ workspace = layer.workspace
268
+
269
+ x_2d = x.view(-1, x.shape[-1])
270
+
271
+ size_m = x_2d.shape[0]
272
+ size_k = x_2d.shape[1]
273
+ size_n = s_ch.shape[1]
274
+
275
+ x_int8, s_tok = ops.scaled_int8_quant(x_2d)
276
+
277
+ output_2d = ops.marlin_qqq_gemm(x_int8, qweight, s_tok, s_ch, s_group,
278
+ workspace, size_m, size_n, size_k)
279
+
280
+ output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
281
+
282
+ if bias is not None:
283
+ output.add_(bias) # In-place add
284
+
285
+ return output
vllm/model_executor/layers/quantization/utils/marlin_utils_test_qqq.py
new
+125
@@ -0,0 +1,125 @@
1
+from typing import List
2
+
3
+import numpy
4
+import torch
5
+
6
+from .marlin_utils_test import marlin_permute_weights
7
+from .quant_utils import get_pack_factor, qqq_quantize_weights
8
+
9
+
10
+def marlin_qqq_weights(q_w, size_k, size_n, num_bits, perm, group_size):
11
+ # Permute
12
+ q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
13
+
14
+ # Pack
15
+ pack_factor = get_pack_factor(num_bits)
16
+ orig_device = q_w.device
17
+
18
+ q_w = q_w.cpu().numpy().astype(numpy.uint32)
19
+
20
+ q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
21
+ dtype=numpy.uint32)
22
+ if group_size == size_k:
23
+ for i in range(pack_factor):
24
+ q_packed |= (q_w[:, i::pack_factor] & 0xF) << num_bits * i
25
+ else:
26
+ for i in range(pack_factor):
27
+ q_packed |= q_w[:, i::pack_factor] << num_bits * i
28
+
29
+ q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
30
+
31
+ return q_packed
32
+
33
+
34
+def get_qqq_scale_perms():
35
+ scale_perm: List[int] = []
36
+ for i in range(8):
37
+ scale_perm.extend([i + 8 * j for j in range(8)])
38
+ scale_perm_single: List[int] = []
39
+ for i in range(4):
40
+ scale_perm_single.extend(
41
+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
42
+ return scale_perm, scale_perm_single
43
+
44
+
45
+# NOTE(HandH1998): QQQ employs different perms for per-group and per-channel weight quantization. # noqa: E501
46
+def get_qqq_weight_perm(num_bits: int, quant_type: str):
47
+ perm_list: List[int] = []
48
+ for i in range(32):
49
+ perm1: List[int] = []
50
+ col = i // 4
51
+ for block in [0, 1]:
52
+ for row in [
53
+ 4 * (i % 4),
54
+ 4 * (i % 4) + 1,
55
+ 4 * (i % 4) + 2,
56
+ 4 * (i % 4) + 3,
57
+ ]:
58
+ perm1.append(16 * row + col + 8 * block)
59
+ for j in range(4):
60
+ perm_list.extend([p + 256 * j for p in perm1])
61
+
62
+ perm = numpy.array(perm_list)
63
+
64
+ assert quant_type in ["per-channel",
65
+ "per-group"], "not supported quantization type"
66
+ if num_bits == 4:
67
+ if quant_type == "per-channel":
68
+ interleave = numpy.array([4, 0, 5, 1, 6, 2, 7, 3])
69
+ else:
70
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
71
+ else:
72
+ raise Exception("num_bits must be 4, got {}".format(num_bits))
73
+
74
+ perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
75
+ perm = torch.from_numpy(perm)
76
+ return perm
77
+
78
+
79
+def marlin_qqq_permute_scales(s_group, s_channel, size_k, size_n, group_size):
80
+ scale_perm, scale_perm_single = get_qqq_scale_perms()
81
+ if group_size < size_k and group_size != -1:
82
+ s_group = s_group.reshape((-1, len(scale_perm)))[:, scale_perm]
83
+ s_channel = s_channel.reshape(
84
+ (-1, len(scale_perm_single)))[:, scale_perm_single]
85
+ s_group = s_group.reshape((-1, size_n)).contiguous()
86
+ else:
87
+ s_channel = s_channel.reshape(
88
+ (-1, len(scale_perm_single)))[:, scale_perm_single]
89
+ s_channel = s_channel.reshape((-1, size_n)).contiguous()
90
+
91
+ return s_group, s_channel
92
+
93
+
94
+def marlin_qqq_quantize(
95
+ w: torch.Tensor,
96
+ num_bits: int,
97
+ group_size: int,
98
+):
99
+ size_k, size_n = w.shape
100
+
101
+ # Normalize group_size
102
+ if group_size == -1:
103
+ group_size = size_k
104
+ assert group_size <= size_k
105
+ quant_type = "per-channel" if group_size == size_k else "per-group"
106
+
107
+ # Quantize
108
+ w_ref, q_w, s_group, s_channel = qqq_quantize_weights(
109
+ w, num_bits, group_size)
110
+
111
+ # Reformat to marlin_qqq
112
+ weight_perm = get_qqq_weight_perm(num_bits, quant_type)
113
+ marlin_qqq_q_w = marlin_qqq_weights(q_w, size_k, size_n, num_bits,
114
+ weight_perm, group_size)
115
+ marlin_qqq_s_group, marlin_qqq_s_channel = marlin_qqq_permute_scales(
116
+ s_group, s_channel, size_k, size_n, group_size)
117
+
118
+ # Create result
119
+ res_list = [
120
+ w_ref, marlin_qqq_q_w, marlin_qqq_s_group, marlin_qqq_s_channel
121
+ ]
122
+ for i in range(len(res_list)):
123
+ res_list[i] = res_list[i].to(w.device)
124
+
125
+ return res_list
vllm/model_executor/layers/quantization/utils/quant_utils.py
+82
@@ -205,6 +205,88 @@ def quantize_weights_with_zp(w: torch.Tensor, num_bits: int, group_size: int):
205
)
206
207
208
+# QQQ employs different quant schemes for per-group and
209
+# per-channel quantization.
210
+def qqq_quantize_weights(w: torch.Tensor, num_bits: int, group_size: int):
211
+ orig_device = w.device
212
+ size_k, size_n = w.shape
213
+
214
+ assert w.is_floating_point(), "w must be float"
215
+ assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
216
+ assert group_size in SUPPORTED_GROUP_SIZES + [
217
+ size_k
218
+ ], f"Unsupported groupsize = {group_size}"
219
+
220
+ if group_size == -1:
221
+ group_size = size_k
222
+ assert group_size <= size_k
223
+
224
+ if group_size < size_k:
225
+ # Reshape to [groupsize, -1]
226
+ w = w.reshape((-1, group_size, size_n))
227
+ w = w.permute(1, 0, 2)
228
+ w = w.reshape((group_size, -1))
229
+
230
+ max_q_val = 2**num_bits - 1
231
+ half_q_val = (max_q_val + 1) // 2
232
+
233
+ # Compute scale for each group
234
+ s_group = torch.max(torch.abs(w), 0, keepdim=True)[0]
235
+ s_group *= 2 / max_q_val # 2 => symmetric
236
+
237
+ # Quantize
238
+ q_w = torch.round(w / s_group).int()
239
+ q_w += half_q_val
240
+ q_w = torch.clamp(q_w, 0, max_q_val)
241
+ # Compute ref (dequantized)
242
+ w_ref = (q_w - half_q_val).half() * s_group
243
+
244
+ # Restore original shapes
245
+ def reshape_w(w):
246
+ w = w.reshape((group_size, -1, size_n))
247
+ w = w.permute(1, 0, 2)
248
+ w = w.reshape((size_k, size_n)).contiguous()
249
+ return w
250
+
251
+ q_w = reshape_w(q_w)
252
+ w_ref = reshape_w(w_ref)
253
+
254
+ # Compute int8 quantization scale for each channel
255
+ s_channel = torch.max(torch.abs(w_ref), 0, keepdim=True)[0]
256
+ s_channel /= 127.0
257
+ t_int8 = (w_ref / s_channel).round().clamp(-128, 127).to(torch.int8)
258
+ w_ref = t_int8.half() * s_channel
259
+ s_channel = s_channel.reshape(1, -1).to(dtype=torch.float)
260
+
261
+ # Fuse scales
262
+ s_group = (s_group.reshape(-1, size_n).contiguous() /
263
+ s_channel).to(dtype=torch.half)
264
+ else:
265
+ max_q_val = 2**(num_bits - 1) - 1
266
+
267
+ # Compute scale for each channel
268
+ s_channel = torch.max(torch.abs(w), 0, keepdim=True)[0]
269
+ s_channel /= max_q_val
270
+
271
+ # Quantize
272
+ q_w = torch.round(w / s_channel).int()
273
+ q_w = torch.clamp(q_w, -max_q_val, max_q_val)
274
+ # Compute ref (dequantized)
275
+ w_ref = q_w.half() * s_channel
276
+
277
+ s_group = torch.tensor([], dtype=torch.half)
278
+ # div 2 ** (8 - self.bits)) to offset right shift in unpacking
279
+ s_channel /= (2**(8 - num_bits))
280
+ s_channel = s_channel.reshape(-1, size_n).contiguous().to(torch.float)
281
+
282
+ return (
283
+ w_ref.to(device=orig_device),
284
+ q_w.to(device=orig_device),
285
+ s_group.to(device=orig_device),
286
+ s_channel.to(device=orig_device),
287
+ )
288
+
289
+
290
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
291
orig_device = q_w.device
292