A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Add GPTQ Marlin 2:4 sparse structured support (#4790)
Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
Alexander Matveev committed
May 16, 2024 at 12:56 UTC
6979ade3840703d055402e78168194644e3cbdd8
18 files changed
+2131
-41
CMakeLists.txt
+2
-1
@@ -176,7 +176,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
176
list(APPEND VLLM_EXT_SRC
177
"csrc/quantization/aqlm/gemm_kernels.cu"
178
"csrc/quantization/awq/gemm_kernels.cu"
179
- "csrc/quantization/marlin/marlin_cuda_kernel.cu"
179
+ "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
180
+ "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
181
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
182
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
183
"csrc/custom_all_reduce.cu")
csrc/ops.h
+11
@@ -125,6 +125,17 @@ torch::Tensor marlin_gemm(
125
int64_t size_n,
126
int64_t size_k);
127
128
+torch::Tensor gptq_marlin_24_gemm(
129
+ torch::Tensor &a,
130
+ torch::Tensor &b_q_weight,
131
+ torch::Tensor &b_meta,
132
+ torch::Tensor &b_scales,
133
+ torch::Tensor &workspace,
134
+ int64_t num_bits,
135
+ int64_t size_m,
136
+ int64_t size_n,
137
+ int64_t size_k);
138
+
139
torch::Tensor gptq_marlin_gemm(
140
torch::Tensor &a,
141
torch::Tensor &b_q_weight,
csrc/pybind.cpp
+2
-1
@@ -66,7 +66,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
66
ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
67
ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
68
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69
- ops.def("marlin_gemm", &marlin_gemm, "Marlin Optimized Quantized GEMM for GPTQ");
69
+ ops.def("marlin_gemm", &marlin_gemm, "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
70
+ ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm, "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
71
ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
72
ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
73
ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
csrc/quantization/marlin/dense/LICENSE
renamed
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
renamed
csrc/quantization/marlin/sparse/LICENSE
new
+203
@@ -0,0 +1,203 @@
1
+Contains code from https://github.com/IST-DASLab/Sparse-Marlin/
2
+
3
+ Apache License
4
+ Version 2.0, January 2004
5
+ http://www.apache.org/licenses/
6
+
7
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8
+
9
+ 1. Definitions.
10
+
11
+ "License" shall mean the terms and conditions for use, reproduction,
12
+ and distribution as defined by Sections 1 through 9 of this document.
13
+
14
+ "Licensor" shall mean the copyright owner or entity authorized by
15
+ the copyright owner that is granting the License.
16
+
17
+ "Legal Entity" shall mean the union of the acting entity and all
18
+ other entities that control, are controlled by, or are under common
19
+ control with that entity. For the purposes of this definition,
20
+ "control" means (i) the power, direct or indirect, to cause the
21
+ direction or management of such entity, whether by contract or
22
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
23
+ outstanding shares, or (iii) beneficial ownership of such entity.
24
+
25
+ "You" (or "Your") shall mean an individual or Legal Entity
26
+ exercising permissions granted by this License.
27
+
28
+ "Source" form shall mean the preferred form for making modifications,
29
+ including but not limited to software source code, documentation
30
+ source, and configuration files.
31
+
32
+ "Object" form shall mean any form resulting from mechanical
33
+ transformation or translation of a Source form, including but
34
+ not limited to compiled object code, generated documentation,
35
+ and conversions to other media types.
36
+
37
+ "Work" shall mean the work of authorship, whether in Source or
38
+ Object form, made available under the License, as indicated by a
39
+ copyright notice that is included in or attached to the work
40
+ (an example is provided in the Appendix below).
41
+
42
+ "Derivative Works" shall mean any work, whether in Source or Object
43
+ form, that is based on (or derived from) the Work and for which the
44
+ editorial revisions, annotations, elaborations, or other modifications
45
+ represent, as a whole, an original work of authorship. For the purposes
46
+ of this License, Derivative Works shall not include works that remain
47
+ separable from, or merely link (or bind by name) to the interfaces of,
48
+ the Work and Derivative Works thereof.
49
+
50
+ "Contribution" shall mean any work of authorship, including
51
+ the original version of the Work and any modifications or additions
52
+ to that Work or Derivative Works thereof, that is intentionally
53
+ submitted to Licensor for inclusion in the Work by the copyright owner
54
+ or by an individual or Legal Entity authorized to submit on behalf of
55
+ the copyright owner. For the purposes of this definition, "submitted"
56
+ means any form of electronic, verbal, or written communication sent
57
+ to the Licensor or its representatives, including but not limited to
58
+ communication on electronic mailing lists, source code control systems,
59
+ and issue tracking systems that are managed by, or on behalf of, the
60
+ Licensor for the purpose of discussing and improving the Work, but
61
+ excluding communication that is conspicuously marked or otherwise
62
+ designated in writing by the copyright owner as "Not a Contribution."
63
+
64
+ "Contributor" shall mean Licensor and any individual or Legal Entity
65
+ on behalf of whom a Contribution has been received by Licensor and
66
+ subsequently incorporated within the Work.
67
+
68
+ 2. Grant of Copyright License. Subject to the terms and conditions of
69
+ this License, each Contributor hereby grants to You a perpetual,
70
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
71
+ copyright license to reproduce, prepare Derivative Works of,
72
+ publicly display, publicly perform, sublicense, and distribute the
73
+ Work and such Derivative Works in Source or Object form.
74
+
75
+ 3. Grant of Patent License. Subject to the terms and conditions of
76
+ this License, each Contributor hereby grants to You a perpetual,
77
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78
+ (except as stated in this section) patent license to make, have made,
79
+ use, offer to sell, sell, import, and otherwise transfer the Work,
80
+ where such license applies only to those patent claims licensable
81
+ by such Contributor that are necessarily infringed by their
82
+ Contribution(s) alone or by combination of their Contribution(s)
83
+ with the Work to which such Contribution(s) was submitted. If You
84
+ institute patent litigation against any entity (including a
85
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
86
+ or a Contribution incorporated within the Work constitutes direct
87
+ or contributory patent infringement, then any patent licenses
88
+ granted to You under this License for that Work shall terminate
89
+ as of the date such litigation is filed.
90
+
91
+ 4. Redistribution. You may reproduce and distribute copies of the
92
+ Work or Derivative Works thereof in any medium, with or without
93
+ modifications, and in Source or Object form, provided that You
94
+ meet the following conditions:
95
+
96
+ (a) You must give any other recipients of the Work or
97
+ Derivative Works a copy of this License; and
98
+
99
+ (b) You must cause any modified files to carry prominent notices
100
+ stating that You changed the files; and
101
+
102
+ (c) You must retain, in the Source form of any Derivative Works
103
+ that You distribute, all copyright, patent, trademark, and
104
+ attribution notices from the Source form of the Work,
105
+ excluding those notices that do not pertain to any part of
106
+ the Derivative Works; and
107
+
108
+ (d) If the Work includes a "NOTICE" text file as part of its
109
+ distribution, then any Derivative Works that You distribute must
110
+ include a readable copy of the attribution notices contained
111
+ within such NOTICE file, excluding those notices that do not
112
+ pertain to any part of the Derivative Works, in at least one
113
+ of the following places: within a NOTICE text file distributed
114
+ as part of the Derivative Works; within the Source form or
115
+ documentation, if provided along with the Derivative Works; or,
116
+ within a display generated by the Derivative Works, if and
117
+ wherever such third-party notices normally appear. The contents
118
+ of the NOTICE file are for informational purposes only and
119
+ do not modify the License. You may add Your own attribution
120
+ notices within Derivative Works that You distribute, alongside
121
+ or as an addendum to the NOTICE text from the Work, provided
122
+ that such additional attribution notices cannot be construed
123
+ as modifying the License.
124
+
125
+ You may add Your own copyright statement to Your modifications and
126
+ may provide additional or different license terms and conditions
127
+ for use, reproduction, or distribution of Your modifications, or
128
+ for any such Derivative Works as a whole, provided Your use,
129
+ reproduction, and distribution of the Work otherwise complies with
130
+ the conditions stated in this License.
131
+
132
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
133
+ any Contribution intentionally submitted for inclusion in the Work
134
+ by You to the Licensor shall be under the terms and conditions of
135
+ this License, without any additional terms or conditions.
136
+ Notwithstanding the above, nothing herein shall supersede or modify
137
+ the terms of any separate license agreement you may have executed
138
+ with Licensor regarding such Contributions.
139
+
140
+ 6. Trademarks. This License does not grant permission to use the trade
141
+ names, trademarks, service marks, or product names of the Licensor,
142
+ except as required for reasonable and customary use in describing the
143
+ origin of the Work and reproducing the content of the NOTICE file.
144
+
145
+ 7. Disclaimer of Warranty. Unless required by applicable law or
146
+ agreed to in writing, Licensor provides the Work (and each
147
+ Contributor provides its Contributions) on an "AS IS" BASIS,
148
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
149
+ implied, including, without limitation, any warranties or conditions
150
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
151
+ PARTICULAR PURPOSE. You are solely responsible for determining the
152
+ appropriateness of using or redistributing the Work and assume any
153
+ risks associated with Your exercise of permissions under this License.
154
+
155
+ 8. Limitation of Liability. In no event and under no legal theory,
156
+ whether in tort (including negligence), contract, or otherwise,
157
+ unless required by applicable law (such as deliberate and grossly
158
+ negligent acts) or agreed to in writing, shall any Contributor be
159
+ liable to You for damages, including any direct, indirect, special,
160
+ incidental, or consequential damages of any character arising as a
161
+ result of this License or out of the use or inability to use the
162
+ Work (including but not limited to damages for loss of goodwill,
163
+ work stoppage, computer failure or malfunction, or any and all
164
+ other commercial damages or losses), even if such Contributor
165
+ has been advised of the possibility of such damages.
166
+
167
+ 9. Accepting Warranty or Additional Liability. While redistributing
168
+ the Work or Derivative Works thereof, You may choose to offer,
169
+ and charge a fee for, acceptance of support, warranty, indemnity,
170
+ or other liability obligations and/or rights consistent with this
171
+ License. However, in accepting such obligations, You may act only
172
+ on Your own behalf and on Your sole responsibility, not on behalf
173
+ of any other Contributor, and only if You agree to indemnify,
174
+ defend, and hold each Contributor harmless for any liability
175
+ incurred by, or claims asserted against, such Contributor by reason
176
+ of your accepting any such warranty or additional liability.
177
+
178
+ END OF TERMS AND CONDITIONS
179
+
180
+ APPENDIX: How to apply the Apache License to your work.
181
+
182
+ To apply the Apache License to your work, attach the following
183
+ boilerplate notice, with the fields enclosed by brackets "[]"
184
+ replaced with your own identifying information. (Don't include
185
+ the brackets!) The text should be enclosed in the appropriate
186
+ comment syntax for the file format. We also recommend that a
187
+ file or class name and description of purpose be included on the
188
+ same "printed page" as the copyright notice for easier
189
+ identification within third-party archives.
190
+
191
+ Copyright [yyyy] [name of copyright owner]
192
+
193
+ Licensed under the Apache License, Version 2.0 (the "License");
194
+ you may not use this file except in compliance with the License.
195
+ You may obtain a copy of the License at
196
+
197
+ http://www.apache.org/licenses/LICENSE-2.0
198
+
199
+ Unless required by applicable law or agreed to in writing, software
200
+ distributed under the License is distributed on an "AS IS" BASIS,
201
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
202
+ See the License for the specific language governing permissions and
203
+ limitations under the License.
\ No newline at end of file
csrc/quantization/marlin/sparse/common/base.h
new
+49
@@ -0,0 +1,49 @@
1
+/*
2
+ * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
3
+ * Rights Reserved.
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * http://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+
18
+#pragma once
19
+
20
+namespace marlin_24 {
21
+
22
+constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
23
+
24
+// Instances of `Vec` are used to organize groups of >>registers<<, as needed
25
+// for instance as inputs to tensor core operations. Consequently, all
26
+// corresponding index accesses must be compile-time constants, which is why we
27
+// extensively use `#pragma unroll` throughout the kernel code to guarantee
28
+// this.
29
+template <typename T, int n> struct Vec {
30
+ T elems[n];
31
+ __device__ T &operator[](int i) { return elems[i]; }
32
+};
33
+
34
+template <int M_, int N_, int K_> struct ShapeBase {
35
+ static constexpr int M = M_, N = N_, K = K_;
36
+};
37
+
38
+using I4 = Vec<int, 4>;
39
+
40
+// Matrix fragments for tensor core instructions; their precise layout is
41
+// documented here:
42
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
43
+using FragA = Vec<half2, 4>;
44
+using FragB = Vec<half2, 2>;
45
+using FragM = Vec<uint, 1>;
46
+using FragC = Vec<float, 4>;
47
+using FragS = Vec<half2, 1>; // quantization scales
48
+
49
+} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mem.h
new
+132
@@ -0,0 +1,132 @@
1
+/*
2
+ * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
3
+ * Rights Reserved.
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * http://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+
18
+#pragma once
19
+#include "base.h"
20
+
21
+namespace marlin_24 {
22
+// Predicated asynchronous global->shared copy; used for inputs A where we apply
23
+// predication to handle batchsizes that are not multiples of 16.
24
+__device__ inline void cp_async4_pred_zfill(void *smem_ptr,
25
+ const void *glob_ptr,
26
+ bool pred = true,
27
+ const bool zfill = false) {
28
+ const int BYTES = 16;
29
+ int src_in_bytes = (zfill ? 0 : BYTES);
30
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
31
+ asm volatile("{\n"
32
+ " .reg .pred p;\n"
33
+ " setp.ne.b32 p, %0, 0;\n"
34
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
35
+ "}\n" ::"r"((int)pred),
36
+ "r"(smem), "l"(glob_ptr), "n"(BYTES), "r"(src_in_bytes));
37
+}
38
+
39
+__device__ inline void cp_async4_pred(void *smem_ptr, const void *glob_ptr,
40
+ bool pred = true) {
41
+ const int BYTES = 16;
42
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
43
+ asm volatile("{\n"
44
+ " .reg .pred p;\n"
45
+ " setp.ne.b32 p, %0, 0;\n"
46
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
47
+ "}\n" ::"r"((int)pred),
48
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
49
+}
50
+
51
+// Asynchronous global->shared copy
52
+__device__ inline void cp_async4(void *smem_ptr, const void *glob_ptr) {
53
+ const int BYTES = 16;
54
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55
+ asm volatile("{\n"
56
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
57
+ "}\n" ::"r"(smem),
58
+ "l"(glob_ptr), "n"(BYTES));
59
+}
60
+
61
+// Async copy fence.
62
+__device__ inline void cp_async_fence() {
63
+ asm volatile("cp.async.commit_group;\n" ::);
64
+}
65
+
66
+// Wait until at most `n` async copy stages are still pending.
67
+template <int n> __device__ inline void cp_async_wait() {
68
+ asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
69
+}
70
+
71
+// Instruction for loading a full 16x16 matrix fragment of operand A from shared
72
+// memory, directly in tensor core layout.
73
+__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
74
+ uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
75
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
76
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
77
+ : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
78
+ : "r"(smem));
79
+}
80
+
81
+__device__ inline void ldsm4_m(FragM &frag_m, const void *smem_ptr) {
82
+ uint32_t *a = reinterpret_cast<uint32_t *>(&frag_m);
83
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
84
+ asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n"
85
+ : "=r"(a[0]), "=r"(a[1])
86
+ : "r"(smem));
87
+}
88
+
89
+// Instruction for loading a full 16x16 matrix fragment of operand A from shared
90
+// memory, directly in tensor core layout.
91
+__device__ inline void ldsm4_t(FragA &frag_a, const void *smem_ptr) {
92
+ uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
93
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
94
+ asm volatile(
95
+ "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
96
+ : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
97
+ : "r"(smem));
98
+}
99
+
100
+// Wait until barrier reaches `count`, then lock for current threadblock.
101
+__device__ inline void barrier_acquire(int *lock, int count) {
102
+ if (threadIdx.x == 0) {
103
+ int state = -1;
104
+ do
105
+ // Guarantee that subsequent writes by this threadblock will be visible
106
+ // globally.
107
+ asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
108
+ : "=r"(state)
109
+ : "l"(lock));
110
+ while (state != count);
111
+ }
112
+ __syncthreads();
113
+}
114
+
115
+// Release barrier and increment visitation count.
116
+__device__ inline void barrier_release(int *lock, bool reset = false) {
117
+ __syncthreads();
118
+ if (threadIdx.x == 0) {
119
+ if (reset) {
120
+ lock[0] = 0;
121
+ return;
122
+ }
123
+ int val = 1;
124
+ // Make sure that all writes since acquiring this barrier are visible
125
+ // globally, while releasing the barrier.
126
+ asm volatile("fence.acq_rel.gpu;\n");
127
+ asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
128
+ :
129
+ : "l"(lock), "r"(val));
130
+ }
131
+}
132
+} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mma.h
new
+175
@@ -0,0 +1,175 @@
1
+/*
2
+ * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
3
+ * Rights Reserved.
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * http://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+
18
+#pragma once
19
+#include "base.h"
20
+
21
+namespace marlin_24 {
22
+
23
+// m16n8k32 sparse tensor core mma instruction with fp16 inputs and fp32
24
+// output/accumulation.
25
+__device__ inline void mma_sp(const FragB &a_frag0, const FragB &a_frag1,
26
+ const FragA &frag_b, FragC &frag_c, FragM &frag_m,
27
+ const int psel) {
28
+ const uint32_t *a0 = reinterpret_cast<const uint32_t *>(&a_frag0);
29
+ const uint32_t *a1 = reinterpret_cast<const uint32_t *>(&a_frag1);
30
+ const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
31
+ const uint32_t *e = reinterpret_cast<const uint32_t *>(&frag_m);
32
+ float *c = reinterpret_cast<float *>(&frag_c);
33
+ if (psel == 0) {
34
+ asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
35
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
36
+ "{%12,%13,%14,%15}, %16, 0x0;\n"
37
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
38
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
39
+ "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
40
+ "f"(c[2]), "f"(c[3]), "r"(e[0]));
41
+ asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
42
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
43
+ "{%12,%13,%14,%15}, %16, 0x0;\n"
44
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
45
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
46
+ "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
47
+ "f"(c[6]), "f"(c[7]), "r"(e[0]));
48
+ } else {
49
+ asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
50
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
51
+ "{%12,%13,%14,%15}, %16, 0x1;\n"
52
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
53
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
54
+ "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
55
+ "f"(c[2]), "f"(c[3]), "r"(e[0]));
56
+ asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
57
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
58
+ "{%12,%13,%14,%15}, %16, 0x1;\n"
59
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
60
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
61
+ "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
62
+ "f"(c[6]), "f"(c[7]), "r"(e[0]));
63
+ }
64
+}
65
+
66
+// Lookup-table based 3-input logical operation; explicitly used for
67
+// dequantization as the compiler does not seem to automatically recognize it in
68
+// all cases.
69
+template <int lut> __device__ inline int lop3(int a, int b, int c) {
70
+ int res;
71
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
72
+ : "=r"(res)
73
+ : "r"(a), "r"(b), "r"(c), "n"(lut));
74
+ return res;
75
+}
76
+
77
+__device__ __forceinline__ uint2 to_half4(float c0, float c1, float c2,
78
+ float c3) {
79
+ uint2 r;
80
+ asm("{\n\t"
81
+ ".reg .f16 a, b, c, d; \n\t"
82
+ "cvt.rn.f16.f32 a, %2; \n\t"
83
+ "cvt.rn.f16.f32 b, %3; \n\t"
84
+ "cvt.rn.f16.f32 c, %4; \n\t"
85
+ "cvt.rn.f16.f32 d, %5; \n\t"
86
+ "mov.b32 %0, {a, b}; \n\t"
87
+ "mov.b32 %1, {c, d}; \n\t"
88
+ "}"
89
+ : "=r"(r.x), "=r"(r.y)
90
+ : "f"(c0), "f"(c1), "f"(c2), "f"(c3));
91
+ return r;
92
+}
93
+
94
+// Constructs destination register by taking bytes from 2 sources (based on
95
+// mask)
96
+template <int start_byte, int mask>
97
+__device__ inline uint32_t prmt(uint32_t a) {
98
+ uint32_t res;
99
+ asm volatile("prmt.b32 %0, %1, %2, %3;\n"
100
+ : "=r"(res)
101
+ : "r"(a), "n"(start_byte), "n"(mask));
102
+ return res;
103
+}
104
+
105
+// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
106
+// values. We mostly follow the strategy in the link below, with some small
107
+// changes:
108
+// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
109
+__device__ inline FragB dequant_4bit(int q) {
110
+ const int LO = 0x000f000f;
111
+ const int HI = 0x00f000f0;
112
+ const int EX = 0x64006400;
113
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
114
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
115
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
116
+ // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
117
+ // directly into `SUB` and `ADD`.
118
+ const int SUB = 0x64086408;
119
+ const int MUL = 0x2c002c00;
120
+ const int ADD = 0xd480d480;
121
+
122
+ FragB frag_b;
123
+ frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
124
+ *reinterpret_cast<const half2 *>(&SUB));
125
+ frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
126
+ *reinterpret_cast<const half2 *>(&MUL),
127
+ *reinterpret_cast<const half2 *>(&ADD));
128
+ return frag_b;
129
+}
130
+
131
+// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
132
+// values. We mostly follow the strategy in the link below, with some small
133
+// changes:
134
+// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
135
+__device__ inline FragB dequant_8bit(int q) {
136
+ static constexpr uint32_t mask_for_elt_01 = 0x5250;
137
+ static constexpr uint32_t mask_for_elt_23 = 0x5351;
138
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
139
+
140
+ uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
141
+ uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
142
+
143
+ static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
144
+
145
+ FragB frag_b;
146
+ frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
147
+ *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
148
+ frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
149
+ *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
150
+ return frag_b;
151
+}
152
+
153
+// Multiply dequantized values by the corresponding quantization scale; used
154
+// only for grouped quantization.
155
+__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
156
+ half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
157
+ frag_b[0] = __hmul2(frag_b[0], s);
158
+ frag_b[1] = __hmul2(frag_b[1], s);
159
+}
160
+
161
+__device__ inline void scale_floats(float *c0, float *c1, float *c2, float *c3,
162
+ FragS &s0, float *c4, float *c5, float *c6,
163
+ float *c7, FragS &s1) {
164
+ *c0 = __fmul_rn(*c0, __half2float(s0[0].x));
165
+ *c1 = __fmul_rn(*c1, __half2float(s0[0].y));
166
+ *c2 = __fmul_rn(*c2, __half2float(s0[1].x));
167
+ *c3 = __fmul_rn(*c3, __half2float(s0[1].y));
168
+
169
+ *c4 = __fmul_rn(*c4, __half2float(s1[0].x));
170
+ *c5 = __fmul_rn(*c5, __half2float(s1[0].y));
171
+ *c6 = __fmul_rn(*c6, __half2float(s1[1].x));
172
+ *c7 = __fmul_rn(*c7, __half2float(s1[1].y));
173
+}
174
+
175
+} // namespace marlin_24
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
new
+1110
@@ -0,0 +1,1110 @@
1
+/*
2
+ * Notice: This file was modified by Neuralmagic inc to include 8-bit support
3
+ *
4
+ * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
5
+ * Rights Reserved.
6
+ *
7
+ * Licensed under the Apache License, Version 2.0 (the "License");
8
+ * you may not use this file except in compliance with the License.
9
+ * You may obtain a copy of the License at
10
+ *
11
+ * http://www.apache.org/licenses/LICENSE-2.0
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+#include <torch/extension.h>
20
+
21
+#include <ATen/cuda/CUDAContext.h>
22
+#include <c10/cuda/CUDAGuard.h>
23
+#include <cuda.h>
24
+#include <cuda_fp16.h>
25
+#include <cuda_runtime.h>
26
+
27
+#include <iostream>
28
+
29
+#include "common/base.h"
30
+
31
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
32
+
33
+#else
34
+
35
+#include "common/mem.h"
36
+#include "common/mma.h"
37
+
38
+#endif
39
+
40
+template <typename T> inline std::string str(T x) { return std::to_string(x); }
41
+
42
+namespace marlin_24 {
43
+
44
+// 8 warps are a good choice since every SM has 4 schedulers and having more
45
+// than 1 warp per schedule allows some more latency hiding. At the same time,
46
+// we want relatively few warps to have many registers per warp and small tiles.
47
+static constexpr int THREADS = 256;
48
+static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
49
+
50
+static constexpr int min_thread_n = 128;
51
+
52
+static constexpr int tile_size = 16;
53
+static constexpr int max_par = 16;
54
+
55
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
56
+
57
+template <const int num_bits, // weight bits
58
+ const int threads, // number of threads in a threadblock
59
+ const int thread_m_blocks, // number of 16x16 blocks in the m
60
+ // dimension (batchsize) of the threadblock
61
+ const int thread_n_blocks, // same for n dimension (output)
62
+ const int thread_k_blocks, // same for k dimension (reduction)
63
+ const int stages, // number of stages for the async global->shared
64
+ // fetch pipeline
65
+ const int group_blocks = -1 // number of consecutive 16x16 blocks with
66
+ // a separate quantization scale
67
+ >
68
+__global__ void Marlin_24(
69
+ const int4 *__restrict__ A, // fp16 input matrix of shape mxk
70
+ const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
71
+ const int4
72
+ *__restrict__ meta, // 2bit metadata information about 2:4 format on B
73
+ int4 *__restrict__ C, // fp16 output buffer of shape mxn
74
+ const int4
75
+ *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
76
+ int prob_m, // batch dimension m
77
+ int prob_n, // output dimension n
78
+ int prob_k, // reduction dimension k
79
+ int *locks // extra global storage for barrier synchronization
80
+) {}
81
+
82
+torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
83
+ torch::Tensor &b_meta,
84
+ torch::Tensor &b_scales,
85
+ torch::Tensor &workspace, int64_t num_bits,
86
+ int64_t size_m, int64_t size_n,
87
+ int64_t size_k) {
88
+ TORCH_CHECK_NOT_IMPLEMENTED(
89
+ false, "gptq_marlin_24_gemm(..) requires CUDA_ARCH >= 8.0");
90
+ return torch::empty({1, 1});
91
+}
92
+
93
+#else
94
+
95
+template <const int num_bits, // weight bits
96
+ const int threads, // number of threads in a threadblock
97
+ const int thread_m_blocks, // number of 16x16 blocks in the m
98
+ // dimension (batchsize) of the threadblock
99
+ const int thread_n_blocks, // same for n dimension (output)
100
+ const int thread_k_blocks, // same for k dimension (reduction)
101
+ const int stages, // number of stages for the async global->shared
102
+ // fetch pipeline
103
+ const int group_blocks = -1 // number of consecutive 16x16 blocks with
104
+ // a separate quantization scale
105
+ >
106
+__global__ void Marlin_24(
107
+ const int4 *__restrict__ A, // fp16 input matrix of shape mxk
108
+ const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
109
+ const int4
110
+ *__restrict__ meta, // 2bit metadata information about 2:4 format on B
111
+ int4 *__restrict__ C, // fp16 output buffer of shape mxn
112
+ const int4
113
+ *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
114
+ int prob_m, // batch dimension m
115
+ int prob_n, // output dimension n
116
+ int prob_k, // reduction dimension k
117
+ int *locks // extra global storage for barrier synchronization
118
+) {
119
+ // Each threadblock processes one "stripe" of the B matrix with (roughly) the
120
+ // same size, which might involve multiple column "slices" (of width 16 *
121
+ // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM
122
+ // example:
123
+ // 0 1 3
124
+ // 0 2 3
125
+ // 1 2 4
126
+ // While this kind of partitioning makes things somewhat more complicated, it
127
+ // ensures good utilization of all SMs for many kinds of shape and GPU
128
+ // configurations, while requiring as few slow global cross-threadblock
129
+ // reductions as possible.
130
+
131
+ // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
132
+ // better partitioning with less reductions
133
+ int parallel = 1;
134
+ if (prob_m > 16 * thread_m_blocks) {
135
+ parallel = prob_m / (16 * thread_m_blocks);
136
+ prob_m = 16 * thread_m_blocks;
137
+ }
138
+
139
+ // number of thread_k_blocks in k-dim
140
+ int k_tiles = prob_k / 32 / thread_k_blocks;
141
+ // number of thread_n_blocks in n-dim
142
+ int n_tiles = prob_n / 16 / thread_n_blocks;
143
+ // iters needed to cover all slices
144
+ int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
145
+
146
+ // Ensure that the number of tiles in each stripe is a multiple of the
147
+ // groupsize; this avoids an annoying special case where a stripe starts in
148
+ // the middle of group.
149
+ if (group_blocks != -1)
150
+ iters = (group_blocks / thread_k_blocks) *
151
+ ceildiv(iters, (group_blocks / thread_k_blocks));
152
+
153
+ int slice_row = (iters * blockIdx.x) % k_tiles;
154
+ int slice_col_par = (iters * blockIdx.x) / k_tiles;
155
+ int slice_col = slice_col_par;
156
+ // number of threadblock tiles in the current slice
157
+ int slice_iters;
158
+ // total number of active threadblocks in the current slice
159
+ int slice_count = 0;
160
+ // index of threadblock in current slice; numbered bottom to top
161
+ int slice_idx;
162
+
163
+ // We can easily implement parallel problem execution by just remapping
164
+ // indices and advancing global pointers
165
+ if (slice_col_par >= n_tiles) {
166
+ A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_k / 8;
167
+ C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
168
+ locks += (slice_col_par / n_tiles) * n_tiles;
169
+ slice_col = slice_col_par % n_tiles;
170
+ }
171
+
172
+ // Compute all information about the current slice which is required for
173
+ // synchronization.
174
+ auto init_slice = [&]() {
175
+ slice_iters =
176
+ iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
177
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
178
+ slice_iters = 0;
179
+ if (slice_iters == 0)
180
+ return;
181
+ if (slice_row + slice_iters > k_tiles)
182
+ slice_iters = k_tiles - slice_row;
183
+ slice_count = 1;
184
+ slice_idx = 0;
185
+ int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
186
+ if (col_first <= k_tiles * (slice_col_par + 1)) {
187
+ int col_off = col_first - k_tiles * slice_col_par;
188
+ slice_count = ceildiv(k_tiles - col_off, iters);
189
+ if (col_off > 0)
190
+ slice_count++;
191
+ int delta_first = iters * blockIdx.x - col_first;
192
+ if (delta_first < 0 || (col_off == 0 && delta_first == 0))
193
+ slice_idx = slice_count - 1;
194
+ else {
195
+ slice_idx = slice_count - 1 - delta_first / iters;
196
+ if (col_off > 0)
197
+ slice_idx--;
198
+ }
199
+ }
200
+ if (slice_col == n_tiles) {
201
+ A += 16 * thread_m_blocks * prob_k / 8;
202
+ C += 16 * thread_m_blocks * prob_n / 8;
203
+ locks += n_tiles;
204
+ slice_col = 0;
205
+ }
206
+ };
207
+ init_slice();
208
+
209
+ // RLC: 8 is vec_size -> 128-bit instructions, 8 fp16 elements
210
+ int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
211
+
212
+ // stride of an A matrix tile in shared memory
213
+ constexpr int a_sh_stride = 32 * thread_k_blocks / 8;
214
+ // delta between subsequent A tiles in global memory
215
+ constexpr int a_gl_rd_delta_o = 32 * thread_k_blocks / 8;
216
+ // between subsequent accesses within a tile
217
+ int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
218
+ // between shared memory writes
219
+ constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
220
+ // between shared memory tile reads //RLC: 2 * #warps k-dim
221
+ constexpr int a_sh_rd_delta_o = 4 * ((threads / 32) / (thread_n_blocks / 4));
222
+ // within a shared memory tile
223
+ constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
224
+ // overall size of a tile
225
+ constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
226
+ // number of shared write iterations for a tile
227
+ constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
228
+
229
+ constexpr int pack_factor = 32 / num_bits;
230
+
231
+ int b_gl_stride = 16 * prob_n / (pack_factor * 4);
232
+ constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
233
+ constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
234
+ constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
235
+ int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
236
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
237
+ constexpr int b_sh_wr_delta = threads * b_thread_vecs;
238
+ constexpr int b_sh_rd_delta = threads * b_thread_vecs;
239
+ constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
240
+ constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
241
+
242
+ int m_gl_stride = 2 * prob_n / 8; // (16*2*4 / 8) = 16
243
+ constexpr int m_sh_stride =
244
+ (16 * thread_n_blocks) / 4; // #warps n-dim * threads/warp
245
+ int m_gl_rd_delta_o = m_gl_stride * thread_k_blocks;
246
+ int m_gl_rd_delta_i = m_gl_stride * (threads / m_sh_stride);
247
+ constexpr int m_sh_wr_delta = threads / 2;
248
+ constexpr int m_sh_rd_delta = threads / 2;
249
+ constexpr int m_sh_stage = m_sh_stride * thread_k_blocks;
250
+ constexpr int m_sh_iters = ceildiv(m_sh_stage, m_sh_wr_delta);
251
+
252
+ int s_gl_stride = prob_n / 8;
253
+ constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
254
+ constexpr int s_sh_stage = s_sh_stride;
255
+ int s_gl_rd_delta = s_gl_stride;
256
+
257
+ // Global A read index of current thread.
258
+ int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
259
+ (threadIdx.x % a_gl_rd_delta_o);
260
+ a_gl_rd += a_gl_rd_delta_o * slice_row;
261
+ // Shared write index of current thread.
262
+ int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
263
+ (threadIdx.x % a_gl_rd_delta_o);
264
+ // Shared read index.
265
+ int a_sh_rd =
266
+ a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
267
+ a_sh_rd += 4 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
268
+
269
+ int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
270
+ (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
271
+ b_gl_rd += b_sh_stride * slice_col;
272
+ b_gl_rd += b_gl_rd_delta_o * slice_row;
273
+ int b_sh_wr = threadIdx.x * b_thread_vecs;
274
+ int b_sh_rd = threadIdx.x * b_thread_vecs;
275
+
276
+ int m_gl_rd = m_gl_stride * (threadIdx.x / (m_sh_stride)) +
277
+ (threadIdx.x % (m_sh_stride));
278
+ m_gl_rd += (m_sh_stride)*slice_col;
279
+ m_gl_rd += m_gl_rd_delta_o * slice_row;
280
+ int m_sh_wr = threadIdx.x;
281
+ int m_sh_rd = threadIdx.x % 16 + (threadIdx.x / 32) * 16;
282
+
283
+ int s_gl_rd;
284
+ if constexpr (group_blocks == -1) {
285
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
286
+ } else {
287
+ s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
288
+ s_sh_stride * slice_col + threadIdx.x;
289
+ }
290
+
291
+ int s_sh_wr = threadIdx.x;
292
+ int s_sh_rd;
293
+ // We use a different scale layout for grouped and column-wise quantization as
294
+ // we scale a `half2` tile in column-major layout in the former and in
295
+ // row-major in the latter case.
296
+ if (group_blocks != -1) {
297
+ s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
298
+ (threadIdx.x % 32) / 4;
299
+ } else {
300
+ s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
301
+ (threadIdx.x % 32) / 4;
302
+ }
303
+
304
+ // Precompute which thread should not read memory in which iterations; this is
305
+ // needed if there are more threads than required for a certain tilesize or
306
+ // when the batchsize is not a multiple of 16.
307
+ bool a_sh_wr_pred[a_sh_wr_iters];
308
+#pragma unroll
309
+ for (int i = 0; i < a_sh_wr_iters; i++) {
310
+ a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
311
+ }
312
+ bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
313
+
314
+ // To ensure that writing and reading A tiles to/from shared memory, the
315
+ // latter in fragment format, is fully bank conflict free, we need to use a
316
+ // rather fancy XOR-based layout. The key here is that neither reads nor
317
+ // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
318
+ // same shared memory banks. Further, it seems (based on NSight-Compute) that
319
+ // each warp must also write a consecutive memory segment?
320
+ auto transform_a = [&](int i) {
321
+ int row = i / a_gl_rd_delta_o;
322
+ return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
323
+ };
324
+ // Since the computation of this remapping is non-trivial and, due to our main
325
+ // loop unrolls, all shared memory accesses are static, we simply precompute
326
+ // both transformed reads and writes.
327
+ int a_sh_wr_trans[a_sh_wr_iters];
328
+#pragma unroll
329
+ for (int i = 0; i < a_sh_wr_iters; i++)
330
+ a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
331
+ int a_sh_rd_trans[2][b_sh_wr_iters][thread_m_blocks];
332
+#pragma unroll
333
+ for (int i = 0; i < b_sh_wr_iters; i++) {
334
+#pragma unroll
335
+ for (int j = 0; j < thread_m_blocks; j++) {
336
+ a_sh_rd_trans[0][i][j] =
337
+ transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
338
+ a_sh_rd_trans[1][i][j] =
339
+ transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd + 2);
340
+ }
341
+ }
342
+
343
+ // Since B-accesses have non-constant stride they have to be computed at
344
+ // runtime; we break dependencies between subsequent accesses with a tile by
345
+ // maintining multiple pointers (we have enough registers), a tiny
346
+ // optimization.
347
+ const int4 *B_ptr[b_sh_wr_iters];
348
+#pragma unroll
349
+ for (int i = 0; i < b_sh_wr_iters; i++)
350
+ B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
351
+
352
+ bool m_sh_wr_pred = threadIdx.x < m_sh_wr_delta;
353
+ const int4 *meta_ptr[m_sh_iters];
354
+#pragma unroll
355
+ for (int i = 0; i < m_sh_iters; i++)
356
+ meta_ptr[i] = meta + m_gl_rd_delta_i * i + m_gl_rd;
357
+
358
+ extern __shared__ int4 sh[];
359
+ // Shared memory storage for global fetch pipelines.
360
+ int4 *sh_a = sh;
361
+ int4 *sh_b = sh_a + (stages * a_sh_stage);
362
+ int4 *sh_s = sh_b + (stages * b_sh_stage);
363
+ int4 *sh_m = sh_s + (stages * s_sh_stage);
364
+ // Register storage for double buffer of shared memory reads.
365
+ FragA frag_a[2][thread_m_blocks][2];
366
+ I4 frag_b_quant[2][b_thread_vecs];
367
+ FragM frag_m[2][2];
368
+ FragC frag_c[thread_m_blocks][4][2];
369
+ FragS frag_s[2][4];
370
+
371
+ // Zero accumulators.
372
+ auto zero_accums = [&]() {
373
+#pragma unroll
374
+ for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
375
+ reinterpret_cast<float *>(frag_c)[i] = 0;
376
+ };
377
+
378
+ // Asynchronously fetch the next A, B and s tile from global to the next
379
+ // shared memory pipeline location.
380
+ auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
381
+ if (pred) {
382
+ int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
383
+#pragma unroll
384
+ for (int i = 0; i < a_sh_wr_iters; i++) {
385
+ cp_async4_pred(
386
+ &sh_a_stage[a_sh_wr_trans[i]],
387
+ &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
388
+ a_sh_wr_pred[i]);
389
+ }
390
+ int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
391
+#pragma unroll
392
+ for (int i = 0; i < b_sh_wr_iters; i++) {
393
+#pragma unroll
394
+ for (int j = 0; j < b_thread_vecs; j++) {
395
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j],
396
+ B_ptr[i] + j);
397
+ }
398
+ B_ptr[i] += b_gl_rd_delta_o;
399
+ }
400
+ int4 *sh_meta_stage = sh_m + m_sh_stage * pipe;
401
+#pragma unroll
402
+ for (int i = 0; i < m_sh_iters; i++) {
403
+ if (m_sh_wr_pred)
404
+ cp_async4(&sh_meta_stage[m_sh_wr_delta * i + m_sh_wr],
405
+ meta_ptr[i]);
406
+ meta_ptr[i] += m_gl_rd_delta_o;
407
+ }
408
+ // Only fetch scales if this tile starts a new group
409
+ if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
410
+ int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
411
+ if (s_sh_wr_pred)
412
+ cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
413
+ s_gl_rd += s_gl_rd_delta;
414
+ }
415
+ }
416
+ // Insert a fence even when we are winding down the pipeline to ensure that
417
+ // waiting is also correct at this point.
418
+ cp_async_fence();
419
+ };
420
+
421
+ // Wait until the next thread tile has been loaded to shared memory.
422
+ auto wait_for_stage = [&]() {
423
+ // We only have `stages - 2` active fetches since we are double buffering
424
+ // and can only issue the next fetch when it is guaranteed that the previous
425
+ // shared memory load is fully complete (as it may otherwise be
426
+ // overwritten).
427
+ cp_async_wait<stages - 2>();
428
+ __syncthreads();
429
+ };
430
+
431
+ // Load the next sub-tile from the current location in the shared memory pipe
432
+ // into the current register buffer.
433
+ auto fetch_to_registers = [&](int k, int pipe) {
434
+ // It may seem inefficient that we reload the groups for every sub-tile;
435
+ // however, this does not seem to be a significant bottleneck, while some
436
+ // theoretically better attempts have lead to bad instruction ordering by
437
+ // the compiler and correspondingly a noticeable drop in performance.
438
+ if (group_blocks != -1) {
439
+ int4 *sh_s_stage =
440
+ sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
441
+ (pipe / (group_blocks / thread_k_blocks)));
442
+ reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
443
+ }
444
+ int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
445
+#pragma unroll
446
+ for (int i = 0; i < thread_m_blocks; i++) {
447
+ ldsm4(frag_a[k % 2][i][0],
448
+ &sh_a_stage[a_sh_rd_trans[0][k % b_sh_wr_iters][i]]);
449
+ ldsm4(frag_a[k % 2][i][1],
450
+ &sh_a_stage[a_sh_rd_trans[1][k % b_sh_wr_iters][i]]);
451
+ }
452
+
453
+ int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
454
+#pragma unroll
455
+ for (int i = 0; i < b_thread_vecs; i++) {
456
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
457
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
458
+ }
459
+
460
+ // Load meta with ldsm4
461
+ int4 *sh_m_stage = sh_m + m_sh_stage * pipe;
462
+ ldsm4_m(frag_m[k % 2][0],
463
+ &sh_m_stage[m_sh_rd_delta * (k % m_sh_iters) + m_sh_rd]);
464
+ };
465
+
466
+ // Execute the actual tensor core matmul of a sub-tile.
467
+ auto matmul = [&](int k) {
468
+// We have the m dimension as the inner loop in order to encourage overlapping
469
+// dequantization and matmul operations.
470
+#pragma unroll
471
+ for (int j = 0; j < 4; j++) {
472
+ FragB frag_b0;
473
+ FragB frag_b1;
474
+
475
+ if constexpr (num_bits == 4) {
476
+ int b_quant = frag_b_quant[k % 2][0][j];
477
+ int b_quant_shift = b_quant >> 8;
478
+
479
+ frag_b0 = dequant_4bit(b_quant);
480
+ frag_b1 = dequant_4bit(b_quant_shift);
481
+
482
+ } else {
483
+ int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
484
+ int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
485
+ int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
486
+
487
+ frag_b0 = dequant_8bit(b_quant_0);
488
+ frag_b1 = dequant_8bit(b_quant_1);
489
+ }
490
+
491
+ // If there are no groups, we can just scale the final output once and can
492
+ // avoid doing so for each weight.
493
+ if constexpr (group_blocks != -1) {
494
+ scale(frag_b0, frag_s[k % 2][j], 0);
495
+ }
496
+ if constexpr (group_blocks != -1) {
497
+ scale(frag_b1, frag_s[k % 2][j], 1);
498
+ }
499
+
500
+#pragma unroll
501
+ for (int i = 0; i < thread_m_blocks; i++) {
502
+ mma_sp(frag_b0, frag_b1, frag_a[k % 2][i][0], frag_c[i][j][0],
503
+ frag_m[k % 2][j / 2], j % 2);
504
+ }
505
+ }
506
+ };
507
+
508
+ // Since we slice across the k dimension of a tile in order to increase the
509
+ // number of warps while keeping the n dimension of a tile reasonable, we have
510
+ // multiple warps that accumulate their partial sums of the same output
511
+ // location; which we have to reduce over in the end. We do in shared memory.
512
+ auto thread_block_reduce = [&]() {
513
+ constexpr int red_off = threads / b_sh_stride_threads / 2;
514
+ if (red_off >= 1) {
515
+ int red_idx = threadIdx.x / b_sh_stride_threads;
516
+ constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
517
+ constexpr int red_sh_delta = b_sh_stride_threads;
518
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
519
+ (threadIdx.x % b_sh_stride_threads);
520
+
521
+// Parallel logarithmic shared memory reduction. We make sure to avoid any
522
+// unnecessary read or write iterations, e.g., for two warps we write only once
523
+// by warp 1 and read only once by warp 0.
524
+#pragma unroll
525
+ for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
526
+#pragma unroll
527
+ for (int i = red_off; i > 0; i /= 2) {
528
+ if (i <= red_idx && red_idx < 2 * i) {
529
+#pragma unroll
530
+ for (int j = 0; j < 4 * 2; j++) {
531
+ int red_sh_wr =
532
+ red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
533
+ if (i < red_off) {
534
+ float *c_rd = reinterpret_cast<float *>(
535
+ &sh[red_sh_delta * j + red_sh_rd]);
536
+ float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
537
+#pragma unroll
538
+ for (int k = 0; k < 4; k++)
539
+ reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
540
+ c_rd[k] + c_wr[k];
541
+ }
542
+ sh[red_sh_wr] =
543
+ reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
544
+ }
545
+ }
546
+ __syncthreads();
547
+ }
548
+ if (red_idx == 0) {
549
+#pragma unroll
550
+ for (int i = 0; i < 4 * 2; i++) {
551
+ float *c_rd =
552
+ reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
553
+#pragma unroll
554
+ for (int j = 0; j < 4; j++)
555
+ reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
556
+ c_rd[j];
557
+ }
558
+ }
559
+ __syncthreads();
560
+ }
561
+ }
562
+ };
563
+
564
+ // Since multiple threadblocks may process parts of the same column slice, we
565
+ // finally have to globally reduce over the results. As the striped partitioning
566
+ // minimizes the number of such reductions and our outputs are usually rather
567
+ // small, we perform this reduction serially in L2 cache.
568
+ auto global_reduce = [&](bool first = false, bool last = false) {
569
+ // We are very careful here to reduce directly in the output buffer to
570
+ // maximize L2 cache utilization in this step. To do this, we write out
571
+ // results in FP16 (but still reduce with FP32 compute).
572
+ constexpr int active_threads = 32 * thread_n_blocks / 4;
573
+ if (threadIdx.x < active_threads) {
574
+ int c_gl_stride = prob_n / 8;
575
+ int c_gl_wr_delta_o = 2 * 4 * c_gl_stride;
576
+ int c_gl_wr_delta_i =
577
+ c_gl_stride; // 8 threads (e.g., 0,4,8,12,16,20,24,28)
578
+ int c_gl_wr = 2 * c_gl_stride * (threadIdx.x % 4) +
579
+ 8 * (threadIdx.x / 32) + (threadIdx.x % 32) / 4;
580
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
581
+ constexpr int c_sh_wr_delta = active_threads;
582
+ int c_sh_wr = threadIdx.x;
583
+
584
+ int col = 2 * ((threadIdx.x % 32) % 4);
585
+
586
+ if (!first) {
587
+// Interestingly, doing direct global accesses here really seems to mess up the
588
+// compiler and lead to slowdowns, hence we also use async-copies even though
589
+// these fetches are not actually asynchronous.
590
+#pragma unroll
591
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
592
+ cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
593
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
594
+ c_gl_wr_delta_i * (i % 2)],
595
+ i < (thread_m_blocks - 1) * 4 ||
596
+ 8 * (i / 2) + col + (i % 2) < prob_m);
597
+ }
598
+ cp_async_fence();
599
+ cp_async_wait<0>();
600
+ }
601
+
602
+#pragma unroll
603
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
604
+ if (i < (thread_m_blocks - 1) * 4 ||
605
+ 8 * (i / 2) + col + (i % 2) < prob_m) {
606
+ if (!first) {
607
+ int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
608
+#pragma unroll
609
+ for (int j2 = 0; j2 < 2; j2++) {
610
+#pragma unroll
611
+ for (int j1 = 0; j1 < 4; j1++) {
612
+ reinterpret_cast<float *>(
613
+ &frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
614
+ 4 * ((i % 4) / 2) + i % 2] +=
615
+ __half2float(
616
+ reinterpret_cast<__half *>(&c_red)[(j2 * 4 + j1)]);
617
+ }
618
+ }
619
+ }
620
+ if (!last) {
621
+ int4 c;
622
+#pragma unroll
623
+ for (int j2 = 0; j2 < 2; j2++) {
624
+#pragma unroll
625
+ for (int j1 = 0; j1 < 4; j1++) {
626
+ reinterpret_cast<__half *>(&c)[(j2 * 4 + j1)] =
627
+ __float2half(reinterpret_cast<float *>(
628
+ &frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
629
+ 4 * ((i % 4) / 2) + i % 2]);
630
+ }
631
+ }
632
+ C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
633
+ c;
634
+ }
635
+ }
636
+ }
637
+ }
638
+ };
639
+
640
+ // Write out the reduce final result in the correct layout. We only actually
641
+ // reshuffle matrix fragments in this step, the reduction above is performed
642
+ // in fragment layout.
643
+ auto write_result = [&]() {
644
+ int c_gl_stride = prob_n / 8;
645
+
646
+ constexpr int c_sh_stride = 2 * thread_n_blocks; // RLC:
647
+ constexpr int c_sh_stride_2 = 2 * c_sh_stride + 2; // RLC:
648
+ constexpr int c_sh_stride_3 = 2 * (2 * thread_n_blocks) + 2; // RLC:
649
+
650
+ int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
651
+
652
+ int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
653
+ (threadIdx.x % (2 * thread_n_blocks));
654
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
655
+
656
+ int c_sh_wr = c_sh_stride_2 * ((threadIdx.x % 32) % 4) +
657
+ ((threadIdx.x % 32) / 4); // RLC:
658
+ c_sh_wr += 8 * (threadIdx.x / 32); // 128/4(half4)
659
+
660
+ constexpr int c_sh_rd_delta =
661
+ c_sh_stride_3 * (threads / (2 * 2 * thread_n_blocks)); // RLC:
662
+ int c_sh_rd = c_sh_stride_3 * (threadIdx.x / (2 * 2 * thread_n_blocks)) +
663
+ (threadIdx.x % (2 * 2 * thread_n_blocks));
664
+
665
+ int c_gl_wr_end = c_gl_stride * prob_m;
666
+
667
+ auto write = [&](int idx, float c0, float c1, float c2, float c3, FragS &s0,
668
+ float c4, float c5, float c6, float c7, FragS &s1) {
669
+ uint2 res[2];
670
+ res[0] = to_half4(c0, c1, c2, c3);
671
+ res[1] = to_half4(c4, c5, c6, c7);
672
+ half2 *tmp = (half2 *)&res;
673
+ // for per-column quantization we finally apply the scale here
674
+ if constexpr (group_blocks == -1 && num_bits == 4) {
675
+ tmp[0] = __hmul2(tmp[0], s0[0]);
676
+ tmp[1] = __hmul2(tmp[1], s0[1]);
677
+ tmp[2] = __hmul2(tmp[2], s1[0]);
678
+ tmp[3] = __hmul2(tmp[3], s1[1]);
679
+ }
680
+ ((int4 *)sh)[idx] = *((int4 *)&res[0]);
681
+ };
682
+
683
+ // RLC: only warp 0 and 1 baseline example
684
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
685
+#pragma unroll
686
+ for (int i = 0; i < thread_m_blocks; i++) {
687
+ int wr = c_sh_wr;
688
+ write(wr, frag_c[i][0][0][0], frag_c[i][1][0][0], frag_c[i][2][0][0],
689
+ frag_c[i][3][0][0], frag_s[0][0], frag_c[i][0][0][2],
690
+ frag_c[i][1][0][2], frag_c[i][2][0][2], frag_c[i][3][0][2],
691
+ frag_s[0][2]);
692
+ write(wr + c_sh_stride, frag_c[i][0][0][1], frag_c[i][1][0][1],
693
+ frag_c[i][2][0][1], frag_c[i][3][0][1], frag_s[0][0],
694
+ frag_c[i][0][0][3], frag_c[i][1][0][3], frag_c[i][2][0][3],
695
+ frag_c[i][3][0][3], frag_s[0][2]);
696
+ write(wr + 4 * c_sh_stride_2, frag_c[i][0][1][0], frag_c[i][1][1][0],
697
+ frag_c[i][2][1][0], frag_c[i][3][1][0], frag_s[0][0],
698
+ frag_c[i][0][1][2], frag_c[i][1][1][2], frag_c[i][2][1][2],
699
+ frag_c[i][3][1][2], frag_s[0][2]);
700
+ write(wr + 4 * c_sh_stride_2 + c_sh_stride, frag_c[i][0][1][1],
701
+ frag_c[i][1][1][1], frag_c[i][2][1][1], frag_c[i][3][1][1],
702
+ frag_s[0][0], frag_c[i][0][1][3], frag_c[i][1][1][3],
703
+ frag_c[i][2][1][3], frag_c[i][3][1][3], frag_s[0][2]);
704
+
705
+ c_sh_wr += 8 * c_sh_stride_2;
706
+ }
707
+ }
708
+ __syncthreads();
709
+
710
+#pragma unroll
711
+ for (int i = 0;
712
+ i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
713
+ i++) {
714
+ if (c_gl_wr < c_gl_wr_end) {
715
+ C[c_gl_wr] = sh[c_sh_rd];
716
+ c_gl_wr += c_gl_wr_delta;
717
+ c_sh_rd += c_sh_rd_delta;
718
+ }
719
+ }
720
+ };
721
+
722
+ // Start global fetch and register load pipelines.
723
+ auto start_pipes = [&]() {
724
+#pragma unroll
725
+ for (int i = 0; i < stages - 1; i++)
726
+ fetch_to_shared(i, i, i < slice_iters);
727
+ zero_accums();
728
+ wait_for_stage();
729
+ fetch_to_registers(0, 0);
730
+ a_gl_rd += a_gl_rd_delta_o * (stages - 1);
731
+ };
732
+ start_pipes();
733
+
734
+ // Main loop.
735
+ while (slice_iters) {
736
+// We unroll over both the global fetch and the register load pipeline to ensure
737
+// all shared memory accesses are static. Note that both pipelines have even
738
+// length meaning that the next iteration will always start at index 0.
739
+#pragma unroll
740
+ for (int pipe = 0; pipe < stages;) {
741
+ fetch_to_shared((pipe + stages - 1) % stages, pipe,
742
+ slice_iters >= stages);
743
+ wait_for_stage();
744
+
745
+ fetch_to_registers(pipe + 1, (pipe + 1) % stages);
746
+ matmul(pipe);
747
+
748
+ pipe++;
749
+ slice_iters--;
750
+ if (slice_iters == 0)
751
+ break;
752
+ }
753
+ a_gl_rd += a_gl_rd_delta_o * stages;
754
+
755
+ // Process results and, if necessary, proceed to the next column slice.
756
+ // While this pattern may not be the most readable, other ways of writing
757
+ // the loop seemed to noticeably worse performance after compilation.
758
+ if (slice_iters == 0) {
759
+ cp_async_wait<0>();
760
+ bool last = slice_idx == slice_count - 1;
761
+ // For per-column scales, we only fetch them here in the final step before
762
+ // write-out
763
+ if constexpr (group_blocks == -1) {
764
+ if constexpr (num_bits == 8) {
765
+ if (s_sh_wr_pred)
766
+ cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
767
+ cp_async_fence();
768
+ } else {
769
+ if (last) {
770
+ if (s_sh_wr_pred)
771
+ cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
772
+ cp_async_fence();
773
+ }
774
+ }
775
+ }
776
+ thread_block_reduce();
777
+
778
+ if constexpr (group_blocks == -1) {
779
+ if constexpr (num_bits == 8) {
780
+ cp_async_wait<0>();
781
+ __syncthreads();
782
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
783
+ *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
784
+ }
785
+ } else {
786
+ if (last) {
787
+ cp_async_wait<0>();
788
+ __syncthreads();
789
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
790
+ *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
791
+ }
792
+ }
793
+ }
794
+ }
795
+
796
+ // For 8-bit channelwise, we apply the scale before the global reduction
797
+ // that converts the fp32 results to fp16 (so that we avoid possible
798
+ // overflow in fp16)
799
+ if constexpr (group_blocks == -1 && num_bits == 8) {
800
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
801
+#pragma unroll
802
+ for (int i = 0; i < thread_m_blocks; i++) {
803
+ scale_floats(&frag_c[i][0][0][0], &frag_c[i][1][0][0],
804
+ &frag_c[i][2][0][0], &frag_c[i][3][0][0], frag_s[0][0],
805
+ &frag_c[i][0][0][2], &frag_c[i][1][0][2],
806
+ &frag_c[i][2][0][2], &frag_c[i][3][0][2],
807
+ frag_s[0][2]);
808
+
809
+ scale_floats(&frag_c[i][0][0][1], &frag_c[i][1][0][1],
810
+ &frag_c[i][2][0][1], &frag_c[i][3][0][1], frag_s[0][0],
811
+ &frag_c[i][0][0][3], &frag_c[i][1][0][3],
812
+ &frag_c[i][2][0][3], &frag_c[i][3][0][3],
813
+ frag_s[0][2]);
814
+
815
+ scale_floats(&frag_c[i][0][1][0], &frag_c[i][1][1][0],
816
+ &frag_c[i][2][1][0], &frag_c[i][3][1][0], frag_s[0][0],
817
+ &frag_c[i][0][1][2], &frag_c[i][1][1][2],
818
+ &frag_c[i][2][1][2], &frag_c[i][3][1][2],
819
+ frag_s[0][2]);
820
+
821
+ scale_floats(&frag_c[i][0][1][1], &frag_c[i][1][1][1],
822
+ &frag_c[i][2][1][1], &frag_c[i][3][1][1], frag_s[0][0],
823
+ &frag_c[i][0][1][3], &frag_c[i][1][1][3],
824
+ &frag_c[i][2][1][3], &frag_c[i][3][1][3],
825
+ frag_s[0][2]);
826
+ }
827
+ }
828
+ }
829
+
830
+ if (slice_count > 1) { // only globally reduce if there is more than one
831
+ // block in a slice
832
+ barrier_acquire(&locks[slice_col], slice_idx);
833
+ global_reduce(slice_idx == 0, last);
834
+ barrier_release(&locks[slice_col], last);
835
+ }
836
+ if (last) // only the last block in a slice actually writes the result
837
+ write_result();
838
+
839
+ slice_row = 0;
840
+ slice_col_par++;
841
+ slice_col++;
842
+ init_slice();
843
+ if (slice_iters) {
844
+ a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
845
+ (threadIdx.x % a_gl_rd_delta_o);
846
+#pragma unroll
847
+ for (int i = 0; i < b_sh_wr_iters; i++)
848
+ B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
849
+#pragma unroll
850
+ for (int i = 0; i < m_sh_iters; i++)
851
+ meta_ptr[i] += (m_sh_stride)-m_gl_rd_delta_o * k_tiles;
852
+ if (slice_col == 0) {
853
+#pragma unroll
854
+ for (int i = 0; i < b_sh_wr_iters; i++)
855
+ B_ptr[i] -= b_gl_stride;
856
+#pragma unroll
857
+ for (int i = 0; i < m_sh_iters; i++)
858
+ meta_ptr[i] -= m_gl_stride;
859
+ }
860
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
861
+ start_pipes();
862
+ }
863
+ }
864
+ }
865
+}
866
+
867
+#endif
868
+
869
+#define CALL_IF_2_4(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
870
+ THREAD_K_BLOCKS, GROUP_BLOCKS) \
871
+ else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
872
+ thread_n_blocks == THREAD_N_BLOCKS && \
873
+ thread_k_blocks == THREAD_K_BLOCKS && \
874
+ group_blocks == GROUP_BLOCKS) { \
875
+ cudaFuncSetAttribute( \
876
+ Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
877
+ THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
878
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
879
+ Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
880
+ THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS> \
881
+ <<<blocks, THREADS, max_shared_mem, stream>>>(A_ptr, B_ptr, meta_ptr, \
882
+ C_ptr, s_ptr, prob_n, \
883
+ prob_m, prob_k, locks); \
884
+ }
885
+
886
+void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
887
+ void *s, int prob_m, int prob_n, int prob_k,
888
+ void *workspace, int num_bits, int groupsize = -1,
889
+ int dev = 0, cudaStream_t stream = 0, int thread_k = -1,
890
+ int thread_m = -1, int sms = -1, int max_par = 16) {
891
+ int tot_n = prob_n;
892
+ int tot_n_blocks = ceildiv(tot_n, 16);
893
+ int pad = 16 * tot_n_blocks - tot_n;
894
+
895
+ if (sms == -1) {
896
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
897
+ }
898
+ TORCH_CHECK(sms > 0);
899
+
900
+ int max_shared_mem = 0;
901
+ cudaDeviceGetAttribute(&max_shared_mem,
902
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
903
+ TORCH_CHECK(max_shared_mem > 0);
904
+
905
+ if (thread_k == -1 || thread_m == -1) {
906
+ if (prob_n <= 16) {
907
+ // For small batchizes, better partitioningif is slightly more important than
908
+ // better compute utilization
909
+ thread_k = 128;
910
+ thread_m = 128;
911
+ } else {
912
+ thread_k = 64;
913
+ thread_m = 256;
914
+ }
915
+ }
916
+
917
+ int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
918
+ int thread_m_blocks = thread_m / 16;
919
+ int group_blocks = (groupsize == -1) ? -1 : groupsize / 16;
920
+ int blocks = sms;
921
+
922
+ TORCH_CHECK(prob_m % thread_m == 0, "prob_m = ", prob_m,
923
+ " is not divisible by thread_m = ", thread_m);
924
+ TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
925
+ " is not divisible by thread_k = ", thread_k);
926
+ if (group_blocks != -1) {
927
+ TORCH_CHECK((prob_k / 2) % group_blocks == 0, "prob_k/2 = ", prob_k / 2,
928
+ " is not divisible by group_blocks = ", group_blocks);
929
+ }
930
+
931
+ TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
932
+ ", ", prob_n, ", ", prob_k, "]");
933
+
934
+ const int4 *A_ptr = (const int4 *)A;
935
+ const int4 *B_ptr = (const int4 *)B;
936
+ const int4 *meta_ptr = (const int4 *)meta;
937
+ int4 *C_ptr = (int4 *)C;
938
+ const int4 *s_ptr = (const int4 *)s;
939
+
940
+ int *locks = (int *)workspace;
941
+ for (int i = 0; i < tot_n_blocks; i += 4) {
942
+ int thread_n_blocks = tot_n_blocks - i;
943
+ prob_n = tot_n - 16 * i;
944
+ int par = 1;
945
+ if (thread_n_blocks > 4) {
946
+ // Note that parallel > 1 currently only works for inputs without any
947
+ // padding
948
+ par = (16 * thread_n_blocks - pad) / 64;
949
+ if (par > max_par)
950
+ par = max_par;
951
+ prob_n = 64 * par;
952
+ i += 4 * (par - 1);
953
+ thread_n_blocks = 4;
954
+ }
955
+
956
+ // For compilation speed, we only define the kernel configurations that have
957
+ // seemed useful (in terms of performance) in our testing, however many more
958
+ // are, in principle, possible.
959
+
960
+ // the false is start of the CALL_IF macros
961
+ if (false) {
962
+ } // BMxBNxBK, group
963
+ // 4-bit
964
+ CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
965
+ CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
966
+ CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
967
+ CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
968
+ CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
969
+ CALL_IF_2_4(4, 16, 2, 2, 4)
970
+ CALL_IF_2_4(4, 16, 3, 2, -1)
971
+ CALL_IF_2_4(4, 16, 3, 2, 4)
972
+ CALL_IF_2_4(4, 16, 4, 2, -1)
973
+ CALL_IF_2_4(4, 16, 4, 2, 4)
974
+
975
+ // 8-bit
976
+ CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
977
+ CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
978
+ CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
979
+ CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
980
+ CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
981
+ CALL_IF_2_4(8, 16, 2, 2, 4)
982
+ CALL_IF_2_4(8, 16, 3, 2, -1)
983
+ CALL_IF_2_4(8, 16, 3, 2, 4)
984
+ CALL_IF_2_4(8, 16, 4, 2, -1)
985
+ CALL_IF_2_4(8, 16, 4, 2, 4)
986
+ else {
987
+ throw std::runtime_error("Unsupported shapes: MKN = [" + str(prob_m) +
988
+ ", " + str(prob_k) + ", " + str(prob_n) + "]" +
989
+ ", groupsize = " + str(groupsize) +
990
+ ", thread_m_blocks = " + str(thread_m_blocks) +
991
+ ", thread_n_blocks = " + str(thread_n_blocks) +
992
+ ", thread_k_blocks = " + str(thread_k_blocks));
993
+ }
994
+
995
+ A_ptr += 16 * thread_n_blocks * (prob_k / 8) * par;
996
+ C_ptr += 16 * thread_n_blocks * (prob_m / 8) * par;
997
+ }
998
+}
999
+
1000
+} // namespace marlin_24
1001
+
1002
+torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1003
+ torch::Tensor &b_meta,
1004
+ torch::Tensor &b_scales,
1005
+ torch::Tensor &workspace, int64_t num_bits,
1006
+ int64_t size_m, int64_t size_n,
1007
+ int64_t size_k) {
1008
+ // Verify num_bits
1009
+ TORCH_CHECK(num_bits == 4 || num_bits == 8,
1010
+ "num_bits must be 4 or 8. Got = ", num_bits);
1011
+ int pack_factor = 32 / num_bits;
1012
+
1013
+ // Verify M
1014
+ TORCH_CHECK(size_m == a.size(0),
1015
+ "Shape mismatch: a.size(0) = " + str(a.size(0)) +
1016
+ ", size_m = " + str(size_m));
1017
+
1018
+ // Verify K
1019
+ TORCH_CHECK(size_k == a.size(1),
1020
+ "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1021
+ ", size_k = " + str(size_k));
1022
+ TORCH_CHECK(size_k % marlin_24::tile_size == 0,
1023
+ "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1024
+ str(marlin_24::tile_size));
1025
+ TORCH_CHECK((size_k / marlin_24::tile_size / 2) == b_q_weight.size(0),
1026
+ "Shape mismatch: b_q_weight.size(0) = " +
1027
+ str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1028
+ ", tile_size = " + str(marlin_24::tile_size));
1029
+
1030
+ // Verify N
1031
+ TORCH_CHECK(b_scales.size(1) == size_n,
1032
+ "b_scales.size(1) = " + str(b_scales.size(1)) +
1033
+ ", size_n = " + str(size_n));
1034
+ TORCH_CHECK(
1035
+ b_q_weight.size(1) % marlin_24::tile_size == 0,
1036
+ "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1037
+ " is not divisible by tile_size = " + str(marlin_24::tile_size));
1038
+
1039
+ int actual_size_n = (b_q_weight.size(1) / marlin_24::tile_size) * pack_factor;
1040
+ TORCH_CHECK(size_n == actual_size_n,
1041
+ "size_n = " + str(size_n) +
1042
+ ", actual_size_n = " + str(actual_size_n));
1043
+
1044
+ // Verify meta
1045
+ TORCH_CHECK(b_meta.size(0) == size_k / 8 / 2 / 2,
1046
+ "b_meta.size(0) = ", b_meta.size(0),
1047
+ " is not size_k / 8 / 2 / 2 = ", size_k / 8 / 2 / 2);
1048
+ TORCH_CHECK(b_meta.size(1) == size_n * 2, "b_meta.size(1) = ", b_meta.size(1),
1049
+ " is not size_n * 2 = ", size_n * 2);
1050
+
1051
+ // Verify A device and strides
1052
+ TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1053
+ TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1054
+
1055
+ // Verify B device and strides
1056
+ TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
1057
+ TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
1058
+
1059
+ // Verify b_meta device and strides
1060
+ TORCH_CHECK(b_meta.device().is_cuda(), "b_meta is not on GPU");
1061
+ TORCH_CHECK(b_meta.is_contiguous(), "b_meta is not contiguous");
1062
+
1063
+ // Verify scales device and strides
1064
+ TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
1065
+ TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
1066
+
1067
+ // Alloc C matrix
1068
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1069
+ auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1070
+ torch::Tensor c = torch::empty({size_m, size_n}, options);
1071
+
1072
+ int thread_k = -1;
1073
+ int thread_m = -1;
1074
+ int sms = -1;
1075
+ int max_par = 16;
1076
+
1077
+ int groupsize = -1;
1078
+ if (b_scales.size(0) > 1) {
1079
+ TORCH_CHECK(size_k % b_scales.size(0) == 0,
1080
+ "size_k = " + str(size_k) +
1081
+ ", is not divisible by b_scales.size(0) = " +
1082
+ str(b_scales.size(0)));
1083
+ groupsize = size_k / b_scales.size(0);
1084
+ groupsize /= 2; // Because of 24
1085
+ }
1086
+
1087
+ // Verify groupsize
1088
+ TORCH_CHECK(groupsize == -1 || groupsize == 64,
1089
+ "Unexpected groupsize = " + str(groupsize));
1090
+
1091
+ // Verify workspace size
1092
+ TORCH_CHECK(size_n % marlin_24::min_thread_n == 0,
1093
+ "size_n = " + str(size_n) +
1094
+ ", is not divisible by min_thread_n = " +
1095
+ str(marlin_24::min_thread_n));
1096
+ int min_workspace_size =
1097
+ (size_n / marlin_24::min_thread_n) * marlin_24::max_par;
1098
+ TORCH_CHECK(workspace.numel() >= min_workspace_size,
1099
+ "workspace.numel = " + str(workspace.numel()) +
1100
+ " is below min_workspace_size = " + str(min_workspace_size));
1101
+
1102
+ int dev = a.get_device();
1103
+ marlin_24::marlin_cuda_2_4(
1104
+ a.data_ptr(), b_q_weight.data_ptr(), b_meta.data_ptr(), c.data_ptr(),
1105
+ b_scales.data_ptr(), size_n, size_m, size_k, workspace.data_ptr(),
1106
+ num_bits, groupsize, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1107
+ thread_m, sms, max_par);
1108
+
1109
+ return c;
1110
+}
tests/models/test_gptq_marlin_24.py
new
+81
@@ -0,0 +1,81 @@
1
+"""Compare the outputs of a GPTQ model to a Marlin_24 model.
2
+
3
+Note: GPTQ and Marlin_24 do not have bitwise correctness.
4
+As a result, in this test, we just confirm that the top selected tokens of the
5
+Marlin/GPTQ models are in the top 3 selections of each other.
6
+
7
+Run `pytest tests/models/test_marlin_24.py`.
8
+"""
9
+from dataclasses import dataclass
10
+
11
+import pytest
12
+import torch
13
+
14
+from tests.models.utils import check_logprobs_close
15
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
+
17
+capability = torch.cuda.get_device_capability()
18
+capability = capability[0] * 10 + capability[1]
19
+marlin_not_supported = (capability <
20
+ QUANTIZATION_METHODS["marlin"].get_min_capability())
21
+
22
+
23
+@dataclass
24
+class ModelPair:
25
+ model_marlin: str
26
+ model_gptq: str
27
+
28
+
29
+model_pairs = [
30
+ # 4-bit, group_size == 128
31
+ ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-g128",
32
+ model_gptq="alexm-nm/tinyllama-24-gptq-4bit-g128"),
33
+ # 4-bit, group_size == channelwise
34
+ ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-channelwise",
35
+ model_gptq="alexm-nm/tinyllama-24-gptq-4bit-channelwise"),
36
+
37
+ # 8-bit, group_size == 128
38
+ ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-g128",
39
+ model_gptq="alexm-nm/tinyllama-24-gptq-8bit-g128"),
40
+ # 8-bit, group_size == channelwise
41
+ ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-channelwise",
42
+ model_gptq="alexm-nm/tinyllama-24-gptq-8bit-channelwise"),
43
+]
44
+
45
+
46
+@pytest.mark.flaky(reruns=2)
47
+@pytest.mark.skipif(marlin_not_supported,
48
+ reason="Marlin24 is not supported on this GPU type.")
49
+@pytest.mark.parametrize("model_pair", model_pairs)
50
+@pytest.mark.parametrize("dtype", ["half"])
51
+@pytest.mark.parametrize("max_tokens", [8])
52
+@pytest.mark.parametrize("num_logprobs", [5])
53
+def test_models(
54
+ vllm_runner,
55
+ example_prompts,
56
+ model_pair: ModelPair,
57
+ dtype: str,
58
+ max_tokens: int,
59
+ num_logprobs: int,
60
+) -> None:
61
+ marlin_24_model = vllm_runner(model_pair.model_marlin,
62
+ dtype=dtype,
63
+ quantization="gptq_marlin_24")
64
+ marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
65
+ example_prompts, max_tokens, num_logprobs)
66
+ del marlin_24_model
67
+
68
+ gptq_model = vllm_runner(model_pair.model_gptq,
69
+ dtype=dtype,
70
+ quantization="gptq")
71
+ gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
72
+ max_tokens,
73
+ num_logprobs)
74
+ del gptq_model
75
+
76
+ check_logprobs_close(
77
+ outputs_0_lst=gptq_outputs,
78
+ outputs_1_lst=marlin_24_outputs,
79
+ name_0="gptq",
80
+ name_1="marlin_24",
81
+ )
vllm/_custom_ops.py
+10
@@ -153,6 +153,16 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
153
size_n, size_k)
154
155
156
+# marlin_24
157
+def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
158
+ b_meta: torch.Tensor, b_scales: torch.Tensor,
159
+ workspace: torch.Tensor, num_bits: int, size_m: int,
160
+ size_n: int, size_k: int) -> torch.Tensor:
161
+ return vllm_ops.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
162
+ workspace, num_bits, size_m, size_n,
163
+ size_k)
164
+
165
+
166
# aqlm
167
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
168
codebooks: torch.Tensor, scales: torch.Tensor,
vllm/config.py
+12
-36
@@ -7,14 +7,11 @@ import torch
7
from transformers import PretrainedConfig
8
9
from vllm.logger import init_logger
10
-from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
11
- get_quantization_config)
10
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11
from vllm.model_executor.models import ModelRegistry
12
from vllm.transformers_utils.config import get_config, get_hf_text_config
13
from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
14
16
-GPTQMarlinConfig = get_quantization_config("gptq_marlin")
17
-
15
if TYPE_CHECKING:
16
from ray.util.placement_group import PlacementGroup
17
@@ -155,37 +152,15 @@ class ModelConfig:
152
quant_cfg = getattr(self.hf_config, "quantization_config", None)
153
if quant_cfg is not None:
154
quant_method = quant_cfg.get("quant_method", "").lower()
158
- # compat: autogptq >=0.8.0 use checkpoint_format: str
159
- # compat: autogptq <=0.7.1 is_marlin_format: bool
160
- is_format_marlin = (quant_cfg.get("checkpoint_format") == "marlin"
161
- or quant_cfg.get("is_marlin_format", False))
162
-
163
- # Check which LinearMethod the GPTQ model should use.
164
- if quant_method == "gptq":
165
- # If serialized in Marlin format, use MarlinLinearMethod.
166
- # TODO (@robertgshaw): migrate under GPTQMarlinLinearMethod.
167
- if is_format_marlin:
168
- logger.info("The model is serialized in Marlin format. "
169
- "Using Marlin kernel.")
170
- quant_method = "marlin"
171
- if self.quantization == "gptq":
172
- self.quantization = quant_method
173
-
174
- # If convertible to Marlin format, use GPTQMarlinLinearMethod
175
- # unless the user explicitly specified GPTQLinearMethod.
176
- elif GPTQMarlinConfig.is_marlin_compatible(quant_cfg):
177
- if self.quantization == "gptq":
178
- logger.warning(
179
- "The model is convertible to Marlin format, but "
180
- "you specified quantization=gptq. Use "
181
- "quantization=marlin for faster inference.")
182
- else:
183
- logger.info(
184
- "The model is convertible to Marlin format. "
185
- "Using Marlin kernel.")
186
- quant_method = "gptq_marlin"
187
- if self.quantization == "marlin":
188
- self.quantization = quant_method
155
+
156
+ # Detect which checkpoint is it
157
+ for name, method in QUANTIZATION_METHODS.items():
158
+ quantization_override = method.override_quantization_method(
159
+ quant_cfg, self.quantization)
160
+ if quantization_override:
161
+ quant_method = quantization_override
162
+ self.quantization = quantization_override
163
+ break
164
165
# Verify quantization configurations.
166
if self.quantization is None:
@@ -207,7 +182,8 @@ class ModelConfig:
182
raise ValueError(
183
f"{self.quantization} quantization is currently not "
184
f"supported in ROCm.")
210
- if (self.quantization not in ["marlin", "gptq_marlin"]):
185
+ if (self.quantization
186
+ not in ["marlin", "gptq_marlin_24", "gptq_marlin"]):
187
logger.warning(
188
"%s quantization is not fully "
189
"optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/__init__.py
+8
-3
@@ -10,18 +10,23 @@ from vllm.model_executor.layers.quantization.fp8 import Fp8Config
10
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
11
from vllm.model_executor.layers.quantization.gptq_marlin import (
12
GPTQMarlinConfig)
13
+from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
14
+ GPTQMarlin24Config)
15
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
16
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
17
18
QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
19
"aqlm": AQLMConfig,
20
"awq": AWQConfig,
21
+ "deepspeedfp": DeepSpeedFPConfig,
22
"fp8": Fp8Config,
23
+ # The order of gptq methods is important for config.py iteration over
24
+ # override_quantization_method(..)
25
+ "marlin": MarlinConfig,
26
+ "gptq_marlin_24": GPTQMarlin24Config,
27
+ "gptq_marlin": GPTQMarlinConfig,
28
"gptq": GPTQConfig,
29
"squeezellm": SqueezeLLMConfig,
22
- "gptq_marlin": GPTQMarlinConfig,
23
- "marlin": MarlinConfig,
24
- "deepspeedfp": DeepSpeedFPConfig
30
}
31
32
vllm/model_executor/layers/quantization/base_config.py
+11
@@ -66,6 +66,17 @@ class QuantizationConfig(ABC):
66
"""Create a config class from the model's quantization config."""
67
raise NotImplementedError
68
69
+ @classmethod
70
+ def override_quantization_method(cls, hf_quant_cfg,
71
+ user_quant) -> Optional[str]:
72
+ """
73
+ Detects if this quantization method can support a given checkpoint
74
+ format by overriding the user specified quantization method --
75
+ this method should only be overwritten by subclasses in exceptional
76
+ circumstances
77
+ """
78
+ return None
79
+
80
@staticmethod
81
def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
82
"""Get a value from the model's quantization config."""
vllm/model_executor/layers/quantization/gptq_marlin.py
+23
@@ -6,11 +6,14 @@ import torch
6
from torch.nn.parameter import Parameter
7
8
from vllm import _custom_ops as ops
9
+from vllm.logger import init_logger
10
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
15
+logger = init_logger(__name__)
16
+
17
GPTQ_MARLIN_TILE = 16
18
GPTQ_MARLIN_MIN_THREAD_N = 64
19
GPTQ_MARLIN_MIN_THREAD_K = 128
@@ -117,6 +120,26 @@ class GPTQMarlinConfig(QuantizationConfig):
120
is_sym = cls.get_from_keys(config, ["sym"])
121
return cls(weight_bits, group_size, desc_act, is_sym)
122
123
+ @classmethod
124
+ def override_quantization_method(cls, hf_quant_cfg,
125
+ user_quant) -> Optional[str]:
126
+ can_convert = cls.is_marlin_compatible(hf_quant_cfg)
127
+
128
+ is_valid_user_quant = (user_quant is None or user_quant == "marlin")
129
+
130
+ if can_convert and is_valid_user_quant:
131
+ msg = ("The model is convertible to {} during runtime."
132
+ " Using {} kernel.".format(cls.get_name(), cls.get_name()))
133
+ logger.info(msg)
134
+ return cls.get_name()
135
+
136
+ if can_convert and user_quant == "gptq":
137
+ logger.info("Detected that the model can run with gptq_marlin"
138
+ ", however you specified quantization=gptq explicitly,"
139
+ " so forcing gptq. Use quantization=gptq_marlin for"
140
+ " faster inference")
141
+ return None
142
+
143
def get_quant_method(
144
self,
145
layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
vllm/model_executor/layers/quantization/gptq_marlin_24.py
new
+280
@@ -0,0 +1,280 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig)
11
+from vllm.model_executor.utils import set_weight_attrs
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+class GPTQMarlin24Config(QuantizationConfig):
17
+ """Config class for Marlin24.
18
+ """
19
+
20
+ def __init__(
21
+ self,
22
+ weight_bits: int,
23
+ group_size: int,
24
+ ) -> None:
25
+ self.weight_bits = weight_bits
26
+ self.group_size = group_size
27
+
28
+ if self.weight_bits != 4 and self.weight_bits != 8:
29
+ raise ValueError("weight_bits must be 4 or 8. Got = {}".format(
30
+ self.weight_bits))
31
+
32
+ if self.group_size != 128 and self.group_size != -1:
33
+ raise ValueError(
34
+ "Currently, only group size 128 and -1 (channelwise) "
35
+ "is supported for Marlin24, but got group_size of "
36
+ f"{self.group_size}")
37
+
38
+ # 4 Bits packed into 32 bit datatype.
39
+ self.pack_factor = 32 // self.weight_bits
40
+
41
+ # Tile size used by marlin kernels.
42
+ self.tile_size = 16
43
+
44
+ # Min out_features dim
45
+ self.min_n_threads = 128
46
+
47
+ # Min in_features dim
48
+ self.min_k_threads = 128
49
+
50
+ # Max parallel problems to solve at once (improves large
51
+ # batch performance)
52
+ self.max_parallel = 16
53
+
54
+ # Permutation length used by the marlin kernels.
55
+ self.perm_len = 1024
56
+
57
+ def __repr__(self) -> str:
58
+ return "Marlin24Config(weight_bits={}, group_size={})".format(
59
+ self.weight_bits, self.group_size)
60
+
61
+ @classmethod
62
+ def get_name(cls) -> str:
63
+ return "gptq_marlin_24"
64
+
65
+ @classmethod
66
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
67
+ return [torch.half]
68
+
69
+ @classmethod
70
+ # Need to figure it out
71
+ def get_min_capability(cls) -> int:
72
+ return 80
73
+
74
+ @classmethod
75
+ def get_config_filenames(cls) -> List[str]:
76
+ return ["quantize_config.json"]
77
+
78
+ @classmethod
79
+ def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlin24Config":
80
+ weight_bits = cls.get_from_keys(config, ["bits"])
81
+ group_size = cls.get_from_keys(config, ["group_size"])
82
+ return cls(weight_bits, group_size)
83
+
84
+ @classmethod
85
+ def override_quantization_method(cls, hf_quant_cfg,
86
+ user_quant) -> Optional[str]:
87
+ is_marlin_24_format = (
88
+ hf_quant_cfg.get("checkpoint_format") == "marlin_24")
89
+
90
+ is_valid_user_quant = (user_quant is None or user_quant == "gptq"
91
+ or user_quant == "gptq_marlin_24")
92
+
93
+ if is_marlin_24_format and is_valid_user_quant:
94
+ msg = ("The model is serialized in {} format. "
95
+ "Using {} kernel.".format(cls.get_name(), cls.get_name()))
96
+ logger.info(msg)
97
+ return cls.get_name()
98
+
99
+ return None
100
+
101
+ def get_quant_method(
102
+ self,
103
+ layer: torch.nn.Module) -> Optional["GPTQMarlin24LinearMethod"]:
104
+ if isinstance(layer, LinearBase):
105
+ return GPTQMarlin24LinearMethod(self)
106
+ return None
107
+
108
+ def get_scaled_act_names(self) -> List[str]:
109
+ return []
110
+
111
+
112
+class GPTQMarlin24LinearMethod(LinearMethodBase):
113
+ """Linear method for Marlin24.
114
+
115
+ Args:
116
+ quant_config: The Marlin24 quantization config.
117
+ """
118
+
119
+ def __init__(self, quant_config: GPTQMarlin24Config):
120
+ self.quant_config = quant_config
121
+
122
+ def create_weights(
123
+ self,
124
+ layer: torch.nn.Module,
125
+ input_size_per_partition: int,
126
+ output_partition_sizes: List[int],
127
+ input_size: int,
128
+ output_size: int,
129
+ params_dtype: torch.dtype,
130
+ **extra_weight_attrs,
131
+ ):
132
+ del output_size # Unused.
133
+
134
+ if params_dtype != torch.float16:
135
+ raise ValueError(
136
+ f"The params dtype must be float16, but got {params_dtype}")
137
+
138
+ # Validate output_size_per_partition
139
+ output_size_per_partition = sum(output_partition_sizes)
140
+ if output_size_per_partition % self.quant_config.min_n_threads != 0:
141
+ raise ValueError(
142
+ f"Weight output_size_per_partition = "
143
+ f"{output_size_per_partition} is not divisible by "
144
+ f"min_n_threads = {self.quant_config.min_n_threads}.")
145
+ if output_size_per_partition % self.quant_config.pack_factor != 0:
146
+ raise ValueError(
147
+ f"Weight output_size_per_partition = "
148
+ f"{output_size_per_partition} is not divisible by "
149
+ f"pack_factor = {self.quant_config.pack_factor}.")
150
+
151
+ # Validate input_size_per_partition
152
+ if input_size_per_partition % self.quant_config.min_k_threads != 0:
153
+ raise ValueError(
154
+ f"Weight input_size_per_partition = "
155
+ f"{input_size_per_partition} is not divisible by "
156
+ f"min_k_threads = {self.quant_config.min_k_threads}.")
157
+ if (self.quant_config.group_size != -1 and
158
+ input_size_per_partition % self.quant_config.group_size != 0):
159
+ raise ValueError(f"Weight input_size_per_partition = "
160
+ f"{input_size_per_partition} is not divisible by "
161
+ f"group_size = {self.quant_config.group_size}.")
162
+
163
+ # Check that we have at least 4 tiles horizontally in the shard
164
+ num_tiles_per_perm = self.quant_config.perm_len // (
165
+ self.quant_config.tile_size**2)
166
+ if output_size_per_partition % num_tiles_per_perm != 0:
167
+ raise ValueError(
168
+ "Each permutation group must reside on the same gpu")
169
+
170
+ # Quantized 4Bit weights packed into Int32.
171
+ qweight = Parameter(
172
+ torch.empty(
173
+ input_size_per_partition // self.quant_config.tile_size // 2,
174
+ output_size_per_partition * self.quant_config.tile_size //
175
+ self.quant_config.pack_factor,
176
+ device="cuda",
177
+ dtype=torch.int32,
178
+ ),
179
+ requires_grad=False,
180
+ )
181
+ set_weight_attrs(
182
+ qweight,
183
+ {
184
+ "input_dim": 0,
185
+ "output_dim": 1,
186
+ "packed_dim": 1,
187
+ "pack_factor": self.quant_config.pack_factor,
188
+ "marlin_tile_size": self.quant_config.tile_size,
189
+ },
190
+ )
191
+
192
+ # Meta
193
+ meta = Parameter(
194
+ torch.empty(
195
+ input_size_per_partition // 8 // 2 // 2,
196
+ output_size_per_partition * 2,
197
+ device="cuda",
198
+ dtype=torch.int16,
199
+ ),
200
+ requires_grad=False,
201
+ )
202
+ set_weight_attrs(
203
+ meta,
204
+ {
205
+ "input_dim": 0,
206
+ "packed_dim": 1,
207
+ "pack_factor": 1,
208
+ "output_dim": 1,
209
+ "marlin_tile_size": 2,
210
+ },
211
+ )
212
+
213
+ # Determine if channelwise or not
214
+ input_groups = (1 if self.quant_config.group_size == -1 else
215
+ input_size_per_partition //
216
+ self.quant_config.group_size)
217
+
218
+ scales = Parameter(
219
+ torch.empty(
220
+ input_groups,
221
+ output_size_per_partition,
222
+ device="cuda",
223
+ dtype=params_dtype,
224
+ ),
225
+ requires_grad=False,
226
+ )
227
+ set_weight_attrs(
228
+ scales,
229
+ {
230
+ "input_dim": None if input_groups == 1 else 0,
231
+ "output_dim": 1,
232
+ },
233
+ )
234
+
235
+ # Allocate workspace (Used for internal locking mechanism)
236
+ max_workspace_size = (
237
+ output_size_per_partition //
238
+ self.quant_config.min_n_threads) * self.quant_config.max_parallel
239
+ workspace = Parameter(torch.zeros(max_workspace_size,
240
+ device="cuda",
241
+ dtype=torch.int),
242
+ requires_grad=False)
243
+
244
+ layer.register_parameter("B_24", qweight)
245
+ set_weight_attrs(qweight, extra_weight_attrs)
246
+ layer.register_parameter("B_meta", meta)
247
+ set_weight_attrs(meta, extra_weight_attrs)
248
+ layer.register_parameter("s", scales)
249
+ set_weight_attrs(scales, extra_weight_attrs)
250
+ layer.register_parameter("workspace", workspace)
251
+ set_weight_attrs(workspace, extra_weight_attrs)
252
+
253
+ def apply(
254
+ self,
255
+ layer: torch.nn.Module,
256
+ x: torch.Tensor,
257
+ bias: Optional[torch.Tensor] = None,
258
+ ) -> torch.Tensor:
259
+ qweight = layer.B_24
260
+ meta = layer.B_meta
261
+ scales = layer.s
262
+ workspace = layer.workspace
263
+
264
+ x_2d = x.view(-1, x.shape[-1])
265
+
266
+ size_m = x_2d.shape[0]
267
+ size_k = x_2d.shape[1]
268
+ size_n = scales.shape[1]
269
+
270
+ output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
271
+ workspace,
272
+ self.quant_config.weight_bits,
273
+ size_m, size_n, size_k)
274
+
275
+ output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
276
+
277
+ if bias is not None:
278
+ output.add_(bias) # In-place add
279
+
280
+ return output
vllm/model_executor/layers/quantization/marlin.py
+22
@@ -4,11 +4,14 @@ import torch
4
from torch.nn.parameter import Parameter
5
6
from vllm import _custom_ops as ops
7
+from vllm.logger import init_logger
8
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
from vllm.model_executor.utils import set_weight_attrs
12
13
+logger = init_logger(__name__)
14
+
15
16
class MarlinConfig(QuantizationConfig):
17
"""Config class for Marlin.
@@ -72,6 +75,25 @@ class MarlinConfig(QuantizationConfig):
75
group_size = cls.get_from_keys(config, ["group_size"])
76
return cls(group_size)
77
78
+ @classmethod
79
+ def override_quantization_method(cls, hf_quant_cfg,
80
+ user_quant) -> Optional[str]:
81
+ # compat: autogptq >=0.8.0 use checkpoint_format: str
82
+ # compat: autogptq <=0.7.1 is_marlin_format: bool
83
+ is_marlin_format = (hf_quant_cfg.get("checkpoint_format") == "marlin"
84
+ or hf_quant_cfg.get("is_marlin_format", False))
85
+
86
+ is_valid_user_quant = (user_quant is None or user_quant == "gptq"
87
+ or user_quant == "marlin")
88
+
89
+ if is_marlin_format and is_valid_user_quant:
90
+ msg = ("The model is serialized in {} format. Using {} kernel.".
91
+ format(cls.get_name(), cls.get_name()))
92
+ logger.info(msg)
93
+ return cls.get_name()
94
+
95
+ return None
96
+
97
def get_quant_method(
98
self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
99
if isinstance(layer, LinearBase):