@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6979ade384
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Add GPTQ Marlin 2:4 sparse structured support (#4790)

Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>

Alexander Matveev committed May 16, 2024 at 12:56 UTC 6979ade3840703d055402e78168194644e3cbdd8
18 files changed +2131 -41
CMakeLists.txt
+2 -1
@@ -176,7 +176,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
176 list(APPEND VLLM_EXT_SRC
177 "csrc/quantization/aqlm/gemm_kernels.cu"
178 "csrc/quantization/awq/gemm_kernels.cu"
179 - "csrc/quantization/marlin/marlin_cuda_kernel.cu"
179 + "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
180 + "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
181 "csrc/quantization/gptq_marlin/gptq_marlin.cu"
182 "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
183 "csrc/custom_all_reduce.cu")
csrc/ops.h
+11
@@ -125,6 +125,17 @@ torch::Tensor marlin_gemm(
125 int64_t size_n,
126 int64_t size_k);
127
128 +torch::Tensor gptq_marlin_24_gemm(
129 + torch::Tensor &a,
130 + torch::Tensor &b_q_weight,
131 + torch::Tensor &b_meta,
132 + torch::Tensor &b_scales,
133 + torch::Tensor &workspace,
134 + int64_t num_bits,
135 + int64_t size_m,
136 + int64_t size_n,
137 + int64_t size_k);
138 +
139 torch::Tensor gptq_marlin_gemm(
140 torch::Tensor &a,
141 torch::Tensor &b_q_weight,
csrc/pybind.cpp
+2 -1
@@ -66,7 +66,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
66 ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
67 ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
68 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69 - ops.def("marlin_gemm", &marlin_gemm, "Marlin Optimized Quantized GEMM for GPTQ");
69 + ops.def("marlin_gemm", &marlin_gemm, "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
70 + ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm, "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
71 ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
72 ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
73 ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
csrc/quantization/marlin/dense/LICENSE renamed
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu renamed
csrc/quantization/marlin/sparse/LICENSE new
+203
@@ -0,0 +1,203 @@
1 +Contains code from https://github.com/IST-DASLab/Sparse-Marlin/
2 +
3 + Apache License
4 + Version 2.0, January 2004
5 + http://www.apache.org/licenses/
6 +
7 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8 +
9 + 1. Definitions.
10 +
11 + "License" shall mean the terms and conditions for use, reproduction,
12 + and distribution as defined by Sections 1 through 9 of this document.
13 +
14 + "Licensor" shall mean the copyright owner or entity authorized by
15 + the copyright owner that is granting the License.
16 +
17 + "Legal Entity" shall mean the union of the acting entity and all
18 + other entities that control, are controlled by, or are under common
19 + control with that entity. For the purposes of this definition,
20 + "control" means (i) the power, direct or indirect, to cause the
21 + direction or management of such entity, whether by contract or
22 + otherwise, or (ii) ownership of fifty percent (50%) or more of the
23 + outstanding shares, or (iii) beneficial ownership of such entity.
24 +
25 + "You" (or "Your") shall mean an individual or Legal Entity
26 + exercising permissions granted by this License.
27 +
28 + "Source" form shall mean the preferred form for making modifications,
29 + including but not limited to software source code, documentation
30 + source, and configuration files.
31 +
32 + "Object" form shall mean any form resulting from mechanical
33 + transformation or translation of a Source form, including but
34 + not limited to compiled object code, generated documentation,
35 + and conversions to other media types.
36 +
37 + "Work" shall mean the work of authorship, whether in Source or
38 + Object form, made available under the License, as indicated by a
39 + copyright notice that is included in or attached to the work
40 + (an example is provided in the Appendix below).
41 +
42 + "Derivative Works" shall mean any work, whether in Source or Object
43 + form, that is based on (or derived from) the Work and for which the
44 + editorial revisions, annotations, elaborations, or other modifications
45 + represent, as a whole, an original work of authorship. For the purposes
46 + of this License, Derivative Works shall not include works that remain
47 + separable from, or merely link (or bind by name) to the interfaces of,
48 + the Work and Derivative Works thereof.
49 +
50 + "Contribution" shall mean any work of authorship, including
51 + the original version of the Work and any modifications or additions
52 + to that Work or Derivative Works thereof, that is intentionally
53 + submitted to Licensor for inclusion in the Work by the copyright owner
54 + or by an individual or Legal Entity authorized to submit on behalf of
55 + the copyright owner. For the purposes of this definition, "submitted"
56 + means any form of electronic, verbal, or written communication sent
57 + to the Licensor or its representatives, including but not limited to
58 + communication on electronic mailing lists, source code control systems,
59 + and issue tracking systems that are managed by, or on behalf of, the
60 + Licensor for the purpose of discussing and improving the Work, but
61 + excluding communication that is conspicuously marked or otherwise
62 + designated in writing by the copyright owner as "Not a Contribution."
63 +
64 + "Contributor" shall mean Licensor and any individual or Legal Entity
65 + on behalf of whom a Contribution has been received by Licensor and
66 + subsequently incorporated within the Work.
67 +
68 + 2. Grant of Copyright License. Subject to the terms and conditions of
69 + this License, each Contributor hereby grants to You a perpetual,
70 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable
71 + copyright license to reproduce, prepare Derivative Works of,
72 + publicly display, publicly perform, sublicense, and distribute the
73 + Work and such Derivative Works in Source or Object form.
74 +
75 + 3. Grant of Patent License. Subject to the terms and conditions of
76 + this License, each Contributor hereby grants to You a perpetual,
77 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78 + (except as stated in this section) patent license to make, have made,
79 + use, offer to sell, sell, import, and otherwise transfer the Work,
80 + where such license applies only to those patent claims licensable
81 + by such Contributor that are necessarily infringed by their
82 + Contribution(s) alone or by combination of their Contribution(s)
83 + with the Work to which such Contribution(s) was submitted. If You
84 + institute patent litigation against any entity (including a
85 + cross-claim or counterclaim in a lawsuit) alleging that the Work
86 + or a Contribution incorporated within the Work constitutes direct
87 + or contributory patent infringement, then any patent licenses
88 + granted to You under this License for that Work shall terminate
89 + as of the date such litigation is filed.
90 +
91 + 4. Redistribution. You may reproduce and distribute copies of the
92 + Work or Derivative Works thereof in any medium, with or without
93 + modifications, and in Source or Object form, provided that You
94 + meet the following conditions:
95 +
96 + (a) You must give any other recipients of the Work or
97 + Derivative Works a copy of this License; and
98 +
99 + (b) You must cause any modified files to carry prominent notices
100 + stating that You changed the files; and
101 +
102 + (c) You must retain, in the Source form of any Derivative Works
103 + that You distribute, all copyright, patent, trademark, and
104 + attribution notices from the Source form of the Work,
105 + excluding those notices that do not pertain to any part of
106 + the Derivative Works; and
107 +
108 + (d) If the Work includes a "NOTICE" text file as part of its
109 + distribution, then any Derivative Works that You distribute must
110 + include a readable copy of the attribution notices contained
111 + within such NOTICE file, excluding those notices that do not
112 + pertain to any part of the Derivative Works, in at least one
113 + of the following places: within a NOTICE text file distributed
114 + as part of the Derivative Works; within the Source form or
115 + documentation, if provided along with the Derivative Works; or,
116 + within a display generated by the Derivative Works, if and
117 + wherever such third-party notices normally appear. The contents
118 + of the NOTICE file are for informational purposes only and
119 + do not modify the License. You may add Your own attribution
120 + notices within Derivative Works that You distribute, alongside
121 + or as an addendum to the NOTICE text from the Work, provided
122 + that such additional attribution notices cannot be construed
123 + as modifying the License.
124 +
125 + You may add Your own copyright statement to Your modifications and
126 + may provide additional or different license terms and conditions
127 + for use, reproduction, or distribution of Your modifications, or
128 + for any such Derivative Works as a whole, provided Your use,
129 + reproduction, and distribution of the Work otherwise complies with
130 + the conditions stated in this License.
131 +
132 + 5. Submission of Contributions. Unless You explicitly state otherwise,
133 + any Contribution intentionally submitted for inclusion in the Work
134 + by You to the Licensor shall be under the terms and conditions of
135 + this License, without any additional terms or conditions.
136 + Notwithstanding the above, nothing herein shall supersede or modify
137 + the terms of any separate license agreement you may have executed
138 + with Licensor regarding such Contributions.
139 +
140 + 6. Trademarks. This License does not grant permission to use the trade
141 + names, trademarks, service marks, or product names of the Licensor,
142 + except as required for reasonable and customary use in describing the
143 + origin of the Work and reproducing the content of the NOTICE file.
144 +
145 + 7. Disclaimer of Warranty. Unless required by applicable law or
146 + agreed to in writing, Licensor provides the Work (and each
147 + Contributor provides its Contributions) on an "AS IS" BASIS,
148 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
149 + implied, including, without limitation, any warranties or conditions
150 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
151 + PARTICULAR PURPOSE. You are solely responsible for determining the
152 + appropriateness of using or redistributing the Work and assume any
153 + risks associated with Your exercise of permissions under this License.
154 +
155 + 8. Limitation of Liability. In no event and under no legal theory,
156 + whether in tort (including negligence), contract, or otherwise,
157 + unless required by applicable law (such as deliberate and grossly
158 + negligent acts) or agreed to in writing, shall any Contributor be
159 + liable to You for damages, including any direct, indirect, special,
160 + incidental, or consequential damages of any character arising as a
161 + result of this License or out of the use or inability to use the
162 + Work (including but not limited to damages for loss of goodwill,
163 + work stoppage, computer failure or malfunction, or any and all
164 + other commercial damages or losses), even if such Contributor
165 + has been advised of the possibility of such damages.
166 +
167 + 9. Accepting Warranty or Additional Liability. While redistributing
168 + the Work or Derivative Works thereof, You may choose to offer,
169 + and charge a fee for, acceptance of support, warranty, indemnity,
170 + or other liability obligations and/or rights consistent with this
171 + License. However, in accepting such obligations, You may act only
172 + on Your own behalf and on Your sole responsibility, not on behalf
173 + of any other Contributor, and only if You agree to indemnify,
174 + defend, and hold each Contributor harmless for any liability
175 + incurred by, or claims asserted against, such Contributor by reason
176 + of your accepting any such warranty or additional liability.
177 +
178 + END OF TERMS AND CONDITIONS
179 +
180 + APPENDIX: How to apply the Apache License to your work.
181 +
182 + To apply the Apache License to your work, attach the following
183 + boilerplate notice, with the fields enclosed by brackets "[]"
184 + replaced with your own identifying information. (Don't include
185 + the brackets!) The text should be enclosed in the appropriate
186 + comment syntax for the file format. We also recommend that a
187 + file or class name and description of purpose be included on the
188 + same "printed page" as the copyright notice for easier
189 + identification within third-party archives.
190 +
191 + Copyright [yyyy] [name of copyright owner]
192 +
193 + Licensed under the Apache License, Version 2.0 (the "License");
194 + you may not use this file except in compliance with the License.
195 + You may obtain a copy of the License at
196 +
197 + http://www.apache.org/licenses/LICENSE-2.0
198 +
199 + Unless required by applicable law or agreed to in writing, software
200 + distributed under the License is distributed on an "AS IS" BASIS,
201 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
202 + See the License for the specific language governing permissions and
203 + limitations under the License.
\ No newline at end of file
csrc/quantization/marlin/sparse/common/base.h new
+49
@@ -0,0 +1,49 @@
1 +/*
2 + * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
3 + * Rights Reserved.
4 + *
5 + * Licensed under the Apache License, Version 2.0 (the "License");
6 + * you may not use this file except in compliance with the License.
7 + * You may obtain a copy of the License at
8 + *
9 + * http://www.apache.org/licenses/LICENSE-2.0
10 + *
11 + * Unless required by applicable law or agreed to in writing, software
12 + * distributed under the License is distributed on an "AS IS" BASIS,
13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 + * See the License for the specific language governing permissions and
15 + * limitations under the License.
16 + */
17 +
18 +#pragma once
19 +
20 +namespace marlin_24 {
21 +
22 +constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
23 +
24 +// Instances of `Vec` are used to organize groups of >>registers<<, as needed
25 +// for instance as inputs to tensor core operations. Consequently, all
26 +// corresponding index accesses must be compile-time constants, which is why we
27 +// extensively use `#pragma unroll` throughout the kernel code to guarantee
28 +// this.
29 +template <typename T, int n> struct Vec {
30 + T elems[n];
31 + __device__ T &operator[](int i) { return elems[i]; }
32 +};
33 +
34 +template <int M_, int N_, int K_> struct ShapeBase {
35 + static constexpr int M = M_, N = N_, K = K_;
36 +};
37 +
38 +using I4 = Vec<int, 4>;
39 +
40 +// Matrix fragments for tensor core instructions; their precise layout is
41 +// documented here:
42 +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
43 +using FragA = Vec<half2, 4>;
44 +using FragB = Vec<half2, 2>;
45 +using FragM = Vec<uint, 1>;
46 +using FragC = Vec<float, 4>;
47 +using FragS = Vec<half2, 1>; // quantization scales
48 +
49 +} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mem.h new
+132
@@ -0,0 +1,132 @@
1 +/*
2 + * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
3 + * Rights Reserved.
4 + *
5 + * Licensed under the Apache License, Version 2.0 (the "License");
6 + * you may not use this file except in compliance with the License.
7 + * You may obtain a copy of the License at
8 + *
9 + * http://www.apache.org/licenses/LICENSE-2.0
10 + *
11 + * Unless required by applicable law or agreed to in writing, software
12 + * distributed under the License is distributed on an "AS IS" BASIS,
13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 + * See the License for the specific language governing permissions and
15 + * limitations under the License.
16 + */
17 +
18 +#pragma once
19 +#include "base.h"
20 +
21 +namespace marlin_24 {
22 +// Predicated asynchronous global->shared copy; used for inputs A where we apply
23 +// predication to handle batchsizes that are not multiples of 16.
24 +__device__ inline void cp_async4_pred_zfill(void *smem_ptr,
25 + const void *glob_ptr,
26 + bool pred = true,
27 + const bool zfill = false) {
28 + const int BYTES = 16;
29 + int src_in_bytes = (zfill ? 0 : BYTES);
30 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
31 + asm volatile("{\n"
32 + " .reg .pred p;\n"
33 + " setp.ne.b32 p, %0, 0;\n"
34 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
35 + "}\n" ::"r"((int)pred),
36 + "r"(smem), "l"(glob_ptr), "n"(BYTES), "r"(src_in_bytes));
37 +}
38 +
39 +__device__ inline void cp_async4_pred(void *smem_ptr, const void *glob_ptr,
40 + bool pred = true) {
41 + const int BYTES = 16;
42 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
43 + asm volatile("{\n"
44 + " .reg .pred p;\n"
45 + " setp.ne.b32 p, %0, 0;\n"
46 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
47 + "}\n" ::"r"((int)pred),
48 + "r"(smem), "l"(glob_ptr), "n"(BYTES));
49 +}
50 +
51 +// Asynchronous global->shared copy
52 +__device__ inline void cp_async4(void *smem_ptr, const void *glob_ptr) {
53 + const int BYTES = 16;
54 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55 + asm volatile("{\n"
56 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
57 + "}\n" ::"r"(smem),
58 + "l"(glob_ptr), "n"(BYTES));
59 +}
60 +
61 +// Async copy fence.
62 +__device__ inline void cp_async_fence() {
63 + asm volatile("cp.async.commit_group;\n" ::);
64 +}
65 +
66 +// Wait until at most `n` async copy stages are still pending.
67 +template <int n> __device__ inline void cp_async_wait() {
68 + asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
69 +}
70 +
71 +// Instruction for loading a full 16x16 matrix fragment of operand A from shared
72 +// memory, directly in tensor core layout.
73 +__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
74 + uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
75 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
76 + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
77 + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
78 + : "r"(smem));
79 +}
80 +
81 +__device__ inline void ldsm4_m(FragM &frag_m, const void *smem_ptr) {
82 + uint32_t *a = reinterpret_cast<uint32_t *>(&frag_m);
83 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
84 + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n"
85 + : "=r"(a[0]), "=r"(a[1])
86 + : "r"(smem));
87 +}
88 +
89 +// Instruction for loading a full 16x16 matrix fragment of operand A from shared
90 +// memory, directly in tensor core layout.
91 +__device__ inline void ldsm4_t(FragA &frag_a, const void *smem_ptr) {
92 + uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
93 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
94 + asm volatile(
95 + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
96 + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
97 + : "r"(smem));
98 +}
99 +
100 +// Wait until barrier reaches `count`, then lock for current threadblock.
101 +__device__ inline void barrier_acquire(int *lock, int count) {
102 + if (threadIdx.x == 0) {
103 + int state = -1;
104 + do
105 + // Guarantee that subsequent writes by this threadblock will be visible
106 + // globally.
107 + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
108 + : "=r"(state)
109 + : "l"(lock));
110 + while (state != count);
111 + }
112 + __syncthreads();
113 +}
114 +
115 +// Release barrier and increment visitation count.
116 +__device__ inline void barrier_release(int *lock, bool reset = false) {
117 + __syncthreads();
118 + if (threadIdx.x == 0) {
119 + if (reset) {
120 + lock[0] = 0;
121 + return;
122 + }
123 + int val = 1;
124 + // Make sure that all writes since acquiring this barrier are visible
125 + // globally, while releasing the barrier.
126 + asm volatile("fence.acq_rel.gpu;\n");
127 + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
128 + :
129 + : "l"(lock), "r"(val));
130 + }
131 +}
132 +} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mma.h new
+175
@@ -0,0 +1,175 @@
1 +/*
2 + * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
3 + * Rights Reserved.
4 + *
5 + * Licensed under the Apache License, Version 2.0 (the "License");
6 + * you may not use this file except in compliance with the License.
7 + * You may obtain a copy of the License at
8 + *
9 + * http://www.apache.org/licenses/LICENSE-2.0
10 + *
11 + * Unless required by applicable law or agreed to in writing, software
12 + * distributed under the License is distributed on an "AS IS" BASIS,
13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 + * See the License for the specific language governing permissions and
15 + * limitations under the License.
16 + */
17 +
18 +#pragma once
19 +#include "base.h"
20 +
21 +namespace marlin_24 {
22 +
23 +// m16n8k32 sparse tensor core mma instruction with fp16 inputs and fp32
24 +// output/accumulation.
25 +__device__ inline void mma_sp(const FragB &a_frag0, const FragB &a_frag1,
26 + const FragA &frag_b, FragC &frag_c, FragM &frag_m,
27 + const int psel) {
28 + const uint32_t *a0 = reinterpret_cast<const uint32_t *>(&a_frag0);
29 + const uint32_t *a1 = reinterpret_cast<const uint32_t *>(&a_frag1);
30 + const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
31 + const uint32_t *e = reinterpret_cast<const uint32_t *>(&frag_m);
32 + float *c = reinterpret_cast<float *>(&frag_c);
33 + if (psel == 0) {
34 + asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
35 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
36 + "{%12,%13,%14,%15}, %16, 0x0;\n"
37 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
38 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
39 + "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
40 + "f"(c[2]), "f"(c[3]), "r"(e[0]));
41 + asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
42 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
43 + "{%12,%13,%14,%15}, %16, 0x0;\n"
44 + : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
45 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
46 + "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
47 + "f"(c[6]), "f"(c[7]), "r"(e[0]));
48 + } else {
49 + asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
50 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
51 + "{%12,%13,%14,%15}, %16, 0x1;\n"
52 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
53 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
54 + "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
55 + "f"(c[2]), "f"(c[3]), "r"(e[0]));
56 + asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
57 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
58 + "{%12,%13,%14,%15}, %16, 0x1;\n"
59 + : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
60 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
61 + "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
62 + "f"(c[6]), "f"(c[7]), "r"(e[0]));
63 + }
64 +}
65 +
66 +// Lookup-table based 3-input logical operation; explicitly used for
67 +// dequantization as the compiler does not seem to automatically recognize it in
68 +// all cases.
69 +template <int lut> __device__ inline int lop3(int a, int b, int c) {
70 + int res;
71 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
72 + : "=r"(res)
73 + : "r"(a), "r"(b), "r"(c), "n"(lut));
74 + return res;
75 +}
76 +
77 +__device__ __forceinline__ uint2 to_half4(float c0, float c1, float c2,
78 + float c3) {
79 + uint2 r;
80 + asm("{\n\t"
81 + ".reg .f16 a, b, c, d; \n\t"
82 + "cvt.rn.f16.f32 a, %2; \n\t"
83 + "cvt.rn.f16.f32 b, %3; \n\t"
84 + "cvt.rn.f16.f32 c, %4; \n\t"
85 + "cvt.rn.f16.f32 d, %5; \n\t"
86 + "mov.b32 %0, {a, b}; \n\t"
87 + "mov.b32 %1, {c, d}; \n\t"
88 + "}"
89 + : "=r"(r.x), "=r"(r.y)
90 + : "f"(c0), "f"(c1), "f"(c2), "f"(c3));
91 + return r;
92 +}
93 +
94 +// Constructs destination register by taking bytes from 2 sources (based on
95 +// mask)
96 +template <int start_byte, int mask>
97 +__device__ inline uint32_t prmt(uint32_t a) {
98 + uint32_t res;
99 + asm volatile("prmt.b32 %0, %1, %2, %3;\n"
100 + : "=r"(res)
101 + : "r"(a), "n"(start_byte), "n"(mask));
102 + return res;
103 +}
104 +
105 +// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
106 +// values. We mostly follow the strategy in the link below, with some small
107 +// changes:
108 +// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
109 +__device__ inline FragB dequant_4bit(int q) {
110 + const int LO = 0x000f000f;
111 + const int HI = 0x00f000f0;
112 + const int EX = 0x64006400;
113 + // Guarantee that the `(a & b) | c` operations are LOP3s.
114 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
115 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
116 + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
117 + // directly into `SUB` and `ADD`.
118 + const int SUB = 0x64086408;
119 + const int MUL = 0x2c002c00;
120 + const int ADD = 0xd480d480;
121 +
122 + FragB frag_b;
123 + frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
124 + *reinterpret_cast<const half2 *>(&SUB));
125 + frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
126 + *reinterpret_cast<const half2 *>(&MUL),
127 + *reinterpret_cast<const half2 *>(&ADD));
128 + return frag_b;
129 +}
130 +
131 +// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
132 +// values. We mostly follow the strategy in the link below, with some small
133 +// changes:
134 +// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
135 +__device__ inline FragB dequant_8bit(int q) {
136 + static constexpr uint32_t mask_for_elt_01 = 0x5250;
137 + static constexpr uint32_t mask_for_elt_23 = 0x5351;
138 + static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
139 +
140 + uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
141 + uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
142 +
143 + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
144 +
145 + FragB frag_b;
146 + frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
147 + *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
148 + frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
149 + *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
150 + return frag_b;
151 +}
152 +
153 +// Multiply dequantized values by the corresponding quantization scale; used
154 +// only for grouped quantization.
155 +__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
156 + half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
157 + frag_b[0] = __hmul2(frag_b[0], s);
158 + frag_b[1] = __hmul2(frag_b[1], s);
159 +}
160 +
161 +__device__ inline void scale_floats(float *c0, float *c1, float *c2, float *c3,
162 + FragS &s0, float *c4, float *c5, float *c6,
163 + float *c7, FragS &s1) {
164 + *c0 = __fmul_rn(*c0, __half2float(s0[0].x));
165 + *c1 = __fmul_rn(*c1, __half2float(s0[0].y));
166 + *c2 = __fmul_rn(*c2, __half2float(s0[1].x));
167 + *c3 = __fmul_rn(*c3, __half2float(s0[1].y));
168 +
169 + *c4 = __fmul_rn(*c4, __half2float(s1[0].x));
170 + *c5 = __fmul_rn(*c5, __half2float(s1[0].y));
171 + *c6 = __fmul_rn(*c6, __half2float(s1[1].x));
172 + *c7 = __fmul_rn(*c7, __half2float(s1[1].y));
173 +}
174 +
175 +} // namespace marlin_24
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu new
+1110
@@ -0,0 +1,1110 @@
1 +/*
2 + * Notice: This file was modified by Neuralmagic inc to include 8-bit support
3 + *
4 + * Copyright (C) 2024 Roberto Lopez Castro (roberto.lopez.castro@udc.es). All
5 + * Rights Reserved.
6 + *
7 + * Licensed under the Apache License, Version 2.0 (the "License");
8 + * you may not use this file except in compliance with the License.
9 + * You may obtain a copy of the License at
10 + *
11 + * http://www.apache.org/licenses/LICENSE-2.0
12 + *
13 + * Unless required by applicable law or agreed to in writing, software
14 + * distributed under the License is distributed on an "AS IS" BASIS,
15 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 + * See the License for the specific language governing permissions and
17 + * limitations under the License.
18 + */
19 +#include <torch/extension.h>
20 +
21 +#include <ATen/cuda/CUDAContext.h>
22 +#include <c10/cuda/CUDAGuard.h>
23 +#include <cuda.h>
24 +#include <cuda_fp16.h>
25 +#include <cuda_runtime.h>
26 +
27 +#include <iostream>
28 +
29 +#include "common/base.h"
30 +
31 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
32 +
33 +#else
34 +
35 +#include "common/mem.h"
36 +#include "common/mma.h"
37 +
38 +#endif
39 +
40 +template <typename T> inline std::string str(T x) { return std::to_string(x); }
41 +
42 +namespace marlin_24 {
43 +
44 +// 8 warps are a good choice since every SM has 4 schedulers and having more
45 +// than 1 warp per schedule allows some more latency hiding. At the same time,
46 +// we want relatively few warps to have many registers per warp and small tiles.
47 +static constexpr int THREADS = 256;
48 +static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
49 +
50 +static constexpr int min_thread_n = 128;
51 +
52 +static constexpr int tile_size = 16;
53 +static constexpr int max_par = 16;
54 +
55 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
56 +
57 +template <const int num_bits, // weight bits
58 + const int threads, // number of threads in a threadblock
59 + const int thread_m_blocks, // number of 16x16 blocks in the m
60 + // dimension (batchsize) of the threadblock
61 + const int thread_n_blocks, // same for n dimension (output)
62 + const int thread_k_blocks, // same for k dimension (reduction)
63 + const int stages, // number of stages for the async global->shared
64 + // fetch pipeline
65 + const int group_blocks = -1 // number of consecutive 16x16 blocks with
66 + // a separate quantization scale
67 + >
68 +__global__ void Marlin_24(
69 + const int4 *__restrict__ A, // fp16 input matrix of shape mxk
70 + const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
71 + const int4
72 + *__restrict__ meta, // 2bit metadata information about 2:4 format on B
73 + int4 *__restrict__ C, // fp16 output buffer of shape mxn
74 + const int4
75 + *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
76 + int prob_m, // batch dimension m
77 + int prob_n, // output dimension n
78 + int prob_k, // reduction dimension k
79 + int *locks // extra global storage for barrier synchronization
80 +) {}
81 +
82 +torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
83 + torch::Tensor &b_meta,
84 + torch::Tensor &b_scales,
85 + torch::Tensor &workspace, int64_t num_bits,
86 + int64_t size_m, int64_t size_n,
87 + int64_t size_k) {
88 + TORCH_CHECK_NOT_IMPLEMENTED(
89 + false, "gptq_marlin_24_gemm(..) requires CUDA_ARCH >= 8.0");
90 + return torch::empty({1, 1});
91 +}
92 +
93 +#else
94 +
95 +template <const int num_bits, // weight bits
96 + const int threads, // number of threads in a threadblock
97 + const int thread_m_blocks, // number of 16x16 blocks in the m
98 + // dimension (batchsize) of the threadblock
99 + const int thread_n_blocks, // same for n dimension (output)
100 + const int thread_k_blocks, // same for k dimension (reduction)
101 + const int stages, // number of stages for the async global->shared
102 + // fetch pipeline
103 + const int group_blocks = -1 // number of consecutive 16x16 blocks with
104 + // a separate quantization scale
105 + >
106 +__global__ void Marlin_24(
107 + const int4 *__restrict__ A, // fp16 input matrix of shape mxk
108 + const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
109 + const int4
110 + *__restrict__ meta, // 2bit metadata information about 2:4 format on B
111 + int4 *__restrict__ C, // fp16 output buffer of shape mxn
112 + const int4
113 + *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
114 + int prob_m, // batch dimension m
115 + int prob_n, // output dimension n
116 + int prob_k, // reduction dimension k
117 + int *locks // extra global storage for barrier synchronization
118 +) {
119 + // Each threadblock processes one "stripe" of the B matrix with (roughly) the
120 + // same size, which might involve multiple column "slices" (of width 16 *
121 + // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM
122 + // example:
123 + // 0 1 3
124 + // 0 2 3
125 + // 1 2 4
126 + // While this kind of partitioning makes things somewhat more complicated, it
127 + // ensures good utilization of all SMs for many kinds of shape and GPU
128 + // configurations, while requiring as few slow global cross-threadblock
129 + // reductions as possible.
130 +
131 + // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
132 + // better partitioning with less reductions
133 + int parallel = 1;
134 + if (prob_m > 16 * thread_m_blocks) {
135 + parallel = prob_m / (16 * thread_m_blocks);
136 + prob_m = 16 * thread_m_blocks;
137 + }
138 +
139 + // number of thread_k_blocks in k-dim
140 + int k_tiles = prob_k / 32 / thread_k_blocks;
141 + // number of thread_n_blocks in n-dim
142 + int n_tiles = prob_n / 16 / thread_n_blocks;
143 + // iters needed to cover all slices
144 + int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
145 +
146 + // Ensure that the number of tiles in each stripe is a multiple of the
147 + // groupsize; this avoids an annoying special case where a stripe starts in
148 + // the middle of group.
149 + if (group_blocks != -1)
150 + iters = (group_blocks / thread_k_blocks) *
151 + ceildiv(iters, (group_blocks / thread_k_blocks));
152 +
153 + int slice_row = (iters * blockIdx.x) % k_tiles;
154 + int slice_col_par = (iters * blockIdx.x) / k_tiles;
155 + int slice_col = slice_col_par;
156 + // number of threadblock tiles in the current slice
157 + int slice_iters;
158 + // total number of active threadblocks in the current slice
159 + int slice_count = 0;
160 + // index of threadblock in current slice; numbered bottom to top
161 + int slice_idx;
162 +
163 + // We can easily implement parallel problem execution by just remapping
164 + // indices and advancing global pointers
165 + if (slice_col_par >= n_tiles) {
166 + A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_k / 8;
167 + C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
168 + locks += (slice_col_par / n_tiles) * n_tiles;
169 + slice_col = slice_col_par % n_tiles;
170 + }
171 +
172 + // Compute all information about the current slice which is required for
173 + // synchronization.
174 + auto init_slice = [&]() {
175 + slice_iters =
176 + iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
177 + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
178 + slice_iters = 0;
179 + if (slice_iters == 0)
180 + return;
181 + if (slice_row + slice_iters > k_tiles)
182 + slice_iters = k_tiles - slice_row;
183 + slice_count = 1;
184 + slice_idx = 0;
185 + int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
186 + if (col_first <= k_tiles * (slice_col_par + 1)) {
187 + int col_off = col_first - k_tiles * slice_col_par;
188 + slice_count = ceildiv(k_tiles - col_off, iters);
189 + if (col_off > 0)
190 + slice_count++;
191 + int delta_first = iters * blockIdx.x - col_first;
192 + if (delta_first < 0 || (col_off == 0 && delta_first == 0))
193 + slice_idx = slice_count - 1;
194 + else {
195 + slice_idx = slice_count - 1 - delta_first / iters;
196 + if (col_off > 0)
197 + slice_idx--;
198 + }
199 + }
200 + if (slice_col == n_tiles) {
201 + A += 16 * thread_m_blocks * prob_k / 8;
202 + C += 16 * thread_m_blocks * prob_n / 8;
203 + locks += n_tiles;
204 + slice_col = 0;
205 + }
206 + };
207 + init_slice();
208 +
209 + // RLC: 8 is vec_size -> 128-bit instructions, 8 fp16 elements
210 + int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
211 +
212 + // stride of an A matrix tile in shared memory
213 + constexpr int a_sh_stride = 32 * thread_k_blocks / 8;
214 + // delta between subsequent A tiles in global memory
215 + constexpr int a_gl_rd_delta_o = 32 * thread_k_blocks / 8;
216 + // between subsequent accesses within a tile
217 + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
218 + // between shared memory writes
219 + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
220 + // between shared memory tile reads //RLC: 2 * #warps k-dim
221 + constexpr int a_sh_rd_delta_o = 4 * ((threads / 32) / (thread_n_blocks / 4));
222 + // within a shared memory tile
223 + constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
224 + // overall size of a tile
225 + constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
226 + // number of shared write iterations for a tile
227 + constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
228 +
229 + constexpr int pack_factor = 32 / num_bits;
230 +
231 + int b_gl_stride = 16 * prob_n / (pack_factor * 4);
232 + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
233 + constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
234 + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
235 + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
236 + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
237 + constexpr int b_sh_wr_delta = threads * b_thread_vecs;
238 + constexpr int b_sh_rd_delta = threads * b_thread_vecs;
239 + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
240 + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
241 +
242 + int m_gl_stride = 2 * prob_n / 8; // (16*2*4 / 8) = 16
243 + constexpr int m_sh_stride =
244 + (16 * thread_n_blocks) / 4; // #warps n-dim * threads/warp
245 + int m_gl_rd_delta_o = m_gl_stride * thread_k_blocks;
246 + int m_gl_rd_delta_i = m_gl_stride * (threads / m_sh_stride);
247 + constexpr int m_sh_wr_delta = threads / 2;
248 + constexpr int m_sh_rd_delta = threads / 2;
249 + constexpr int m_sh_stage = m_sh_stride * thread_k_blocks;
250 + constexpr int m_sh_iters = ceildiv(m_sh_stage, m_sh_wr_delta);
251 +
252 + int s_gl_stride = prob_n / 8;
253 + constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
254 + constexpr int s_sh_stage = s_sh_stride;
255 + int s_gl_rd_delta = s_gl_stride;
256 +
257 + // Global A read index of current thread.
258 + int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
259 + (threadIdx.x % a_gl_rd_delta_o);
260 + a_gl_rd += a_gl_rd_delta_o * slice_row;
261 + // Shared write index of current thread.
262 + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
263 + (threadIdx.x % a_gl_rd_delta_o);
264 + // Shared read index.
265 + int a_sh_rd =
266 + a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
267 + a_sh_rd += 4 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
268 +
269 + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
270 + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
271 + b_gl_rd += b_sh_stride * slice_col;
272 + b_gl_rd += b_gl_rd_delta_o * slice_row;
273 + int b_sh_wr = threadIdx.x * b_thread_vecs;
274 + int b_sh_rd = threadIdx.x * b_thread_vecs;
275 +
276 + int m_gl_rd = m_gl_stride * (threadIdx.x / (m_sh_stride)) +
277 + (threadIdx.x % (m_sh_stride));
278 + m_gl_rd += (m_sh_stride)*slice_col;
279 + m_gl_rd += m_gl_rd_delta_o * slice_row;
280 + int m_sh_wr = threadIdx.x;
281 + int m_sh_rd = threadIdx.x % 16 + (threadIdx.x / 32) * 16;
282 +
283 + int s_gl_rd;
284 + if constexpr (group_blocks == -1) {
285 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
286 + } else {
287 + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
288 + s_sh_stride * slice_col + threadIdx.x;
289 + }
290 +
291 + int s_sh_wr = threadIdx.x;
292 + int s_sh_rd;
293 + // We use a different scale layout for grouped and column-wise quantization as
294 + // we scale a `half2` tile in column-major layout in the former and in
295 + // row-major in the latter case.
296 + if (group_blocks != -1) {
297 + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
298 + (threadIdx.x % 32) / 4;
299 + } else {
300 + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
301 + (threadIdx.x % 32) / 4;
302 + }
303 +
304 + // Precompute which thread should not read memory in which iterations; this is
305 + // needed if there are more threads than required for a certain tilesize or
306 + // when the batchsize is not a multiple of 16.
307 + bool a_sh_wr_pred[a_sh_wr_iters];
308 +#pragma unroll
309 + for (int i = 0; i < a_sh_wr_iters; i++) {
310 + a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
311 + }
312 + bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
313 +
314 + // To ensure that writing and reading A tiles to/from shared memory, the
315 + // latter in fragment format, is fully bank conflict free, we need to use a
316 + // rather fancy XOR-based layout. The key here is that neither reads nor
317 + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
318 + // same shared memory banks. Further, it seems (based on NSight-Compute) that
319 + // each warp must also write a consecutive memory segment?
320 + auto transform_a = [&](int i) {
321 + int row = i / a_gl_rd_delta_o;
322 + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
323 + };
324 + // Since the computation of this remapping is non-trivial and, due to our main
325 + // loop unrolls, all shared memory accesses are static, we simply precompute
326 + // both transformed reads and writes.
327 + int a_sh_wr_trans[a_sh_wr_iters];
328 +#pragma unroll
329 + for (int i = 0; i < a_sh_wr_iters; i++)
330 + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
331 + int a_sh_rd_trans[2][b_sh_wr_iters][thread_m_blocks];
332 +#pragma unroll
333 + for (int i = 0; i < b_sh_wr_iters; i++) {
334 +#pragma unroll
335 + for (int j = 0; j < thread_m_blocks; j++) {
336 + a_sh_rd_trans[0][i][j] =
337 + transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
338 + a_sh_rd_trans[1][i][j] =
339 + transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd + 2);
340 + }
341 + }
342 +
343 + // Since B-accesses have non-constant stride they have to be computed at
344 + // runtime; we break dependencies between subsequent accesses with a tile by
345 + // maintining multiple pointers (we have enough registers), a tiny
346 + // optimization.
347 + const int4 *B_ptr[b_sh_wr_iters];
348 +#pragma unroll
349 + for (int i = 0; i < b_sh_wr_iters; i++)
350 + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
351 +
352 + bool m_sh_wr_pred = threadIdx.x < m_sh_wr_delta;
353 + const int4 *meta_ptr[m_sh_iters];
354 +#pragma unroll
355 + for (int i = 0; i < m_sh_iters; i++)
356 + meta_ptr[i] = meta + m_gl_rd_delta_i * i + m_gl_rd;
357 +
358 + extern __shared__ int4 sh[];
359 + // Shared memory storage for global fetch pipelines.
360 + int4 *sh_a = sh;
361 + int4 *sh_b = sh_a + (stages * a_sh_stage);
362 + int4 *sh_s = sh_b + (stages * b_sh_stage);
363 + int4 *sh_m = sh_s + (stages * s_sh_stage);
364 + // Register storage for double buffer of shared memory reads.
365 + FragA frag_a[2][thread_m_blocks][2];
366 + I4 frag_b_quant[2][b_thread_vecs];
367 + FragM frag_m[2][2];
368 + FragC frag_c[thread_m_blocks][4][2];
369 + FragS frag_s[2][4];
370 +
371 + // Zero accumulators.
372 + auto zero_accums = [&]() {
373 +#pragma unroll
374 + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
375 + reinterpret_cast<float *>(frag_c)[i] = 0;
376 + };
377 +
378 + // Asynchronously fetch the next A, B and s tile from global to the next
379 + // shared memory pipeline location.
380 + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
381 + if (pred) {
382 + int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
383 +#pragma unroll
384 + for (int i = 0; i < a_sh_wr_iters; i++) {
385 + cp_async4_pred(
386 + &sh_a_stage[a_sh_wr_trans[i]],
387 + &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
388 + a_sh_wr_pred[i]);
389 + }
390 + int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
391 +#pragma unroll
392 + for (int i = 0; i < b_sh_wr_iters; i++) {
393 +#pragma unroll
394 + for (int j = 0; j < b_thread_vecs; j++) {
395 + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j],
396 + B_ptr[i] + j);
397 + }
398 + B_ptr[i] += b_gl_rd_delta_o;
399 + }
400 + int4 *sh_meta_stage = sh_m + m_sh_stage * pipe;
401 +#pragma unroll
402 + for (int i = 0; i < m_sh_iters; i++) {
403 + if (m_sh_wr_pred)
404 + cp_async4(&sh_meta_stage[m_sh_wr_delta * i + m_sh_wr],
405 + meta_ptr[i]);
406 + meta_ptr[i] += m_gl_rd_delta_o;
407 + }
408 + // Only fetch scales if this tile starts a new group
409 + if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
410 + int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
411 + if (s_sh_wr_pred)
412 + cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
413 + s_gl_rd += s_gl_rd_delta;
414 + }
415 + }
416 + // Insert a fence even when we are winding down the pipeline to ensure that
417 + // waiting is also correct at this point.
418 + cp_async_fence();
419 + };
420 +
421 + // Wait until the next thread tile has been loaded to shared memory.
422 + auto wait_for_stage = [&]() {
423 + // We only have `stages - 2` active fetches since we are double buffering
424 + // and can only issue the next fetch when it is guaranteed that the previous
425 + // shared memory load is fully complete (as it may otherwise be
426 + // overwritten).
427 + cp_async_wait<stages - 2>();
428 + __syncthreads();
429 + };
430 +
431 + // Load the next sub-tile from the current location in the shared memory pipe
432 + // into the current register buffer.
433 + auto fetch_to_registers = [&](int k, int pipe) {
434 + // It may seem inefficient that we reload the groups for every sub-tile;
435 + // however, this does not seem to be a significant bottleneck, while some
436 + // theoretically better attempts have lead to bad instruction ordering by
437 + // the compiler and correspondingly a noticeable drop in performance.
438 + if (group_blocks != -1) {
439 + int4 *sh_s_stage =
440 + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
441 + (pipe / (group_blocks / thread_k_blocks)));
442 + reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
443 + }
444 + int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
445 +#pragma unroll
446 + for (int i = 0; i < thread_m_blocks; i++) {
447 + ldsm4(frag_a[k % 2][i][0],
448 + &sh_a_stage[a_sh_rd_trans[0][k % b_sh_wr_iters][i]]);
449 + ldsm4(frag_a[k % 2][i][1],
450 + &sh_a_stage[a_sh_rd_trans[1][k % b_sh_wr_iters][i]]);
451 + }
452 +
453 + int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
454 +#pragma unroll
455 + for (int i = 0; i < b_thread_vecs; i++) {
456 + frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
457 + &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
458 + }
459 +
460 + // Load meta with ldsm4
461 + int4 *sh_m_stage = sh_m + m_sh_stage * pipe;
462 + ldsm4_m(frag_m[k % 2][0],
463 + &sh_m_stage[m_sh_rd_delta * (k % m_sh_iters) + m_sh_rd]);
464 + };
465 +
466 + // Execute the actual tensor core matmul of a sub-tile.
467 + auto matmul = [&](int k) {
468 +// We have the m dimension as the inner loop in order to encourage overlapping
469 +// dequantization and matmul operations.
470 +#pragma unroll
471 + for (int j = 0; j < 4; j++) {
472 + FragB frag_b0;
473 + FragB frag_b1;
474 +
475 + if constexpr (num_bits == 4) {
476 + int b_quant = frag_b_quant[k % 2][0][j];
477 + int b_quant_shift = b_quant >> 8;
478 +
479 + frag_b0 = dequant_4bit(b_quant);
480 + frag_b1 = dequant_4bit(b_quant_shift);
481 +
482 + } else {
483 + int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
484 + int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
485 + int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
486 +
487 + frag_b0 = dequant_8bit(b_quant_0);
488 + frag_b1 = dequant_8bit(b_quant_1);
489 + }
490 +
491 + // If there are no groups, we can just scale the final output once and can
492 + // avoid doing so for each weight.
493 + if constexpr (group_blocks != -1) {
494 + scale(frag_b0, frag_s[k % 2][j], 0);
495 + }
496 + if constexpr (group_blocks != -1) {
497 + scale(frag_b1, frag_s[k % 2][j], 1);
498 + }
499 +
500 +#pragma unroll
501 + for (int i = 0; i < thread_m_blocks; i++) {
502 + mma_sp(frag_b0, frag_b1, frag_a[k % 2][i][0], frag_c[i][j][0],
503 + frag_m[k % 2][j / 2], j % 2);
504 + }
505 + }
506 + };
507 +
508 + // Since we slice across the k dimension of a tile in order to increase the
509 + // number of warps while keeping the n dimension of a tile reasonable, we have
510 + // multiple warps that accumulate their partial sums of the same output
511 + // location; which we have to reduce over in the end. We do in shared memory.
512 + auto thread_block_reduce = [&]() {
513 + constexpr int red_off = threads / b_sh_stride_threads / 2;
514 + if (red_off >= 1) {
515 + int red_idx = threadIdx.x / b_sh_stride_threads;
516 + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
517 + constexpr int red_sh_delta = b_sh_stride_threads;
518 + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
519 + (threadIdx.x % b_sh_stride_threads);
520 +
521 +// Parallel logarithmic shared memory reduction. We make sure to avoid any
522 +// unnecessary read or write iterations, e.g., for two warps we write only once
523 +// by warp 1 and read only once by warp 0.
524 +#pragma unroll
525 + for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
526 +#pragma unroll
527 + for (int i = red_off; i > 0; i /= 2) {
528 + if (i <= red_idx && red_idx < 2 * i) {
529 +#pragma unroll
530 + for (int j = 0; j < 4 * 2; j++) {
531 + int red_sh_wr =
532 + red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
533 + if (i < red_off) {
534 + float *c_rd = reinterpret_cast<float *>(
535 + &sh[red_sh_delta * j + red_sh_rd]);
536 + float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
537 +#pragma unroll
538 + for (int k = 0; k < 4; k++)
539 + reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
540 + c_rd[k] + c_wr[k];
541 + }
542 + sh[red_sh_wr] =
543 + reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
544 + }
545 + }
546 + __syncthreads();
547 + }
548 + if (red_idx == 0) {
549 +#pragma unroll
550 + for (int i = 0; i < 4 * 2; i++) {
551 + float *c_rd =
552 + reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
553 +#pragma unroll
554 + for (int j = 0; j < 4; j++)
555 + reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
556 + c_rd[j];
557 + }
558 + }
559 + __syncthreads();
560 + }
561 + }
562 + };
563 +
564 + // Since multiple threadblocks may process parts of the same column slice, we
565 + // finally have to globally reduce over the results. As the striped partitioning
566 + // minimizes the number of such reductions and our outputs are usually rather
567 + // small, we perform this reduction serially in L2 cache.
568 + auto global_reduce = [&](bool first = false, bool last = false) {
569 + // We are very careful here to reduce directly in the output buffer to
570 + // maximize L2 cache utilization in this step. To do this, we write out
571 + // results in FP16 (but still reduce with FP32 compute).
572 + constexpr int active_threads = 32 * thread_n_blocks / 4;
573 + if (threadIdx.x < active_threads) {
574 + int c_gl_stride = prob_n / 8;
575 + int c_gl_wr_delta_o = 2 * 4 * c_gl_stride;
576 + int c_gl_wr_delta_i =
577 + c_gl_stride; // 8 threads (e.g., 0,4,8,12,16,20,24,28)
578 + int c_gl_wr = 2 * c_gl_stride * (threadIdx.x % 4) +
579 + 8 * (threadIdx.x / 32) + (threadIdx.x % 32) / 4;
580 + c_gl_wr += (2 * thread_n_blocks) * slice_col;
581 + constexpr int c_sh_wr_delta = active_threads;
582 + int c_sh_wr = threadIdx.x;
583 +
584 + int col = 2 * ((threadIdx.x % 32) % 4);
585 +
586 + if (!first) {
587 +// Interestingly, doing direct global accesses here really seems to mess up the
588 +// compiler and lead to slowdowns, hence we also use async-copies even though
589 +// these fetches are not actually asynchronous.
590 +#pragma unroll
591 + for (int i = 0; i < thread_m_blocks * 4; i++) {
592 + cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
593 + &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
594 + c_gl_wr_delta_i * (i % 2)],
595 + i < (thread_m_blocks - 1) * 4 ||
596 + 8 * (i / 2) + col + (i % 2) < prob_m);
597 + }
598 + cp_async_fence();
599 + cp_async_wait<0>();
600 + }
601 +
602 +#pragma unroll
603 + for (int i = 0; i < thread_m_blocks * 4; i++) {
604 + if (i < (thread_m_blocks - 1) * 4 ||
605 + 8 * (i / 2) + col + (i % 2) < prob_m) {
606 + if (!first) {
607 + int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
608 +#pragma unroll
609 + for (int j2 = 0; j2 < 2; j2++) {
610 +#pragma unroll
611 + for (int j1 = 0; j1 < 4; j1++) {
612 + reinterpret_cast<float *>(
613 + &frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
614 + 4 * ((i % 4) / 2) + i % 2] +=
615 + __half2float(
616 + reinterpret_cast<__half *>(&c_red)[(j2 * 4 + j1)]);
617 + }
618 + }
619 + }
620 + if (!last) {
621 + int4 c;
622 +#pragma unroll
623 + for (int j2 = 0; j2 < 2; j2++) {
624 +#pragma unroll
625 + for (int j1 = 0; j1 < 4; j1++) {
626 + reinterpret_cast<__half *>(&c)[(j2 * 4 + j1)] =
627 + __float2half(reinterpret_cast<float *>(
628 + &frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
629 + 4 * ((i % 4) / 2) + i % 2]);
630 + }
631 + }
632 + C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
633 + c;
634 + }
635 + }
636 + }
637 + }
638 + };
639 +
640 + // Write out the reduce final result in the correct layout. We only actually
641 + // reshuffle matrix fragments in this step, the reduction above is performed
642 + // in fragment layout.
643 + auto write_result = [&]() {
644 + int c_gl_stride = prob_n / 8;
645 +
646 + constexpr int c_sh_stride = 2 * thread_n_blocks; // RLC:
647 + constexpr int c_sh_stride_2 = 2 * c_sh_stride + 2; // RLC:
648 + constexpr int c_sh_stride_3 = 2 * (2 * thread_n_blocks) + 2; // RLC:
649 +
650 + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
651 +
652 + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
653 + (threadIdx.x % (2 * thread_n_blocks));
654 + c_gl_wr += (2 * thread_n_blocks) * slice_col;
655 +
656 + int c_sh_wr = c_sh_stride_2 * ((threadIdx.x % 32) % 4) +
657 + ((threadIdx.x % 32) / 4); // RLC:
658 + c_sh_wr += 8 * (threadIdx.x / 32); // 128/4(half4)
659 +
660 + constexpr int c_sh_rd_delta =
661 + c_sh_stride_3 * (threads / (2 * 2 * thread_n_blocks)); // RLC:
662 + int c_sh_rd = c_sh_stride_3 * (threadIdx.x / (2 * 2 * thread_n_blocks)) +
663 + (threadIdx.x % (2 * 2 * thread_n_blocks));
664 +
665 + int c_gl_wr_end = c_gl_stride * prob_m;
666 +
667 + auto write = [&](int idx, float c0, float c1, float c2, float c3, FragS &s0,
668 + float c4, float c5, float c6, float c7, FragS &s1) {
669 + uint2 res[2];
670 + res[0] = to_half4(c0, c1, c2, c3);
671 + res[1] = to_half4(c4, c5, c6, c7);
672 + half2 *tmp = (half2 *)&res;
673 + // for per-column quantization we finally apply the scale here
674 + if constexpr (group_blocks == -1 && num_bits == 4) {
675 + tmp[0] = __hmul2(tmp[0], s0[0]);
676 + tmp[1] = __hmul2(tmp[1], s0[1]);
677 + tmp[2] = __hmul2(tmp[2], s1[0]);
678 + tmp[3] = __hmul2(tmp[3], s1[1]);
679 + }
680 + ((int4 *)sh)[idx] = *((int4 *)&res[0]);
681 + };
682 +
683 + // RLC: only warp 0 and 1 baseline example
684 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
685 +#pragma unroll
686 + for (int i = 0; i < thread_m_blocks; i++) {
687 + int wr = c_sh_wr;
688 + write(wr, frag_c[i][0][0][0], frag_c[i][1][0][0], frag_c[i][2][0][0],
689 + frag_c[i][3][0][0], frag_s[0][0], frag_c[i][0][0][2],
690 + frag_c[i][1][0][2], frag_c[i][2][0][2], frag_c[i][3][0][2],
691 + frag_s[0][2]);
692 + write(wr + c_sh_stride, frag_c[i][0][0][1], frag_c[i][1][0][1],
693 + frag_c[i][2][0][1], frag_c[i][3][0][1], frag_s[0][0],
694 + frag_c[i][0][0][3], frag_c[i][1][0][3], frag_c[i][2][0][3],
695 + frag_c[i][3][0][3], frag_s[0][2]);
696 + write(wr + 4 * c_sh_stride_2, frag_c[i][0][1][0], frag_c[i][1][1][0],
697 + frag_c[i][2][1][0], frag_c[i][3][1][0], frag_s[0][0],
698 + frag_c[i][0][1][2], frag_c[i][1][1][2], frag_c[i][2][1][2],
699 + frag_c[i][3][1][2], frag_s[0][2]);
700 + write(wr + 4 * c_sh_stride_2 + c_sh_stride, frag_c[i][0][1][1],
701 + frag_c[i][1][1][1], frag_c[i][2][1][1], frag_c[i][3][1][1],
702 + frag_s[0][0], frag_c[i][0][1][3], frag_c[i][1][1][3],
703 + frag_c[i][2][1][3], frag_c[i][3][1][3], frag_s[0][2]);
704 +
705 + c_sh_wr += 8 * c_sh_stride_2;
706 + }
707 + }
708 + __syncthreads();
709 +
710 +#pragma unroll
711 + for (int i = 0;
712 + i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
713 + i++) {
714 + if (c_gl_wr < c_gl_wr_end) {
715 + C[c_gl_wr] = sh[c_sh_rd];
716 + c_gl_wr += c_gl_wr_delta;
717 + c_sh_rd += c_sh_rd_delta;
718 + }
719 + }
720 + };
721 +
722 + // Start global fetch and register load pipelines.
723 + auto start_pipes = [&]() {
724 +#pragma unroll
725 + for (int i = 0; i < stages - 1; i++)
726 + fetch_to_shared(i, i, i < slice_iters);
727 + zero_accums();
728 + wait_for_stage();
729 + fetch_to_registers(0, 0);
730 + a_gl_rd += a_gl_rd_delta_o * (stages - 1);
731 + };
732 + start_pipes();
733 +
734 + // Main loop.
735 + while (slice_iters) {
736 +// We unroll over both the global fetch and the register load pipeline to ensure
737 +// all shared memory accesses are static. Note that both pipelines have even
738 +// length meaning that the next iteration will always start at index 0.
739 +#pragma unroll
740 + for (int pipe = 0; pipe < stages;) {
741 + fetch_to_shared((pipe + stages - 1) % stages, pipe,
742 + slice_iters >= stages);
743 + wait_for_stage();
744 +
745 + fetch_to_registers(pipe + 1, (pipe + 1) % stages);
746 + matmul(pipe);
747 +
748 + pipe++;
749 + slice_iters--;
750 + if (slice_iters == 0)
751 + break;
752 + }
753 + a_gl_rd += a_gl_rd_delta_o * stages;
754 +
755 + // Process results and, if necessary, proceed to the next column slice.
756 + // While this pattern may not be the most readable, other ways of writing
757 + // the loop seemed to noticeably worse performance after compilation.
758 + if (slice_iters == 0) {
759 + cp_async_wait<0>();
760 + bool last = slice_idx == slice_count - 1;
761 + // For per-column scales, we only fetch them here in the final step before
762 + // write-out
763 + if constexpr (group_blocks == -1) {
764 + if constexpr (num_bits == 8) {
765 + if (s_sh_wr_pred)
766 + cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
767 + cp_async_fence();
768 + } else {
769 + if (last) {
770 + if (s_sh_wr_pred)
771 + cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
772 + cp_async_fence();
773 + }
774 + }
775 + }
776 + thread_block_reduce();
777 +
778 + if constexpr (group_blocks == -1) {
779 + if constexpr (num_bits == 8) {
780 + cp_async_wait<0>();
781 + __syncthreads();
782 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
783 + *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
784 + }
785 + } else {
786 + if (last) {
787 + cp_async_wait<0>();
788 + __syncthreads();
789 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
790 + *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
791 + }
792 + }
793 + }
794 + }
795 +
796 + // For 8-bit channelwise, we apply the scale before the global reduction
797 + // that converts the fp32 results to fp16 (so that we avoid possible
798 + // overflow in fp16)
799 + if constexpr (group_blocks == -1 && num_bits == 8) {
800 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
801 +#pragma unroll
802 + for (int i = 0; i < thread_m_blocks; i++) {
803 + scale_floats(&frag_c[i][0][0][0], &frag_c[i][1][0][0],
804 + &frag_c[i][2][0][0], &frag_c[i][3][0][0], frag_s[0][0],
805 + &frag_c[i][0][0][2], &frag_c[i][1][0][2],
806 + &frag_c[i][2][0][2], &frag_c[i][3][0][2],
807 + frag_s[0][2]);
808 +
809 + scale_floats(&frag_c[i][0][0][1], &frag_c[i][1][0][1],
810 + &frag_c[i][2][0][1], &frag_c[i][3][0][1], frag_s[0][0],
811 + &frag_c[i][0][0][3], &frag_c[i][1][0][3],
812 + &frag_c[i][2][0][3], &frag_c[i][3][0][3],
813 + frag_s[0][2]);
814 +
815 + scale_floats(&frag_c[i][0][1][0], &frag_c[i][1][1][0],
816 + &frag_c[i][2][1][0], &frag_c[i][3][1][0], frag_s[0][0],
817 + &frag_c[i][0][1][2], &frag_c[i][1][1][2],
818 + &frag_c[i][2][1][2], &frag_c[i][3][1][2],
819 + frag_s[0][2]);
820 +
821 + scale_floats(&frag_c[i][0][1][1], &frag_c[i][1][1][1],
822 + &frag_c[i][2][1][1], &frag_c[i][3][1][1], frag_s[0][0],
823 + &frag_c[i][0][1][3], &frag_c[i][1][1][3],
824 + &frag_c[i][2][1][3], &frag_c[i][3][1][3],
825 + frag_s[0][2]);
826 + }
827 + }
828 + }
829 +
830 + if (slice_count > 1) { // only globally reduce if there is more than one
831 + // block in a slice
832 + barrier_acquire(&locks[slice_col], slice_idx);
833 + global_reduce(slice_idx == 0, last);
834 + barrier_release(&locks[slice_col], last);
835 + }
836 + if (last) // only the last block in a slice actually writes the result
837 + write_result();
838 +
839 + slice_row = 0;
840 + slice_col_par++;
841 + slice_col++;
842 + init_slice();
843 + if (slice_iters) {
844 + a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
845 + (threadIdx.x % a_gl_rd_delta_o);
846 +#pragma unroll
847 + for (int i = 0; i < b_sh_wr_iters; i++)
848 + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
849 +#pragma unroll
850 + for (int i = 0; i < m_sh_iters; i++)
851 + meta_ptr[i] += (m_sh_stride)-m_gl_rd_delta_o * k_tiles;
852 + if (slice_col == 0) {
853 +#pragma unroll
854 + for (int i = 0; i < b_sh_wr_iters; i++)
855 + B_ptr[i] -= b_gl_stride;
856 +#pragma unroll
857 + for (int i = 0; i < m_sh_iters; i++)
858 + meta_ptr[i] -= m_gl_stride;
859 + }
860 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
861 + start_pipes();
862 + }
863 + }
864 + }
865 +}
866 +
867 +#endif
868 +
869 +#define CALL_IF_2_4(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
870 + THREAD_K_BLOCKS, GROUP_BLOCKS) \
871 + else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
872 + thread_n_blocks == THREAD_N_BLOCKS && \
873 + thread_k_blocks == THREAD_K_BLOCKS && \
874 + group_blocks == GROUP_BLOCKS) { \
875 + cudaFuncSetAttribute( \
876 + Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
877 + THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
878 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
879 + Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
880 + THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS> \
881 + <<<blocks, THREADS, max_shared_mem, stream>>>(A_ptr, B_ptr, meta_ptr, \
882 + C_ptr, s_ptr, prob_n, \
883 + prob_m, prob_k, locks); \
884 + }
885 +
886 +void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
887 + void *s, int prob_m, int prob_n, int prob_k,
888 + void *workspace, int num_bits, int groupsize = -1,
889 + int dev = 0, cudaStream_t stream = 0, int thread_k = -1,
890 + int thread_m = -1, int sms = -1, int max_par = 16) {
891 + int tot_n = prob_n;
892 + int tot_n_blocks = ceildiv(tot_n, 16);
893 + int pad = 16 * tot_n_blocks - tot_n;
894 +
895 + if (sms == -1) {
896 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
897 + }
898 + TORCH_CHECK(sms > 0);
899 +
900 + int max_shared_mem = 0;
901 + cudaDeviceGetAttribute(&max_shared_mem,
902 + cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
903 + TORCH_CHECK(max_shared_mem > 0);
904 +
905 + if (thread_k == -1 || thread_m == -1) {
906 + if (prob_n <= 16) {
907 + // For small batchizes, better partitioningif is slightly more important than
908 + // better compute utilization
909 + thread_k = 128;
910 + thread_m = 128;
911 + } else {
912 + thread_k = 64;
913 + thread_m = 256;
914 + }
915 + }
916 +
917 + int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
918 + int thread_m_blocks = thread_m / 16;
919 + int group_blocks = (groupsize == -1) ? -1 : groupsize / 16;
920 + int blocks = sms;
921 +
922 + TORCH_CHECK(prob_m % thread_m == 0, "prob_m = ", prob_m,
923 + " is not divisible by thread_m = ", thread_m);
924 + TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
925 + " is not divisible by thread_k = ", thread_k);
926 + if (group_blocks != -1) {
927 + TORCH_CHECK((prob_k / 2) % group_blocks == 0, "prob_k/2 = ", prob_k / 2,
928 + " is not divisible by group_blocks = ", group_blocks);
929 + }
930 +
931 + TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
932 + ", ", prob_n, ", ", prob_k, "]");
933 +
934 + const int4 *A_ptr = (const int4 *)A;
935 + const int4 *B_ptr = (const int4 *)B;
936 + const int4 *meta_ptr = (const int4 *)meta;
937 + int4 *C_ptr = (int4 *)C;
938 + const int4 *s_ptr = (const int4 *)s;
939 +
940 + int *locks = (int *)workspace;
941 + for (int i = 0; i < tot_n_blocks; i += 4) {
942 + int thread_n_blocks = tot_n_blocks - i;
943 + prob_n = tot_n - 16 * i;
944 + int par = 1;
945 + if (thread_n_blocks > 4) {
946 + // Note that parallel > 1 currently only works for inputs without any
947 + // padding
948 + par = (16 * thread_n_blocks - pad) / 64;
949 + if (par > max_par)
950 + par = max_par;
951 + prob_n = 64 * par;
952 + i += 4 * (par - 1);
953 + thread_n_blocks = 4;
954 + }
955 +
956 + // For compilation speed, we only define the kernel configurations that have
957 + // seemed useful (in terms of performance) in our testing, however many more
958 + // are, in principle, possible.
959 +
960 + // the false is start of the CALL_IF macros
961 + if (false) {
962 + } // BMxBNxBK, group
963 + // 4-bit
964 + CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
965 + CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
966 + CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
967 + CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
968 + CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
969 + CALL_IF_2_4(4, 16, 2, 2, 4)
970 + CALL_IF_2_4(4, 16, 3, 2, -1)
971 + CALL_IF_2_4(4, 16, 3, 2, 4)
972 + CALL_IF_2_4(4, 16, 4, 2, -1)
973 + CALL_IF_2_4(4, 16, 4, 2, 4)
974 +
975 + // 8-bit
976 + CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
977 + CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
978 + CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
979 + CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
980 + CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
981 + CALL_IF_2_4(8, 16, 2, 2, 4)
982 + CALL_IF_2_4(8, 16, 3, 2, -1)
983 + CALL_IF_2_4(8, 16, 3, 2, 4)
984 + CALL_IF_2_4(8, 16, 4, 2, -1)
985 + CALL_IF_2_4(8, 16, 4, 2, 4)
986 + else {
987 + throw std::runtime_error("Unsupported shapes: MKN = [" + str(prob_m) +
988 + ", " + str(prob_k) + ", " + str(prob_n) + "]" +
989 + ", groupsize = " + str(groupsize) +
990 + ", thread_m_blocks = " + str(thread_m_blocks) +
991 + ", thread_n_blocks = " + str(thread_n_blocks) +
992 + ", thread_k_blocks = " + str(thread_k_blocks));
993 + }
994 +
995 + A_ptr += 16 * thread_n_blocks * (prob_k / 8) * par;
996 + C_ptr += 16 * thread_n_blocks * (prob_m / 8) * par;
997 + }
998 +}
999 +
1000 +} // namespace marlin_24
1001 +
1002 +torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1003 + torch::Tensor &b_meta,
1004 + torch::Tensor &b_scales,
1005 + torch::Tensor &workspace, int64_t num_bits,
1006 + int64_t size_m, int64_t size_n,
1007 + int64_t size_k) {
1008 + // Verify num_bits
1009 + TORCH_CHECK(num_bits == 4 || num_bits == 8,
1010 + "num_bits must be 4 or 8. Got = ", num_bits);
1011 + int pack_factor = 32 / num_bits;
1012 +
1013 + // Verify M
1014 + TORCH_CHECK(size_m == a.size(0),
1015 + "Shape mismatch: a.size(0) = " + str(a.size(0)) +
1016 + ", size_m = " + str(size_m));
1017 +
1018 + // Verify K
1019 + TORCH_CHECK(size_k == a.size(1),
1020 + "Shape mismatch: a.size(1) = " + str(a.size(1)) +
1021 + ", size_k = " + str(size_k));
1022 + TORCH_CHECK(size_k % marlin_24::tile_size == 0,
1023 + "size_k = " + str(size_k) + " is not divisible by tile_size = " +
1024 + str(marlin_24::tile_size));
1025 + TORCH_CHECK((size_k / marlin_24::tile_size / 2) == b_q_weight.size(0),
1026 + "Shape mismatch: b_q_weight.size(0) = " +
1027 + str(b_q_weight.size(0)) + ", size_k = " + str(size_k) +
1028 + ", tile_size = " + str(marlin_24::tile_size));
1029 +
1030 + // Verify N
1031 + TORCH_CHECK(b_scales.size(1) == size_n,
1032 + "b_scales.size(1) = " + str(b_scales.size(1)) +
1033 + ", size_n = " + str(size_n));
1034 + TORCH_CHECK(
1035 + b_q_weight.size(1) % marlin_24::tile_size == 0,
1036 + "b_q_weight.size(1) = " + str(b_q_weight.size(1)) +
1037 + " is not divisible by tile_size = " + str(marlin_24::tile_size));
1038 +
1039 + int actual_size_n = (b_q_weight.size(1) / marlin_24::tile_size) * pack_factor;
1040 + TORCH_CHECK(size_n == actual_size_n,
1041 + "size_n = " + str(size_n) +
1042 + ", actual_size_n = " + str(actual_size_n));
1043 +
1044 + // Verify meta
1045 + TORCH_CHECK(b_meta.size(0) == size_k / 8 / 2 / 2,
1046 + "b_meta.size(0) = ", b_meta.size(0),
1047 + " is not size_k / 8 / 2 / 2 = ", size_k / 8 / 2 / 2);
1048 + TORCH_CHECK(b_meta.size(1) == size_n * 2, "b_meta.size(1) = ", b_meta.size(1),
1049 + " is not size_n * 2 = ", size_n * 2);
1050 +
1051 + // Verify A device and strides
1052 + TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1053 + TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1054 +
1055 + // Verify B device and strides
1056 + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
1057 + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
1058 +
1059 + // Verify b_meta device and strides
1060 + TORCH_CHECK(b_meta.device().is_cuda(), "b_meta is not on GPU");
1061 + TORCH_CHECK(b_meta.is_contiguous(), "b_meta is not contiguous");
1062 +
1063 + // Verify scales device and strides
1064 + TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
1065 + TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
1066 +
1067 + // Alloc C matrix
1068 + const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1069 + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1070 + torch::Tensor c = torch::empty({size_m, size_n}, options);
1071 +
1072 + int thread_k = -1;
1073 + int thread_m = -1;
1074 + int sms = -1;
1075 + int max_par = 16;
1076 +
1077 + int groupsize = -1;
1078 + if (b_scales.size(0) > 1) {
1079 + TORCH_CHECK(size_k % b_scales.size(0) == 0,
1080 + "size_k = " + str(size_k) +
1081 + ", is not divisible by b_scales.size(0) = " +
1082 + str(b_scales.size(0)));
1083 + groupsize = size_k / b_scales.size(0);
1084 + groupsize /= 2; // Because of 24
1085 + }
1086 +
1087 + // Verify groupsize
1088 + TORCH_CHECK(groupsize == -1 || groupsize == 64,
1089 + "Unexpected groupsize = " + str(groupsize));
1090 +
1091 + // Verify workspace size
1092 + TORCH_CHECK(size_n % marlin_24::min_thread_n == 0,
1093 + "size_n = " + str(size_n) +
1094 + ", is not divisible by min_thread_n = " +
1095 + str(marlin_24::min_thread_n));
1096 + int min_workspace_size =
1097 + (size_n / marlin_24::min_thread_n) * marlin_24::max_par;
1098 + TORCH_CHECK(workspace.numel() >= min_workspace_size,
1099 + "workspace.numel = " + str(workspace.numel()) +
1100 + " is below min_workspace_size = " + str(min_workspace_size));
1101 +
1102 + int dev = a.get_device();
1103 + marlin_24::marlin_cuda_2_4(
1104 + a.data_ptr(), b_q_weight.data_ptr(), b_meta.data_ptr(), c.data_ptr(),
1105 + b_scales.data_ptr(), size_n, size_m, size_k, workspace.data_ptr(),
1106 + num_bits, groupsize, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1107 + thread_m, sms, max_par);
1108 +
1109 + return c;
1110 +}
tests/models/test_gptq_marlin_24.py new
+81
@@ -0,0 +1,81 @@
1 +"""Compare the outputs of a GPTQ model to a Marlin_24 model.
2 +
3 +Note: GPTQ and Marlin_24 do not have bitwise correctness.
4 +As a result, in this test, we just confirm that the top selected tokens of the
5 +Marlin/GPTQ models are in the top 3 selections of each other.
6 +
7 +Run `pytest tests/models/test_marlin_24.py`.
8 +"""
9 +from dataclasses import dataclass
10 +
11 +import pytest
12 +import torch
13 +
14 +from tests.models.utils import check_logprobs_close
15 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 +
17 +capability = torch.cuda.get_device_capability()
18 +capability = capability[0] * 10 + capability[1]
19 +marlin_not_supported = (capability <
20 + QUANTIZATION_METHODS["marlin"].get_min_capability())
21 +
22 +
23 +@dataclass
24 +class ModelPair:
25 + model_marlin: str
26 + model_gptq: str
27 +
28 +
29 +model_pairs = [
30 + # 4-bit, group_size == 128
31 + ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-g128",
32 + model_gptq="alexm-nm/tinyllama-24-gptq-4bit-g128"),
33 + # 4-bit, group_size == channelwise
34 + ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-channelwise",
35 + model_gptq="alexm-nm/tinyllama-24-gptq-4bit-channelwise"),
36 +
37 + # 8-bit, group_size == 128
38 + ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-g128",
39 + model_gptq="alexm-nm/tinyllama-24-gptq-8bit-g128"),
40 + # 8-bit, group_size == channelwise
41 + ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-channelwise",
42 + model_gptq="alexm-nm/tinyllama-24-gptq-8bit-channelwise"),
43 +]
44 +
45 +
46 +@pytest.mark.flaky(reruns=2)
47 +@pytest.mark.skipif(marlin_not_supported,
48 + reason="Marlin24 is not supported on this GPU type.")
49 +@pytest.mark.parametrize("model_pair", model_pairs)
50 +@pytest.mark.parametrize("dtype", ["half"])
51 +@pytest.mark.parametrize("max_tokens", [8])
52 +@pytest.mark.parametrize("num_logprobs", [5])
53 +def test_models(
54 + vllm_runner,
55 + example_prompts,
56 + model_pair: ModelPair,
57 + dtype: str,
58 + max_tokens: int,
59 + num_logprobs: int,
60 +) -> None:
61 + marlin_24_model = vllm_runner(model_pair.model_marlin,
62 + dtype=dtype,
63 + quantization="gptq_marlin_24")
64 + marlin_24_outputs = marlin_24_model.generate_greedy_logprobs(
65 + example_prompts, max_tokens, num_logprobs)
66 + del marlin_24_model
67 +
68 + gptq_model = vllm_runner(model_pair.model_gptq,
69 + dtype=dtype,
70 + quantization="gptq")
71 + gptq_outputs = gptq_model.generate_greedy_logprobs(example_prompts,
72 + max_tokens,
73 + num_logprobs)
74 + del gptq_model
75 +
76 + check_logprobs_close(
77 + outputs_0_lst=gptq_outputs,
78 + outputs_1_lst=marlin_24_outputs,
79 + name_0="gptq",
80 + name_1="marlin_24",
81 + )
vllm/_custom_ops.py
+10
@@ -153,6 +153,16 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
153 size_n, size_k)
154
155
156 +# marlin_24
157 +def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
158 + b_meta: torch.Tensor, b_scales: torch.Tensor,
159 + workspace: torch.Tensor, num_bits: int, size_m: int,
160 + size_n: int, size_k: int) -> torch.Tensor:
161 + return vllm_ops.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
162 + workspace, num_bits, size_m, size_n,
163 + size_k)
164 +
165 +
166 # aqlm
167 def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
168 codebooks: torch.Tensor, scales: torch.Tensor,
vllm/config.py
+12 -36
@@ -7,14 +7,11 @@ import torch
7 from transformers import PretrainedConfig
8
9 from vllm.logger import init_logger
10 -from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
11 - get_quantization_config)
10 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11 from vllm.model_executor.models import ModelRegistry
12 from vllm.transformers_utils.config import get_config, get_hf_text_config
13 from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
14
16 -GPTQMarlinConfig = get_quantization_config("gptq_marlin")
17 -
15 if TYPE_CHECKING:
16 from ray.util.placement_group import PlacementGroup
17
@@ -155,37 +152,15 @@ class ModelConfig:
152 quant_cfg = getattr(self.hf_config, "quantization_config", None)
153 if quant_cfg is not None:
154 quant_method = quant_cfg.get("quant_method", "").lower()
158 - # compat: autogptq >=0.8.0 use checkpoint_format: str
159 - # compat: autogptq <=0.7.1 is_marlin_format: bool
160 - is_format_marlin = (quant_cfg.get("checkpoint_format") == "marlin"
161 - or quant_cfg.get("is_marlin_format", False))
162 -
163 - # Check which LinearMethod the GPTQ model should use.
164 - if quant_method == "gptq":
165 - # If serialized in Marlin format, use MarlinLinearMethod.
166 - # TODO (@robertgshaw): migrate under GPTQMarlinLinearMethod.
167 - if is_format_marlin:
168 - logger.info("The model is serialized in Marlin format. "
169 - "Using Marlin kernel.")
170 - quant_method = "marlin"
171 - if self.quantization == "gptq":
172 - self.quantization = quant_method
173 -
174 - # If convertible to Marlin format, use GPTQMarlinLinearMethod
175 - # unless the user explicitly specified GPTQLinearMethod.
176 - elif GPTQMarlinConfig.is_marlin_compatible(quant_cfg):
177 - if self.quantization == "gptq":
178 - logger.warning(
179 - "The model is convertible to Marlin format, but "
180 - "you specified quantization=gptq. Use "
181 - "quantization=marlin for faster inference.")
182 - else:
183 - logger.info(
184 - "The model is convertible to Marlin format. "
185 - "Using Marlin kernel.")
186 - quant_method = "gptq_marlin"
187 - if self.quantization == "marlin":
188 - self.quantization = quant_method
155 +
156 + # Detect which checkpoint is it
157 + for name, method in QUANTIZATION_METHODS.items():
158 + quantization_override = method.override_quantization_method(
159 + quant_cfg, self.quantization)
160 + if quantization_override:
161 + quant_method = quantization_override
162 + self.quantization = quantization_override
163 + break
164
165 # Verify quantization configurations.
166 if self.quantization is None:
@@ -207,7 +182,8 @@ class ModelConfig:
182 raise ValueError(
183 f"{self.quantization} quantization is currently not "
184 f"supported in ROCm.")
210 - if (self.quantization not in ["marlin", "gptq_marlin"]):
185 + if (self.quantization
186 + not in ["marlin", "gptq_marlin_24", "gptq_marlin"]):
187 logger.warning(
188 "%s quantization is not fully "
189 "optimized yet. The speed can be slower than "
vllm/model_executor/layers/quantization/__init__.py
+8 -3
@@ -10,18 +10,23 @@ from vllm.model_executor.layers.quantization.fp8 import Fp8Config
10 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
11 from vllm.model_executor.layers.quantization.gptq_marlin import (
12 GPTQMarlinConfig)
13 +from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
14 + GPTQMarlin24Config)
15 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
16 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
17
18 QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
19 "aqlm": AQLMConfig,
20 "awq": AWQConfig,
21 + "deepspeedfp": DeepSpeedFPConfig,
22 "fp8": Fp8Config,
23 + # The order of gptq methods is important for config.py iteration over
24 + # override_quantization_method(..)
25 + "marlin": MarlinConfig,
26 + "gptq_marlin_24": GPTQMarlin24Config,
27 + "gptq_marlin": GPTQMarlinConfig,
28 "gptq": GPTQConfig,
29 "squeezellm": SqueezeLLMConfig,
22 - "gptq_marlin": GPTQMarlinConfig,
23 - "marlin": MarlinConfig,
24 - "deepspeedfp": DeepSpeedFPConfig
30 }
31
32
vllm/model_executor/layers/quantization/base_config.py
+11
@@ -66,6 +66,17 @@ class QuantizationConfig(ABC):
66 """Create a config class from the model's quantization config."""
67 raise NotImplementedError
68
69 + @classmethod
70 + def override_quantization_method(cls, hf_quant_cfg,
71 + user_quant) -> Optional[str]:
72 + """
73 + Detects if this quantization method can support a given checkpoint
74 + format by overriding the user specified quantization method --
75 + this method should only be overwritten by subclasses in exceptional
76 + circumstances
77 + """
78 + return None
79 +
80 @staticmethod
81 def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
82 """Get a value from the model's quantization config."""
vllm/model_executor/layers/quantization/gptq_marlin.py
+23
@@ -6,11 +6,14 @@ import torch
6 from torch.nn.parameter import Parameter
7
8 from vllm import _custom_ops as ops
9 +from vllm.logger import init_logger
10 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14
15 +logger = init_logger(__name__)
16 +
17 GPTQ_MARLIN_TILE = 16
18 GPTQ_MARLIN_MIN_THREAD_N = 64
19 GPTQ_MARLIN_MIN_THREAD_K = 128
@@ -117,6 +120,26 @@ class GPTQMarlinConfig(QuantizationConfig):
120 is_sym = cls.get_from_keys(config, ["sym"])
121 return cls(weight_bits, group_size, desc_act, is_sym)
122
123 + @classmethod
124 + def override_quantization_method(cls, hf_quant_cfg,
125 + user_quant) -> Optional[str]:
126 + can_convert = cls.is_marlin_compatible(hf_quant_cfg)
127 +
128 + is_valid_user_quant = (user_quant is None or user_quant == "marlin")
129 +
130 + if can_convert and is_valid_user_quant:
131 + msg = ("The model is convertible to {} during runtime."
132 + " Using {} kernel.".format(cls.get_name(), cls.get_name()))
133 + logger.info(msg)
134 + return cls.get_name()
135 +
136 + if can_convert and user_quant == "gptq":
137 + logger.info("Detected that the model can run with gptq_marlin"
138 + ", however you specified quantization=gptq explicitly,"
139 + " so forcing gptq. Use quantization=gptq_marlin for"
140 + " faster inference")
141 + return None
142 +
143 def get_quant_method(
144 self,
145 layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
vllm/model_executor/layers/quantization/gptq_marlin_24.py new
+280
@@ -0,0 +1,280 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm import _custom_ops as ops
7 +from vllm.logger import init_logger
8 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 +from vllm.model_executor.layers.quantization.base_config import (
10 + QuantizationConfig)
11 +from vllm.model_executor.utils import set_weight_attrs
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class GPTQMarlin24Config(QuantizationConfig):
17 + """Config class for Marlin24.
18 + """
19 +
20 + def __init__(
21 + self,
22 + weight_bits: int,
23 + group_size: int,
24 + ) -> None:
25 + self.weight_bits = weight_bits
26 + self.group_size = group_size
27 +
28 + if self.weight_bits != 4 and self.weight_bits != 8:
29 + raise ValueError("weight_bits must be 4 or 8. Got = {}".format(
30 + self.weight_bits))
31 +
32 + if self.group_size != 128 and self.group_size != -1:
33 + raise ValueError(
34 + "Currently, only group size 128 and -1 (channelwise) "
35 + "is supported for Marlin24, but got group_size of "
36 + f"{self.group_size}")
37 +
38 + # 4 Bits packed into 32 bit datatype.
39 + self.pack_factor = 32 // self.weight_bits
40 +
41 + # Tile size used by marlin kernels.
42 + self.tile_size = 16
43 +
44 + # Min out_features dim
45 + self.min_n_threads = 128
46 +
47 + # Min in_features dim
48 + self.min_k_threads = 128
49 +
50 + # Max parallel problems to solve at once (improves large
51 + # batch performance)
52 + self.max_parallel = 16
53 +
54 + # Permutation length used by the marlin kernels.
55 + self.perm_len = 1024
56 +
57 + def __repr__(self) -> str:
58 + return "Marlin24Config(weight_bits={}, group_size={})".format(
59 + self.weight_bits, self.group_size)
60 +
61 + @classmethod
62 + def get_name(cls) -> str:
63 + return "gptq_marlin_24"
64 +
65 + @classmethod
66 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
67 + return [torch.half]
68 +
69 + @classmethod
70 + # Need to figure it out
71 + def get_min_capability(cls) -> int:
72 + return 80
73 +
74 + @classmethod
75 + def get_config_filenames(cls) -> List[str]:
76 + return ["quantize_config.json"]
77 +
78 + @classmethod
79 + def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlin24Config":
80 + weight_bits = cls.get_from_keys(config, ["bits"])
81 + group_size = cls.get_from_keys(config, ["group_size"])
82 + return cls(weight_bits, group_size)
83 +
84 + @classmethod
85 + def override_quantization_method(cls, hf_quant_cfg,
86 + user_quant) -> Optional[str]:
87 + is_marlin_24_format = (
88 + hf_quant_cfg.get("checkpoint_format") == "marlin_24")
89 +
90 + is_valid_user_quant = (user_quant is None or user_quant == "gptq"
91 + or user_quant == "gptq_marlin_24")
92 +
93 + if is_marlin_24_format and is_valid_user_quant:
94 + msg = ("The model is serialized in {} format. "
95 + "Using {} kernel.".format(cls.get_name(), cls.get_name()))
96 + logger.info(msg)
97 + return cls.get_name()
98 +
99 + return None
100 +
101 + def get_quant_method(
102 + self,
103 + layer: torch.nn.Module) -> Optional["GPTQMarlin24LinearMethod"]:
104 + if isinstance(layer, LinearBase):
105 + return GPTQMarlin24LinearMethod(self)
106 + return None
107 +
108 + def get_scaled_act_names(self) -> List[str]:
109 + return []
110 +
111 +
112 +class GPTQMarlin24LinearMethod(LinearMethodBase):
113 + """Linear method for Marlin24.
114 +
115 + Args:
116 + quant_config: The Marlin24 quantization config.
117 + """
118 +
119 + def __init__(self, quant_config: GPTQMarlin24Config):
120 + self.quant_config = quant_config
121 +
122 + def create_weights(
123 + self,
124 + layer: torch.nn.Module,
125 + input_size_per_partition: int,
126 + output_partition_sizes: List[int],
127 + input_size: int,
128 + output_size: int,
129 + params_dtype: torch.dtype,
130 + **extra_weight_attrs,
131 + ):
132 + del output_size # Unused.
133 +
134 + if params_dtype != torch.float16:
135 + raise ValueError(
136 + f"The params dtype must be float16, but got {params_dtype}")
137 +
138 + # Validate output_size_per_partition
139 + output_size_per_partition = sum(output_partition_sizes)
140 + if output_size_per_partition % self.quant_config.min_n_threads != 0:
141 + raise ValueError(
142 + f"Weight output_size_per_partition = "
143 + f"{output_size_per_partition} is not divisible by "
144 + f"min_n_threads = {self.quant_config.min_n_threads}.")
145 + if output_size_per_partition % self.quant_config.pack_factor != 0:
146 + raise ValueError(
147 + f"Weight output_size_per_partition = "
148 + f"{output_size_per_partition} is not divisible by "
149 + f"pack_factor = {self.quant_config.pack_factor}.")
150 +
151 + # Validate input_size_per_partition
152 + if input_size_per_partition % self.quant_config.min_k_threads != 0:
153 + raise ValueError(
154 + f"Weight input_size_per_partition = "
155 + f"{input_size_per_partition} is not divisible by "
156 + f"min_k_threads = {self.quant_config.min_k_threads}.")
157 + if (self.quant_config.group_size != -1 and
158 + input_size_per_partition % self.quant_config.group_size != 0):
159 + raise ValueError(f"Weight input_size_per_partition = "
160 + f"{input_size_per_partition} is not divisible by "
161 + f"group_size = {self.quant_config.group_size}.")
162 +
163 + # Check that we have at least 4 tiles horizontally in the shard
164 + num_tiles_per_perm = self.quant_config.perm_len // (
165 + self.quant_config.tile_size**2)
166 + if output_size_per_partition % num_tiles_per_perm != 0:
167 + raise ValueError(
168 + "Each permutation group must reside on the same gpu")
169 +
170 + # Quantized 4Bit weights packed into Int32.
171 + qweight = Parameter(
172 + torch.empty(
173 + input_size_per_partition // self.quant_config.tile_size // 2,
174 + output_size_per_partition * self.quant_config.tile_size //
175 + self.quant_config.pack_factor,
176 + device="cuda",
177 + dtype=torch.int32,
178 + ),
179 + requires_grad=False,
180 + )
181 + set_weight_attrs(
182 + qweight,
183 + {
184 + "input_dim": 0,
185 + "output_dim": 1,
186 + "packed_dim": 1,
187 + "pack_factor": self.quant_config.pack_factor,
188 + "marlin_tile_size": self.quant_config.tile_size,
189 + },
190 + )
191 +
192 + # Meta
193 + meta = Parameter(
194 + torch.empty(
195 + input_size_per_partition // 8 // 2 // 2,
196 + output_size_per_partition * 2,
197 + device="cuda",
198 + dtype=torch.int16,
199 + ),
200 + requires_grad=False,
201 + )
202 + set_weight_attrs(
203 + meta,
204 + {
205 + "input_dim": 0,
206 + "packed_dim": 1,
207 + "pack_factor": 1,
208 + "output_dim": 1,
209 + "marlin_tile_size": 2,
210 + },
211 + )
212 +
213 + # Determine if channelwise or not
214 + input_groups = (1 if self.quant_config.group_size == -1 else
215 + input_size_per_partition //
216 + self.quant_config.group_size)
217 +
218 + scales = Parameter(
219 + torch.empty(
220 + input_groups,
221 + output_size_per_partition,
222 + device="cuda",
223 + dtype=params_dtype,
224 + ),
225 + requires_grad=False,
226 + )
227 + set_weight_attrs(
228 + scales,
229 + {
230 + "input_dim": None if input_groups == 1 else 0,
231 + "output_dim": 1,
232 + },
233 + )
234 +
235 + # Allocate workspace (Used for internal locking mechanism)
236 + max_workspace_size = (
237 + output_size_per_partition //
238 + self.quant_config.min_n_threads) * self.quant_config.max_parallel
239 + workspace = Parameter(torch.zeros(max_workspace_size,
240 + device="cuda",
241 + dtype=torch.int),
242 + requires_grad=False)
243 +
244 + layer.register_parameter("B_24", qweight)
245 + set_weight_attrs(qweight, extra_weight_attrs)
246 + layer.register_parameter("B_meta", meta)
247 + set_weight_attrs(meta, extra_weight_attrs)
248 + layer.register_parameter("s", scales)
249 + set_weight_attrs(scales, extra_weight_attrs)
250 + layer.register_parameter("workspace", workspace)
251 + set_weight_attrs(workspace, extra_weight_attrs)
252 +
253 + def apply(
254 + self,
255 + layer: torch.nn.Module,
256 + x: torch.Tensor,
257 + bias: Optional[torch.Tensor] = None,
258 + ) -> torch.Tensor:
259 + qweight = layer.B_24
260 + meta = layer.B_meta
261 + scales = layer.s
262 + workspace = layer.workspace
263 +
264 + x_2d = x.view(-1, x.shape[-1])
265 +
266 + size_m = x_2d.shape[0]
267 + size_k = x_2d.shape[1]
268 + size_n = scales.shape[1]
269 +
270 + output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
271 + workspace,
272 + self.quant_config.weight_bits,
273 + size_m, size_n, size_k)
274 +
275 + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
276 +
277 + if bias is not None:
278 + output.add_(bias) # In-place add
279 +
280 + return output
vllm/model_executor/layers/quantization/marlin.py
+22
@@ -4,11 +4,14 @@ import torch
4 from torch.nn.parameter import Parameter
5
6 from vllm import _custom_ops as ops
7 +from vllm.logger import init_logger
8 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 from vllm.model_executor.utils import set_weight_attrs
12
13 +logger = init_logger(__name__)
14 +
15
16 class MarlinConfig(QuantizationConfig):
17 """Config class for Marlin.
@@ -72,6 +75,25 @@ class MarlinConfig(QuantizationConfig):
75 group_size = cls.get_from_keys(config, ["group_size"])
76 return cls(group_size)
77
78 + @classmethod
79 + def override_quantization_method(cls, hf_quant_cfg,
80 + user_quant) -> Optional[str]:
81 + # compat: autogptq >=0.8.0 use checkpoint_format: str
82 + # compat: autogptq <=0.7.1 is_marlin_format: bool
83 + is_marlin_format = (hf_quant_cfg.get("checkpoint_format") == "marlin"
84 + or hf_quant_cfg.get("is_marlin_format", False))
85 +
86 + is_valid_user_quant = (user_quant is None or user_quant == "gptq"
87 + or user_quant == "marlin")
88 +
89 + if is_marlin_format and is_valid_user_quant:
90 + msg = ("The model is serialized in {} format. Using {} kernel.".
91 + format(cls.get_name(), cls.get_name()))
92 + logger.info(msg)
93 + return cls.get_name()
94 +
95 + return None
96 +
97 def get_quant_method(
98 self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
99 if isinstance(layer, LinearBase):