@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99caa49106
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] add bfloat16 support for gptq marlin kernel (#4788)

Jinzhen Lin committed May 16, 2024 at 21:55 UTC 99caa4910651754f3f68de518ca42349c8c424d1
4 files changed +246 -73
csrc/quantization/gptq_marlin/gptq_marlin.cu
+173 -67
@@ -20,6 +20,11 @@
20 */
21
22 #include "gptq_marlin.cuh"
23 +#include "gptq_marlin_dtypes.cuh"
24 +
25 +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) static_assert(\
26 + std::is_same<scalar_t, half>::value || std::is_same<scalar_t, nv_bfloat16>::value, \
27 + "only float16 and bfloat16 is supported");
28
29 template <typename T> inline std::string str(T x) { return std::to_string(x); }
30
@@ -32,7 +37,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
37 int4 *__restrict__ out_int4_ptr, int size_m,
38 int size_k, int block_rows) {}
39
35 -template <const int num_bits, // number of bits used for weights
40 +template <typename scalar_t, // compute dtype, half or nv_float16
41 + const int num_bits, // number of bits used for weights
42 const int threads, // number of threads in a threadblock
43 const int thread_m_blocks, // number of 16x16 blocks in the m
44 // dimension (batchsize) of the threadblock
@@ -72,31 +78,36 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
78
79 #else
80
75 -// Matrix fragments for tensor core instructions; their precise layout is
76 -// documented here:
77 -// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
78 -using FragA = Vec<half2, 4>;
79 -using FragB = Vec<half2, 2>;
80 -using FragC = Vec<float, 4>;
81 -using FragS = Vec<half2, 1>; // quantization scales
82 -
81 // m16n8k16 tensor core mma instruction with fp16 inputs and fp32
82 // output/accumulation.
85 -__device__ inline void mma(const FragA &a_frag, const FragB &frag_b,
86 - FragC &frag_c) {
83 +template <typename scalar_t>
84 +__device__ inline void mma(const typename ScalarType<scalar_t>::FragA &a_frag,
85 + const typename ScalarType<scalar_t>::FragB &frag_b,
86 + typename ScalarType<scalar_t>::FragC &frag_c) {
87 const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
88 const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
89 float *c = reinterpret_cast<float *>(&frag_c);
90 - asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
91 - "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
92 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
93 - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
94 - "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
90 + if constexpr (std::is_same<scalar_t, half>::value) {
91 + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
92 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
93 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
94 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
95 + "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
96 + } else if constexpr (std::is_same<scalar_t, nv_bfloat16>::value) {
97 + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
98 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
99 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
100 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
101 + "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
102 + } else {
103 + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
104 + }
105 }
106
107 // Instruction for loading a full 16x16 matrix fragment of operand A from shared
108 // memory, directly in tensor core layout.
99 -__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
109 +template <typename scalar_t>
110 +__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA &frag_a, const void *smem_ptr) {
111 uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
112 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
113 asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
@@ -129,8 +140,15 @@ __device__ inline uint32_t prmt(uint32_t a) {
140 // Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
141 // values. We mostly follow the strategy in the link below, with some small
142 // changes:
132 -// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
133 -__device__ inline FragB dequant_4bit(int q) {
143 +// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
144 +// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
145 +template <typename scalar_t>
146 +__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit(int q) {
147 + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
148 +}
149 +
150 +template <>
151 +__device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
152 const int LO = 0x000f000f;
153 const int HI = 0x00f000f0;
154 const int EX = 0x64006400;
@@ -142,7 +160,7 @@ __device__ inline FragB dequant_4bit(int q) {
160 const int SUB = 0x64086408;
161 const int MUL = 0x2c002c00;
162 const int ADD = 0xd480d480;
145 - FragB frag_b;
163 + typename ScalarType<half>::FragB frag_b;
164 frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
165 *reinterpret_cast<const half2 *>(&SUB));
166 frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
@@ -151,7 +169,41 @@ __device__ inline FragB dequant_4bit(int q) {
169 return frag_b;
170 }
171
154 -__device__ inline FragB dequant_8bit(int q) {
172 +template <>
173 +__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat16>(int q) {
174 + static constexpr uint32_t MASK = 0x000f000f;
175 + static constexpr uint32_t EX = 0x43004300;
176 +
177 + // Guarantee that the `(a & b) | c` operations are LOP3s.
178 +
179 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
180 + q >>= 4;
181 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
182 +
183 + typename ScalarType<nv_bfloat16>::FragB frag_b;
184 + static constexpr uint32_t MUL = 0x3F803F80;
185 + static constexpr uint32_t ADD = 0xC308C308;
186 +
187 + frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162 *>(&lo),
188 + *reinterpret_cast<const nv_bfloat162*>(&MUL),
189 + *reinterpret_cast<const nv_bfloat162*>(&ADD));
190 + frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
191 + *reinterpret_cast<const nv_bfloat162*>(&MUL),
192 + *reinterpret_cast<const nv_bfloat162*>(&ADD));
193 + return frag_b;
194 +}
195 +
196 +// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or bf16
197 +// Reference:
198 +// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
199 +// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
200 +template <typename scalar_t>
201 +__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
202 + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
203 +}
204 +
205 +template <>
206 +__device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
207 static constexpr uint32_t mask_for_elt_01 = 0x5250;
208 static constexpr uint32_t mask_for_elt_23 = 0x5351;
209 static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
@@ -161,7 +213,7 @@ __device__ inline FragB dequant_8bit(int q) {
213
214 static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
215
164 - FragB frag_b;
216 + typename ScalarType<half>::FragB frag_b;
217 frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
218 *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
219 frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
@@ -169,34 +221,69 @@ __device__ inline FragB dequant_8bit(int q) {
221 return frag_b;
222 }
223
224 +template <>
225 +__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat16>(int q) {
226 + typename ScalarType<nv_bfloat16>::FragB frag_b;
227 +
228 + float fp32_intermediates[4];
229 + uint32_t * fp32_intermediates_casted = reinterpret_cast<uint32_t*>(fp32_intermediates);
230 +
231 + static constexpr uint32_t fp32_base = 0x4B000000;
232 + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
233 + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652);
234 + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
235 + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653);
236 +
237 + fp32_intermediates[0] -= 8388736.f;
238 + fp32_intermediates[1] -= 8388736.f;
239 + fp32_intermediates[2] -= 8388736.f;
240 + fp32_intermediates[3] -= 8388736.f;
241 +
242 + uint32_t* bf16_result_ptr = reinterpret_cast<uint32_t*>(&frag_b);
243 + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632);
244 + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632);
245 +
246 + return frag_b;
247 +}
248 +
249 // Multiply dequantized values by the corresponding quantization scale; used
250 // only for grouped quantization.
174 -__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
175 - half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
251 +template <typename scalar_t>
252 +__device__ inline void scale(typename ScalarType<scalar_t>::FragB &frag_b,
253 + typename ScalarType<scalar_t>::FragS &frag_s, int i) {
254 + using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
255 + scalar_t2 s = ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t *>(&frag_s)[i]);
256 frag_b[0] = __hmul2(frag_b[0], s);
257 frag_b[1] = __hmul2(frag_b[1], s);
258 }
259
260 // Same as above, but for act_order (each K is multiplied individually)
181 -__device__ inline void scale4(FragB &frag_b, FragS &frag_s_1, FragS &frag_s_2,
182 - FragS &frag_s_3, FragS &frag_s_4, int i) {
183 - __half2 s_val_1_2;
184 - s_val_1_2.x = reinterpret_cast<__half *>(&frag_s_1)[i];
185 - s_val_1_2.y = reinterpret_cast<__half *>(&frag_s_2)[i];
186 -
187 - __half2 s_val_3_4;
188 - s_val_3_4.x = reinterpret_cast<__half *>(&frag_s_3)[i];
189 - s_val_3_4.y = reinterpret_cast<__half *>(&frag_s_4)[i];
261 +template <typename scalar_t>
262 +__device__ inline void scale4(typename ScalarType<scalar_t>::FragB &frag_b,
263 + typename ScalarType<scalar_t>::FragS &frag_s_1,
264 + typename ScalarType<scalar_t>::FragS &frag_s_2,
265 + typename ScalarType<scalar_t>::FragS &frag_s_3,
266 + typename ScalarType<scalar_t>::FragS &frag_s_4,
267 + int i) {
268 + using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
269 + scalar_t2 s_val_1_2;
270 + s_val_1_2.x = reinterpret_cast<scalar_t *>(&frag_s_1)[i];
271 + s_val_1_2.y = reinterpret_cast<scalar_t *>(&frag_s_2)[i];
272 +
273 + scalar_t2 s_val_3_4;
274 + s_val_3_4.x = reinterpret_cast<scalar_t *>(&frag_s_3)[i];
275 + s_val_3_4.y = reinterpret_cast<scalar_t *>(&frag_s_4)[i];
276
277 frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
278 frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
279 }
280
281 // Given 2 floats multiply by 2 scales (halves)
196 -__device__ inline void scale_float(float *c, FragS &s) {
197 - __half *s_ptr = reinterpret_cast<__half *>(&s);
198 - c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
199 - c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
282 +template <typename scalar_t>
283 +__device__ inline void scale_float(float *c, typename ScalarType<scalar_t>::FragS &s) {
284 + scalar_t *s_ptr = reinterpret_cast<scalar_t *>(&s);
285 + c[0] = __fmul_rn(c[0], ScalarType<scalar_t>::num2float(s_ptr[0]));
286 + c[1] = __fmul_rn(c[1], ScalarType<scalar_t>::num2float(s_ptr[1]));
287 }
288
289 // Wait until barrier reaches `count`, then lock for current threadblock.
@@ -287,7 +374,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
374 }
375 }
376
290 -template <const int num_bits, // number of bits used for weights
377 +template <typename scalar_t, // compute dtype, half or nv_float16
378 + const int num_bits, // number of bits used for weights
379 const int threads, // number of threads in a threadblock
380 const int thread_m_blocks, // number of 16x16 blocks in the m
381 // dimension (batchsize) of the threadblock
@@ -323,6 +411,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
411 // ensures good utilization of all SMs for many kinds of shape and GPU
412 // configurations, while requiring as few slow global cross-threadblock
413 // reductions as possible.
414 + using Dtype = ScalarType<scalar_t>;
415 + using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
416 + using FragA = typename ScalarType<scalar_t>::FragA;
417 + using FragB = typename ScalarType<scalar_t>::FragB;
418 + using FragC = typename ScalarType<scalar_t>::FragC;
419 + using FragS = typename ScalarType<scalar_t>::FragS;
420
421 constexpr int pack_factor = 32 / num_bits;
422
@@ -691,7 +785,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
785 int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
786 #pragma unroll
787 for (int i = 0; i < thread_m_blocks; i++)
694 - ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
788 + ldsm4<scalar_t>(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
789 int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
790
791 #pragma unroll
@@ -835,43 +929,43 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
929 int b_quant = frag_b_quant[k % 2][0][j];
930 int b_quant_shift = b_quant >> 8;
931
838 - frag_b0 = dequant_4bit(b_quant);
839 - frag_b1 = dequant_4bit(b_quant_shift);
932 + frag_b0 = dequant_4bit<scalar_t>(b_quant);
933 + frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
934
935 } else {
936 int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
937 int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
938 int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
939
846 - frag_b0 = dequant_8bit(b_quant_0);
847 - frag_b1 = dequant_8bit(b_quant_1);
940 + frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
941 + frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
942 }
943
944 // Apply scale to frag_b0
945 if constexpr (has_act_order) {
852 - scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
946 + scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
947 act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
948 } else {
949 if constexpr (group_blocks != -1) {
856 - scale(frag_b0, frag_s[k % 2][j], 0);
950 + scale<scalar_t>(frag_b0, frag_s[k % 2][j], 0);
951 }
952 }
953
954 // Apply scale to frag_b1
955 if constexpr (has_act_order) {
862 - scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
956 + scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
957 act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
958
959 } else {
960 if constexpr (group_blocks != -1) {
867 - scale(frag_b1, frag_s[k % 2][j], 1);
961 + scale<scalar_t>(frag_b1, frag_s[k % 2][j], 1);
962 }
963 }
964
965 #pragma unroll
966 for (int i = 0; i < thread_m_blocks; i++) {
873 - mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
874 - mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
967 + mma<scalar_t>(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
968 + mma<scalar_t>(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
969 }
970 }
971 };
@@ -979,15 +1073,15 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1073 for (int j = 0; j < 2 * 4; j++) {
1074 reinterpret_cast<float *>(
1075 &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
982 - __half2float(reinterpret_cast<__half *>(&c_red)[j]);
1076 + Dtype::num2float(reinterpret_cast<scalar_t *>(&c_red)[j]);
1077 }
1078 }
1079 if (!last) {
1080 int4 c;
1081 #pragma unroll
1082 for (int j = 0; j < 2 * 4; j++) {
989 - reinterpret_cast<__half *>(&c)[j] =
990 - __float2half(reinterpret_cast<float *>(
1083 + reinterpret_cast<scalar_t *>(&c)[j] =
1084 + Dtype::float2num(reinterpret_cast<float *>(
1085 &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
1086 }
1087 C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
@@ -1022,7 +1116,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1116 // We first reorder in shared memory to guarantee the most efficient final
1117 // global write patterns
1118 auto write = [&](int idx, float c0, float c1, FragS &s) {
1025 - half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1119 + scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
1120
1121 // For per-column quantization we finally apply the scale here (only for
1122 // 4-bit)
@@ -1030,7 +1124,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1124 res = __hmul2(res, s[0]);
1125 }
1126
1033 - ((half2 *)sh)[idx] = res;
1127 + ((scalar_t2 *)sh)[idx] = res;
1128 };
1129
1130 if (threadIdx.x / 32 < thread_n_blocks / 4) {
@@ -1192,14 +1286,14 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1286 for (int i = 0; i < thread_m_blocks; i++) {
1287 #pragma unroll
1288 for (int j = 0; j < 4; j++) {
1195 - scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1289 + scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1290 frag_s[j / 2][2 * (j % 2) + 0]);
1197 - scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1291 + scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1292 frag_s[j / 2][2 * (j % 2) + 0]);
1293
1200 - scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1294 + scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1295 frag_s[j / 2][2 * (j % 2) + 1]);
1202 - scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1296 + scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1297 frag_s[j / 2][2 * (j % 2) + 1]);
1298 }
1299 }
@@ -1255,10 +1349,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1349 has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1350 num_threads == NUM_THREADS) { \
1351 cudaFuncSetAttribute( \
1258 - Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1352 + Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1353 THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1354 cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1261 - Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1355 + Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1356 THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1357 <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1358 A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
@@ -1462,6 +1556,7 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1556 __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1557 __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1558
1559 +template <typename scalar_t>
1560 void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1561 void *g_idx, void *perm, void *a_tmp, int prob_m,
1562 int prob_n, int prob_k, void *workspace, int num_bits,
@@ -1731,14 +1826,25 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1826 " is below min_workspace_size = ", min_workspace_size);
1827
1828 int dev = a.get_device();
1734 - gptq_marlin::marlin_mm_f16i4(
1735 - a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), b_scales.data_ptr(),
1736 - g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(), size_m, size_n,
1737 - size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1738 - num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1739 - thread_k, thread_n, sms, gptq_marlin::max_par);
1829 + if (a.scalar_type() == at::ScalarType::Half) {
1830 + gptq_marlin::marlin_mm_f16i4<half>(
1831 + a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(), b_scales.data_ptr<at::Half>(),
1832 + g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n,
1833 + size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1834 + num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1835 + thread_k, thread_n, sms, gptq_marlin::max_par);
1836 + } else if (a.scalar_type() == at::ScalarType::BFloat16) {
1837 + gptq_marlin::marlin_mm_f16i4<nv_bfloat16>(
1838 + a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(), c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1839 + g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n,
1840 + size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1841 + num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1842 + thread_k, thread_n, sms, gptq_marlin::max_par);
1843 + } else {
1844 + TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
1845 + }
1846
1847 return c;
1848 }
1849
1744 -#endif
1850 +#endif
\ No newline at end of file
csrc/quantization/gptq_marlin/gptq_marlin_dtypes.cuh new
+62
@@ -0,0 +1,62 @@
1 +
2 +#ifndef _data_types_cuh
3 +#define _data_types_cuh
4 +#include "gptq_marlin.cuh"
5 +#include <cuda_fp16.h>
6 +#include <cuda_bf16.h>
7 +
8 +
9 +namespace gptq_marlin {
10 +
11 +template <typename scalar_t>
12 +class ScalarType {
13 +};
14 +
15 +template <>
16 +class ScalarType<half> {
17 +public:
18 + using scalar_t = half;
19 + using scalar_t2 = half2;
20 +
21 + // Matrix fragments for tensor core instructions; their precise layout is
22 + // documented here:
23 + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
24 + using FragA = Vec<half2, 4>;
25 + using FragB = Vec<half2, 2>;
26 + using FragC = Vec<float, 4>;
27 + using FragS = Vec<half2, 1>;
28 +
29 + static __device__ float inline num2float(const half x) { return __half2float(x); }
30 +
31 + static __device__ half2 inline num2num2(const half x) { return __half2half2(x); }
32 +
33 + static __device__ half2 inline nums2num2(const half x1, const half x2) { return __halves2half2(x1, x2); }
34 +
35 + static __host__ __device__ half inline float2num(const float x) { return __float2half(x); }
36 +};
37 +
38 +template <>
39 +class ScalarType<nv_bfloat16> {
40 +public:
41 + using scalar_t = nv_bfloat16;
42 + using scalar_t2 = nv_bfloat162;
43 +
44 + using FragA = Vec<nv_bfloat162, 4>;
45 + using FragB = Vec<nv_bfloat162, 2>;
46 + using FragC = Vec<float, 4>;
47 + using FragS = Vec<nv_bfloat162, 1>;
48 +
49 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
50 + static __device__ float inline num2float(const nv_bfloat16 x) { return __bfloat162float(x); }
51 +
52 + static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { return __bfloat162bfloat162(x); }
53 +
54 + static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1, const nv_bfloat16 x2) { return __halves2bfloat162(x1, x2); }
55 +
56 + static __host__ __device__ nv_bfloat16 inline float2num(const float x) { return __float2bfloat16(x); }
57 +#endif
58 +};
59 +
60 +}
61 +
62 +#endif
tests/models/test_gptq_marlin.py
+7 -2
@@ -14,6 +14,7 @@ import pytest
14 import torch
15
16 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
17 +from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
18
19 from .utils import check_logprobs_close
20
@@ -52,7 +53,7 @@ MODELS = [
53 @pytest.mark.skipif(gptq_marlin_not_supported,
54 reason="gptq_marlin is not supported on this GPU type.")
55 @pytest.mark.parametrize("model", MODELS)
55 -@pytest.mark.parametrize("dtype", ["half"])
56 +@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
57 @pytest.mark.parametrize("max_tokens", [32])
58 @pytest.mark.parametrize("num_logprobs", [5])
59 def test_models(
@@ -76,11 +77,15 @@ def test_models(
77 gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
78 example_prompts[:-1], max_tokens, num_logprobs)
79 del gptq_marlin_model
80 + _ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
81
82 # Run gptq.
83 + # The naive gptq kernel doesn't support bf16 yet.
84 + # Here we always compare fp16/bf16 gpt marlin kernel
85 + # to fp16 gptq kernel.
86 gptq_model = vllm_runner(model_name=model_name,
87 revision=revision,
83 - dtype=dtype,
88 + dtype="half",
89 quantization="gptq",
90 max_model_len=MAX_MODEL_LEN,
91 tensor_parallel_size=1)
vllm/model_executor/layers/quantization/gptq_marlin.py
+4 -4
@@ -99,7 +99,7 @@ class GPTQMarlinConfig(QuantizationConfig):
99
100 @classmethod
101 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
102 - return [torch.half]
102 + return [torch.half, torch.bfloat16]
103
104 @classmethod
105 def get_min_capability(cls) -> int:
@@ -186,9 +186,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
186 group_size = input_size
187
188 # Validate dtype
189 - if params_dtype != torch.float16:
190 - raise ValueError(
191 - f"The params dtype must be float16, but got {params_dtype}")
189 + if params_dtype not in [torch.float16, torch.bfloat16]:
190 + raise ValueError(f"The params dtype must be float16 "
191 + f"or bfloat16, but got {params_dtype}")
192
193 # Validate output_size_per_partition
194 output_size_per_partition = sum(output_partition_sizes)