A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] add bfloat16 support for gptq marlin kernel (#4788)
Jinzhen Lin committed
May 16, 2024 at 21:55 UTC
99caa4910651754f3f68de518ca42349c8c424d1
4 files changed
+246
-73
csrc/quantization/gptq_marlin/gptq_marlin.cu
+173
-67
@@ -20,6 +20,11 @@
20
*/
21
22
#include "gptq_marlin.cuh"
23
+#include "gptq_marlin_dtypes.cuh"
24
+
25
+#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) static_assert(\
26
+ std::is_same<scalar_t, half>::value || std::is_same<scalar_t, nv_bfloat16>::value, \
27
+ "only float16 and bfloat16 is supported");
28
29
template <typename T> inline std::string str(T x) { return std::to_string(x); }
30
@@ -32,7 +37,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
37
int4 *__restrict__ out_int4_ptr, int size_m,
38
int size_k, int block_rows) {}
39
35
-template <const int num_bits, // number of bits used for weights
40
+template <typename scalar_t, // compute dtype, half or nv_float16
41
+ const int num_bits, // number of bits used for weights
42
const int threads, // number of threads in a threadblock
43
const int thread_m_blocks, // number of 16x16 blocks in the m
44
// dimension (batchsize) of the threadblock
@@ -72,31 +78,36 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
78
79
#else
80
75
-// Matrix fragments for tensor core instructions; their precise layout is
76
-// documented here:
77
-// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
78
-using FragA = Vec<half2, 4>;
79
-using FragB = Vec<half2, 2>;
80
-using FragC = Vec<float, 4>;
81
-using FragS = Vec<half2, 1>; // quantization scales
82
-
81
// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
82
// output/accumulation.
85
-__device__ inline void mma(const FragA &a_frag, const FragB &frag_b,
86
- FragC &frag_c) {
83
+template <typename scalar_t>
84
+__device__ inline void mma(const typename ScalarType<scalar_t>::FragA &a_frag,
85
+ const typename ScalarType<scalar_t>::FragB &frag_b,
86
+ typename ScalarType<scalar_t>::FragC &frag_c) {
87
const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
88
const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
89
float *c = reinterpret_cast<float *>(&frag_c);
90
- asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
91
- "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
92
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
93
- : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
94
- "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
90
+ if constexpr (std::is_same<scalar_t, half>::value) {
91
+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
92
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
93
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
94
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
95
+ "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
96
+ } else if constexpr (std::is_same<scalar_t, nv_bfloat16>::value) {
97
+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
98
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
99
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
100
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
101
+ "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
102
+ } else {
103
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
104
+ }
105
}
106
107
// Instruction for loading a full 16x16 matrix fragment of operand A from shared
108
// memory, directly in tensor core layout.
99
-__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
109
+template <typename scalar_t>
110
+__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA &frag_a, const void *smem_ptr) {
111
uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
112
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
113
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
@@ -129,8 +140,15 @@ __device__ inline uint32_t prmt(uint32_t a) {
140
// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
141
// values. We mostly follow the strategy in the link below, with some small
142
// changes:
132
-// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
133
-__device__ inline FragB dequant_4bit(int q) {
143
+// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
144
+// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
145
+template <typename scalar_t>
146
+__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit(int q) {
147
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
148
+}
149
+
150
+template <>
151
+__device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
152
const int LO = 0x000f000f;
153
const int HI = 0x00f000f0;
154
const int EX = 0x64006400;
@@ -142,7 +160,7 @@ __device__ inline FragB dequant_4bit(int q) {
160
const int SUB = 0x64086408;
161
const int MUL = 0x2c002c00;
162
const int ADD = 0xd480d480;
145
- FragB frag_b;
163
+ typename ScalarType<half>::FragB frag_b;
164
frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
165
*reinterpret_cast<const half2 *>(&SUB));
166
frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
@@ -151,7 +169,41 @@ __device__ inline FragB dequant_4bit(int q) {
169
return frag_b;
170
}
171
154
-__device__ inline FragB dequant_8bit(int q) {
172
+template <>
173
+__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat16>(int q) {
174
+ static constexpr uint32_t MASK = 0x000f000f;
175
+ static constexpr uint32_t EX = 0x43004300;
176
+
177
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
178
+
179
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
180
+ q >>= 4;
181
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
182
+
183
+ typename ScalarType<nv_bfloat16>::FragB frag_b;
184
+ static constexpr uint32_t MUL = 0x3F803F80;
185
+ static constexpr uint32_t ADD = 0xC308C308;
186
+
187
+ frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162 *>(&lo),
188
+ *reinterpret_cast<const nv_bfloat162*>(&MUL),
189
+ *reinterpret_cast<const nv_bfloat162*>(&ADD));
190
+ frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
191
+ *reinterpret_cast<const nv_bfloat162*>(&MUL),
192
+ *reinterpret_cast<const nv_bfloat162*>(&ADD));
193
+ return frag_b;
194
+}
195
+
196
+// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or bf16
197
+// Reference:
198
+// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
199
+// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
200
+template <typename scalar_t>
201
+__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
202
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
203
+}
204
+
205
+template <>
206
+__device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
207
static constexpr uint32_t mask_for_elt_01 = 0x5250;
208
static constexpr uint32_t mask_for_elt_23 = 0x5351;
209
static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
@@ -161,7 +213,7 @@ __device__ inline FragB dequant_8bit(int q) {
213
214
static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
215
164
- FragB frag_b;
216
+ typename ScalarType<half>::FragB frag_b;
217
frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
218
*reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
219
frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
@@ -169,34 +221,69 @@ __device__ inline FragB dequant_8bit(int q) {
221
return frag_b;
222
}
223
224
+template <>
225
+__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat16>(int q) {
226
+ typename ScalarType<nv_bfloat16>::FragB frag_b;
227
+
228
+ float fp32_intermediates[4];
229
+ uint32_t * fp32_intermediates_casted = reinterpret_cast<uint32_t*>(fp32_intermediates);
230
+
231
+ static constexpr uint32_t fp32_base = 0x4B000000;
232
+ fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
233
+ fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652);
234
+ fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
235
+ fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653);
236
+
237
+ fp32_intermediates[0] -= 8388736.f;
238
+ fp32_intermediates[1] -= 8388736.f;
239
+ fp32_intermediates[2] -= 8388736.f;
240
+ fp32_intermediates[3] -= 8388736.f;
241
+
242
+ uint32_t* bf16_result_ptr = reinterpret_cast<uint32_t*>(&frag_b);
243
+ bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632);
244
+ bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632);
245
+
246
+ return frag_b;
247
+}
248
+
249
// Multiply dequantized values by the corresponding quantization scale; used
250
// only for grouped quantization.
174
-__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
175
- half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
251
+template <typename scalar_t>
252
+__device__ inline void scale(typename ScalarType<scalar_t>::FragB &frag_b,
253
+ typename ScalarType<scalar_t>::FragS &frag_s, int i) {
254
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
255
+ scalar_t2 s = ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t *>(&frag_s)[i]);
256
frag_b[0] = __hmul2(frag_b[0], s);
257
frag_b[1] = __hmul2(frag_b[1], s);
258
}
259
260
// Same as above, but for act_order (each K is multiplied individually)
181
-__device__ inline void scale4(FragB &frag_b, FragS &frag_s_1, FragS &frag_s_2,
182
- FragS &frag_s_3, FragS &frag_s_4, int i) {
183
- __half2 s_val_1_2;
184
- s_val_1_2.x = reinterpret_cast<__half *>(&frag_s_1)[i];
185
- s_val_1_2.y = reinterpret_cast<__half *>(&frag_s_2)[i];
186
-
187
- __half2 s_val_3_4;
188
- s_val_3_4.x = reinterpret_cast<__half *>(&frag_s_3)[i];
189
- s_val_3_4.y = reinterpret_cast<__half *>(&frag_s_4)[i];
261
+template <typename scalar_t>
262
+__device__ inline void scale4(typename ScalarType<scalar_t>::FragB &frag_b,
263
+ typename ScalarType<scalar_t>::FragS &frag_s_1,
264
+ typename ScalarType<scalar_t>::FragS &frag_s_2,
265
+ typename ScalarType<scalar_t>::FragS &frag_s_3,
266
+ typename ScalarType<scalar_t>::FragS &frag_s_4,
267
+ int i) {
268
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
269
+ scalar_t2 s_val_1_2;
270
+ s_val_1_2.x = reinterpret_cast<scalar_t *>(&frag_s_1)[i];
271
+ s_val_1_2.y = reinterpret_cast<scalar_t *>(&frag_s_2)[i];
272
+
273
+ scalar_t2 s_val_3_4;
274
+ s_val_3_4.x = reinterpret_cast<scalar_t *>(&frag_s_3)[i];
275
+ s_val_3_4.y = reinterpret_cast<scalar_t *>(&frag_s_4)[i];
276
277
frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
278
frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
279
}
280
281
// Given 2 floats multiply by 2 scales (halves)
196
-__device__ inline void scale_float(float *c, FragS &s) {
197
- __half *s_ptr = reinterpret_cast<__half *>(&s);
198
- c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
199
- c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
282
+template <typename scalar_t>
283
+__device__ inline void scale_float(float *c, typename ScalarType<scalar_t>::FragS &s) {
284
+ scalar_t *s_ptr = reinterpret_cast<scalar_t *>(&s);
285
+ c[0] = __fmul_rn(c[0], ScalarType<scalar_t>::num2float(s_ptr[0]));
286
+ c[1] = __fmul_rn(c[1], ScalarType<scalar_t>::num2float(s_ptr[1]));
287
}
288
289
// Wait until barrier reaches `count`, then lock for current threadblock.
@@ -287,7 +374,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
374
}
375
}
376
290
-template <const int num_bits, // number of bits used for weights
377
+template <typename scalar_t, // compute dtype, half or nv_float16
378
+ const int num_bits, // number of bits used for weights
379
const int threads, // number of threads in a threadblock
380
const int thread_m_blocks, // number of 16x16 blocks in the m
381
// dimension (batchsize) of the threadblock
@@ -323,6 +411,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
411
// ensures good utilization of all SMs for many kinds of shape and GPU
412
// configurations, while requiring as few slow global cross-threadblock
413
// reductions as possible.
414
+ using Dtype = ScalarType<scalar_t>;
415
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
416
+ using FragA = typename ScalarType<scalar_t>::FragA;
417
+ using FragB = typename ScalarType<scalar_t>::FragB;
418
+ using FragC = typename ScalarType<scalar_t>::FragC;
419
+ using FragS = typename ScalarType<scalar_t>::FragS;
420
421
constexpr int pack_factor = 32 / num_bits;
422
@@ -691,7 +785,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
785
int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
786
#pragma unroll
787
for (int i = 0; i < thread_m_blocks; i++)
694
- ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
788
+ ldsm4<scalar_t>(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
789
int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
790
791
#pragma unroll
@@ -835,43 +929,43 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
929
int b_quant = frag_b_quant[k % 2][0][j];
930
int b_quant_shift = b_quant >> 8;
931
838
- frag_b0 = dequant_4bit(b_quant);
839
- frag_b1 = dequant_4bit(b_quant_shift);
932
+ frag_b0 = dequant_4bit<scalar_t>(b_quant);
933
+ frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
934
935
} else {
936
int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
937
int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
938
int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
939
846
- frag_b0 = dequant_8bit(b_quant_0);
847
- frag_b1 = dequant_8bit(b_quant_1);
940
+ frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
941
+ frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
942
}
943
944
// Apply scale to frag_b0
945
if constexpr (has_act_order) {
852
- scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
946
+ scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
947
act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
948
} else {
949
if constexpr (group_blocks != -1) {
856
- scale(frag_b0, frag_s[k % 2][j], 0);
950
+ scale<scalar_t>(frag_b0, frag_s[k % 2][j], 0);
951
}
952
}
953
954
// Apply scale to frag_b1
955
if constexpr (has_act_order) {
862
- scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
956
+ scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
957
act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
958
959
} else {
960
if constexpr (group_blocks != -1) {
867
- scale(frag_b1, frag_s[k % 2][j], 1);
961
+ scale<scalar_t>(frag_b1, frag_s[k % 2][j], 1);
962
}
963
}
964
965
#pragma unroll
966
for (int i = 0; i < thread_m_blocks; i++) {
873
- mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
874
- mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
967
+ mma<scalar_t>(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
968
+ mma<scalar_t>(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
969
}
970
}
971
};
@@ -979,15 +1073,15 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1073
for (int j = 0; j < 2 * 4; j++) {
1074
reinterpret_cast<float *>(
1075
&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
982
- __half2float(reinterpret_cast<__half *>(&c_red)[j]);
1076
+ Dtype::num2float(reinterpret_cast<scalar_t *>(&c_red)[j]);
1077
}
1078
}
1079
if (!last) {
1080
int4 c;
1081
#pragma unroll
1082
for (int j = 0; j < 2 * 4; j++) {
989
- reinterpret_cast<__half *>(&c)[j] =
990
- __float2half(reinterpret_cast<float *>(
1083
+ reinterpret_cast<scalar_t *>(&c)[j] =
1084
+ Dtype::float2num(reinterpret_cast<float *>(
1085
&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
1086
}
1087
C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
@@ -1022,7 +1116,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1116
// We first reorder in shared memory to guarantee the most efficient final
1117
// global write patterns
1118
auto write = [&](int idx, float c0, float c1, FragS &s) {
1025
- half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1119
+ scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
1120
1121
// For per-column quantization we finally apply the scale here (only for
1122
// 4-bit)
@@ -1030,7 +1124,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1124
res = __hmul2(res, s[0]);
1125
}
1126
1033
- ((half2 *)sh)[idx] = res;
1127
+ ((scalar_t2 *)sh)[idx] = res;
1128
};
1129
1130
if (threadIdx.x / 32 < thread_n_blocks / 4) {
@@ -1192,14 +1286,14 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1286
for (int i = 0; i < thread_m_blocks; i++) {
1287
#pragma unroll
1288
for (int j = 0; j < 4; j++) {
1195
- scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1289
+ scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1290
frag_s[j / 2][2 * (j % 2) + 0]);
1197
- scale_float(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1291
+ scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1292
frag_s[j / 2][2 * (j % 2) + 0]);
1293
1200
- scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1294
+ scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1295
frag_s[j / 2][2 * (j % 2) + 1]);
1202
- scale_float(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1296
+ scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1297
frag_s[j / 2][2 * (j % 2) + 1]);
1298
}
1299
}
@@ -1255,10 +1349,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1349
has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1350
num_threads == NUM_THREADS) { \
1351
cudaFuncSetAttribute( \
1258
- Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1352
+ Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1353
THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1354
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1261
- Marlin<NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1355
+ Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1356
THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1357
<<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1358
A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
@@ -1462,6 +1556,7 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1556
__CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1557
__CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1558
1559
+template <typename scalar_t>
1560
void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1561
void *g_idx, void *perm, void *a_tmp, int prob_m,
1562
int prob_n, int prob_k, void *workspace, int num_bits,
@@ -1731,14 +1826,25 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1826
" is below min_workspace_size = ", min_workspace_size);
1827
1828
int dev = a.get_device();
1734
- gptq_marlin::marlin_mm_f16i4(
1735
- a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), b_scales.data_ptr(),
1736
- g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(), size_m, size_n,
1737
- size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1738
- num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1739
- thread_k, thread_n, sms, gptq_marlin::max_par);
1829
+ if (a.scalar_type() == at::ScalarType::Half) {
1830
+ gptq_marlin::marlin_mm_f16i4<half>(
1831
+ a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(), b_scales.data_ptr<at::Half>(),
1832
+ g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n,
1833
+ size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1834
+ num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1835
+ thread_k, thread_n, sms, gptq_marlin::max_par);
1836
+ } else if (a.scalar_type() == at::ScalarType::BFloat16) {
1837
+ gptq_marlin::marlin_mm_f16i4<nv_bfloat16>(
1838
+ a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(), c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1839
+ g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n,
1840
+ size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1841
+ num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1842
+ thread_k, thread_n, sms, gptq_marlin::max_par);
1843
+ } else {
1844
+ TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
1845
+ }
1846
1847
return c;
1848
}
1849
1744
-#endif
1850
+#endif
\ No newline at end of file
csrc/quantization/gptq_marlin/gptq_marlin_dtypes.cuh
new
+62
@@ -0,0 +1,62 @@
1
+
2
+#ifndef _data_types_cuh
3
+#define _data_types_cuh
4
+#include "gptq_marlin.cuh"
5
+#include <cuda_fp16.h>
6
+#include <cuda_bf16.h>
7
+
8
+
9
+namespace gptq_marlin {
10
+
11
+template <typename scalar_t>
12
+class ScalarType {
13
+};
14
+
15
+template <>
16
+class ScalarType<half> {
17
+public:
18
+ using scalar_t = half;
19
+ using scalar_t2 = half2;
20
+
21
+ // Matrix fragments for tensor core instructions; their precise layout is
22
+ // documented here:
23
+ // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
24
+ using FragA = Vec<half2, 4>;
25
+ using FragB = Vec<half2, 2>;
26
+ using FragC = Vec<float, 4>;
27
+ using FragS = Vec<half2, 1>;
28
+
29
+ static __device__ float inline num2float(const half x) { return __half2float(x); }
30
+
31
+ static __device__ half2 inline num2num2(const half x) { return __half2half2(x); }
32
+
33
+ static __device__ half2 inline nums2num2(const half x1, const half x2) { return __halves2half2(x1, x2); }
34
+
35
+ static __host__ __device__ half inline float2num(const float x) { return __float2half(x); }
36
+};
37
+
38
+template <>
39
+class ScalarType<nv_bfloat16> {
40
+public:
41
+ using scalar_t = nv_bfloat16;
42
+ using scalar_t2 = nv_bfloat162;
43
+
44
+ using FragA = Vec<nv_bfloat162, 4>;
45
+ using FragB = Vec<nv_bfloat162, 2>;
46
+ using FragC = Vec<float, 4>;
47
+ using FragS = Vec<nv_bfloat162, 1>;
48
+
49
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
50
+ static __device__ float inline num2float(const nv_bfloat16 x) { return __bfloat162float(x); }
51
+
52
+ static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { return __bfloat162bfloat162(x); }
53
+
54
+ static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1, const nv_bfloat16 x2) { return __halves2bfloat162(x1, x2); }
55
+
56
+ static __host__ __device__ nv_bfloat16 inline float2num(const float x) { return __float2bfloat16(x); }
57
+#endif
58
+};
59
+
60
+}
61
+
62
+#endif
tests/models/test_gptq_marlin.py
+7
-2
@@ -14,6 +14,7 @@ import pytest
14
import torch
15
16
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
17
+from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
18
19
from .utils import check_logprobs_close
20
@@ -52,7 +53,7 @@ MODELS = [
53
@pytest.mark.skipif(gptq_marlin_not_supported,
54
reason="gptq_marlin is not supported on this GPU type.")
55
@pytest.mark.parametrize("model", MODELS)
55
-@pytest.mark.parametrize("dtype", ["half"])
56
+@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
57
@pytest.mark.parametrize("max_tokens", [32])
58
@pytest.mark.parametrize("num_logprobs", [5])
59
def test_models(
@@ -76,11 +77,15 @@ def test_models(
77
gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
78
example_prompts[:-1], max_tokens, num_logprobs)
79
del gptq_marlin_model
80
+ _ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
81
82
# Run gptq.
83
+ # The naive gptq kernel doesn't support bf16 yet.
84
+ # Here we always compare fp16/bf16 gpt marlin kernel
85
+ # to fp16 gptq kernel.
86
gptq_model = vllm_runner(model_name=model_name,
87
revision=revision,
83
- dtype=dtype,
88
+ dtype="half",
89
quantization="gptq",
90
max_model_len=MAX_MODEL_LEN,
91
tensor_parallel_size=1)
vllm/model_executor/layers/quantization/gptq_marlin.py
+4
-4
@@ -99,7 +99,7 @@ class GPTQMarlinConfig(QuantizationConfig):
99
100
@classmethod
101
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
102
- return [torch.half]
102
+ return [torch.half, torch.bfloat16]
103
104
@classmethod
105
def get_min_capability(cls) -> int:
@@ -186,9 +186,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
186
group_size = input_size
187
188
# Validate dtype
189
- if params_dtype != torch.float16:
190
- raise ValueError(
191
- f"The params dtype must be float16, but got {params_dtype}")
189
+ if params_dtype not in [torch.float16, torch.bfloat16]:
190
+ raise ValueError(f"The params dtype must be float16 "
191
+ f"or bfloat16, but got {params_dtype}")
192
193
# Validate output_size_per_partition
194
output_size_per_partition = sum(output_partition_sizes)