A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Use scalar type to dispatch to different `gptq_marlin` kernels (#7323)
Lucas Wilkinson committed
Aug 12, 2024 at 14:40 UTC
6aa33cb2ddd769e764a3312627cab5bffaa383cc
2 files changed
+334
-220
csrc/core/scalar_type.hpp
+166
-31
@@ -20,7 +20,7 @@ namespace vllm {
20
//
21
class ScalarType {
22
public:
23
- enum NanRepr : int64_t {
23
+ enum NanRepr : uint8_t {
24
NAN_NONE = 0, // nans are not supported
25
NAN_IEEE_754 = 1, // nans are: exp all 1s, mantissa not all 0s
26
NAN_EXTD_RANGE_MAX_MIN = 2, // nans are: exp all 1s, mantissa all 1s
@@ -28,33 +28,33 @@ class ScalarType {
28
NAN_REPR_ID_MAX
29
};
30
31
- constexpr ScalarType(bool signed_, int64_t exponent, int64_t mantissa,
32
- int64_t bias, bool finite_values_only = false,
31
+ constexpr ScalarType(uint8_t exponent, uint8_t mantissa, bool signed_,
32
+ int32_t bias, bool finite_values_only = false,
33
NanRepr nan_repr = NAN_IEEE_754)
34
: exponent(exponent),
35
mantissa(mantissa),
36
- bias(bias),
36
signed_(signed_),
37
+ bias(bias),
38
finite_values_only(finite_values_only),
39
nan_repr(nan_repr){};
40
41
- static constexpr ScalarType int_(int64_t size_bits, int64_t bias = 0) {
42
- return ScalarType(true, 0, size_bits - 1, bias);
41
+ static constexpr ScalarType int_(uint8_t size_bits, int32_t bias = 0) {
42
+ return ScalarType(0, size_bits - 1, true, bias);
43
}
44
45
- static constexpr ScalarType uint(int64_t size_bits, int64_t bias = 0) {
46
- return ScalarType(false, 0, size_bits, bias);
45
+ static constexpr ScalarType uint(uint8_t size_bits, int32_t bias = 0) {
46
+ return ScalarType(0, size_bits, false, bias);
47
}
48
49
// IEEE 754 compliant floating point type
50
- static constexpr ScalarType float_IEEE754(int64_t exponent,
51
- int64_t mantissa) {
50
+ static constexpr ScalarType float_IEEE754(uint8_t exponent,
51
+ uint8_t mantissa) {
52
TORCH_CHECK(mantissa > 0 && exponent > 0);
53
- return ScalarType(true, exponent, mantissa, 0, false, NAN_IEEE_754);
53
+ return ScalarType(exponent, mantissa, true, 0, false, NAN_IEEE_754);
54
}
55
56
// IEEE 754 non-compliant floating point type
57
- static constexpr ScalarType float_(int64_t exponent, int64_t mantissa,
57
+ static constexpr ScalarType float_(uint8_t exponent, uint8_t mantissa,
58
bool finite_values_only,
59
NanRepr nan_repr) {
60
TORCH_CHECK(nan_repr < NAN_REPR_ID_MAX, "Invalid NanRepr");
@@ -62,36 +62,121 @@ class ScalarType {
62
TORCH_CHECK(nan_repr != NAN_IEEE_754,
63
"use `float_IEEE754` constructor for floating point types that "
64
"follow IEEE 754 conventions");
65
- return ScalarType(true, exponent, mantissa, 0, finite_values_only,
65
+ return ScalarType(exponent, mantissa, true, 0, finite_values_only,
66
nan_repr);
67
}
68
69
- int64_t const exponent; // size of the exponent field (0 for integer types)
70
- int64_t const mantissa; // size of the mantissa field (size of the integer
69
+ uint8_t const exponent; // size of the exponent field (0 for integer types)
70
+ uint8_t const mantissa; // size of the mantissa field (size of the integer
71
// excluding the sign bit for integer types)
72
- int64_t const bias; // stored values equal value + bias,
73
- // used for quantized type
72
bool const signed_; // flag if the type supports negative numbers (i.e. has a
73
// sign bit)
74
+ int32_t const bias; // stored values equal value + bias,
75
+ // used for quantized type
76
77
// Extra Floating point info
78
bool const finite_values_only; // i.e. no +/-inf if true
79
NanRepr const nan_repr; // how NaNs are represented
80
// (not applicable for integer types)
81
82
- int64_t size_bits() const { return mantissa + exponent + is_signed(); }
83
- bool is_signed() const { return signed_; }
84
- bool is_integer() const { return exponent == 0; }
85
- bool is_floating_point() const { return exponent > 0; }
86
- bool is_ieee_754() const {
82
+ using Id = int64_t;
83
+
84
+ private:
85
+ // Field size in id
86
+ template <typename T_>
87
+ static constexpr size_t member_id_field_width() {
88
+ using T = std::decay_t<T_>;
89
+ return std::is_same_v<T, bool> ? 1 : sizeof(T) * 8;
90
+ }
91
+
92
+ template <typename Fn, typename Init, typename Member, typename... Rest>
93
+ static constexpr auto reduce_members_helper(Fn f, Init val, Member member,
94
+ Rest... rest) {
95
+ auto new_val = f(val, member);
96
+ if constexpr (sizeof...(rest) > 0) {
97
+ return reduce_members_helper(f, new_val, rest...);
98
+ } else {
99
+ return new_val;
100
+ };
101
+ }
102
+
103
+ template <typename Fn, typename Init>
104
+ constexpr auto reduce_members(Fn f, Init init) const {
105
+ // Should be in constructor order for `from_id`
106
+ return reduce_members_helper(f, init, exponent, mantissa, signed_, bias,
107
+ finite_values_only, nan_repr);
108
+ };
109
+
110
+ template <typename Fn, typename Init>
111
+ static constexpr auto reduce_member_types(Fn f, Init init) {
112
+ constexpr auto dummy_type = ScalarType(0, 0, false, 0, false, NAN_NONE);
113
+ return dummy_type.reduce_members(f, init);
114
+ };
115
+
116
+ static constexpr auto id_size_bits() {
117
+ return reduce_member_types(
118
+ [](int acc, auto member) -> int {
119
+ return acc + member_id_field_width<decltype(member)>();
120
+ },
121
+ 0);
122
+ }
123
+
124
+ public:
125
+ // unique id for this scalar type that can be computed at compile time for
126
+ // c++17 template specialization this is not needed once we migrate to
127
+ // c++20 and can pass literal classes as template parameters
128
+ constexpr Id id() const {
129
+ static_assert(id_size_bits() <= sizeof(Id) * 8,
130
+ "ScalarType id is too large to be stored");
131
+
132
+ auto or_and_advance = [](std::pair<Id, uint32_t> result,
133
+ auto member) -> std::pair<Id, uint32_t> {
134
+ auto [id, bit_offset] = result;
135
+ auto constexpr bits = member_id_field_width<decltype(member)>();
136
+ return {id | (int64_t(member) & ((uint64_t(1) << bits) - 1))
137
+ << bit_offset,
138
+ bit_offset + bits};
139
+ };
140
+ return reduce_members(or_and_advance, std::pair<Id, uint32_t>{}).first;
141
+ }
142
+
143
+ // create a ScalarType from an id, for c++17 template specialization,
144
+ // this is not needed once we migrate to c++20 and can pass literal
145
+ // classes as template parameters
146
+ static constexpr ScalarType from_id(Id id) {
147
+ auto extract_and_advance = [id](auto result, auto member) {
148
+ using T = decltype(member);
149
+ auto [tuple, bit_offset] = result;
150
+ auto constexpr bits = member_id_field_width<T>();
151
+ auto extracted_val = static_cast<T>((int64_t(id) >> bit_offset) &
152
+ ((uint64_t(1) << bits) - 1));
153
+ auto new_tuple = std::tuple_cat(tuple, std::make_tuple(extracted_val));
154
+ return std::pair<decltype(new_tuple), int>{new_tuple, bit_offset + bits};
155
+ };
156
+
157
+ auto [tuple_args, _] = reduce_member_types(extract_and_advance,
158
+ std::pair<std::tuple<>, int>{});
159
+ return std::apply([](auto... args) { return ScalarType(args...); },
160
+ tuple_args);
161
+ }
162
+
163
+ constexpr int64_t size_bits() const {
164
+ return mantissa + exponent + is_signed();
165
+ }
166
+ constexpr bool is_signed() const { return signed_; }
167
+ constexpr bool is_integer() const { return exponent == 0; }
168
+ constexpr bool is_floating_point() const { return exponent > 0; }
169
+ constexpr bool is_ieee_754() const {
170
return is_floating_point() && finite_values_only == false &&
171
nan_repr == NAN_IEEE_754;
172
}
90
- bool has_nans() const { return is_floating_point() && nan_repr != NAN_NONE; }
91
- bool has_infs() const {
173
+ constexpr bool has_nans() const {
174
+ return is_floating_point() && nan_repr != NAN_NONE;
175
+ }
176
+ constexpr bool has_infs() const {
177
return is_floating_point() && finite_values_only == false;
178
}
94
- bool has_bias() const { return bias != 0; }
179
+ constexpr bool has_bias() const { return bias != 0; }
180
181
private:
182
double _floating_point_max() const {
@@ -131,7 +216,7 @@ class ScalarType {
216
return *reinterpret_cast<double*>(&double_raw);
217
}
218
134
- std::variant<int64_t, double> _raw_max() const {
219
+ constexpr std::variant<int64_t, double> _raw_max() const {
220
if (is_floating_point()) {
221
return {_floating_point_max()};
222
} else {
@@ -141,7 +226,7 @@ class ScalarType {
226
}
227
}
228
144
- std::variant<int64_t, double> _raw_min() const {
229
+ constexpr std::variant<int64_t, double> _raw_min() const {
230
if (is_floating_point()) {
231
TORCH_CHECK(is_signed(),
232
"We currently assume all floating point types are signed");
@@ -168,7 +253,7 @@ class ScalarType {
253
public:
254
// Max representable value for this scalar type.
255
// (accounting for bias if there is one)
171
- std::variant<int64_t, double> max() const {
256
+ constexpr std::variant<int64_t, double> max() const {
257
return std::visit(
258
[this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
259
_raw_max());
@@ -176,7 +261,7 @@ class ScalarType {
261
262
// Min representable value for this scalar type.
263
// (accounting for bias if there is one)
179
- std::variant<int64_t, double> min() const {
264
+ constexpr std::variant<int64_t, double> min() const {
265
return std::visit(
266
[this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
267
_raw_min());
@@ -215,7 +300,7 @@ class ScalarType {
300
}
301
}
302
218
- bool operator==(ScalarType const& other) const {
303
+ constexpr bool operator==(ScalarType const& other) const {
304
return mantissa == other.mantissa && exponent == other.exponent &&
305
bias == other.bias && signed_ == other.signed_ &&
306
finite_values_only == other.finite_values_only &&
@@ -240,23 +325,59 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
325
using Self = ScalarTypeTorch;
326
using SelfPtr = c10::intrusive_ptr<Self>;
327
328
+ static void check_size_bits(int64_t size_bits, bool signed_) {
329
+ TORCH_CHECK(
330
+ size_bits <=
331
+ std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
332
+ "size_bits bit width is too large to be represented");
333
+ }
334
+
335
+ static void check_bias(int64_t bias) {
336
+ using Bias = decltype(std::declval<Self>().bias);
337
+ TORCH_CHECK(bias <= std::numeric_limits<Bias>::max() &&
338
+ bias >= std::numeric_limits<Bias>::min(),
339
+ "bias too large or small to be represented");
340
+ }
341
+
342
+ static void check_exponent(int64_t exponent) {
343
+ TORCH_CHECK(
344
+ exponent <=
345
+ std::numeric_limits<decltype(std::declval<Self>().exponent)>::max(),
346
+ "exponent bit width is too large to be represented");
347
+ }
348
+
349
+ static void check_mantissa(int64_t mantissa) {
350
+ TORCH_CHECK(
351
+ mantissa <=
352
+ std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
353
+ "mantissa bit width is too large to be represented");
354
+ }
355
+
356
static SelfPtr int_(int64_t size_bits, c10::optional<int64_t> bias) {
357
+ check_size_bits(size_bits, true);
358
+ check_bias(bias.value_or(0));
359
return c10::make_intrusive<Self>(
360
ScalarType::int_(size_bits, bias.value_or(0)));
361
}
362
363
static SelfPtr uint(int64_t size_bits, c10::optional<int64_t> bias) {
364
+ check_size_bits(size_bits, true);
365
+ check_bias(bias.value_or(0));
366
return c10::make_intrusive<Self>(
367
ScalarType::uint(size_bits, bias.value_or(0)));
368
}
369
370
static SelfPtr float_IEEE754(int64_t exponent, int64_t mantissa) {
371
+ check_mantissa(mantissa);
372
+ check_exponent(exponent);
373
return c10::make_intrusive<Self>(
374
ScalarType::float_IEEE754(exponent, mantissa));
375
}
376
377
static SelfPtr float_(int64_t exponent, int64_t mantissa,
378
bool finite_values_only, int64_t nan_repr) {
379
+ check_mantissa(mantissa);
380
+ check_exponent(exponent);
381
return c10::make_intrusive<Self>(ScalarType::float_(
382
exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
383
}
@@ -264,7 +385,7 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
385
template <typename T>
386
static void bind_readonly_property(torch::class_<Self>& cls,
387
std::string const& name, T Base::*field) {
267
- auto getter_func = [field = std::move(field)](SelfPtr const& self) {
388
+ auto getter_func_helper = [field = std::move(field)](SelfPtr const& self) {
389
if constexpr (std::is_member_function_pointer_v<decltype(field)>) {
390
return (self.get()->*field)();
391
} else {
@@ -272,6 +393,18 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
393
}
394
};
395
396
+ auto getter_func = [field = std::move(field),
397
+ getter_func_helper = std::move(getter_func_helper)](
398
+ SelfPtr const& self) {
399
+ auto val = getter_func_helper(self);
400
+ // upconvert uint8_t, int32_t etc. to int64_t for python
401
+ if constexpr (std::is_integral_v<T>) {
402
+ return static_cast<int64_t>(val);
403
+ } else {
404
+ return val;
405
+ }
406
+ };
407
+
408
cls.def_property(name, getter_func);
409
}
410
@@ -340,6 +473,7 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
473
}
474
};
475
476
+using ScalarTypeId = int64_t;
477
using ScalarTypeTorchPtr = c10::intrusive_ptr<ScalarTypeTorch>;
478
479
// "rust style" names generally following:
@@ -379,4 +513,5 @@ static inline constexpr auto kHalf = kFE5M10;
513
static inline constexpr auto kFloat16 = kHalf;
514
static inline constexpr auto kBFloat16 = kFE8M7;
515
516
+static inline constexpr auto kFloat16Id = kFloat16.id();
517
}; // namespace vllm
csrc/quantization/gptq_marlin/gptq_marlin.cu
+168
-189
@@ -42,8 +42,8 @@ __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
42
int4* __restrict__ out_int4_ptr, int size_m,
43
int size_k, int block_rows) {}
44
45
-template <typename scalar_t, // compute dtype, half or nv_float16
46
- const int num_bits, // number of bits used for weights
45
+template <typename scalar_t, // compute dtype, half or nv_float16
46
+ const vllm::ScalarTypeId w_type_id, // weight ScalarType id
47
const int threads, // number of threads in a threadblock
48
const int thread_m_blocks, // number of 16x16 blocks in the m
49
// dimension (batchsize) of the
@@ -151,20 +151,21 @@ __device__ inline uint32_t prmt(uint32_t a) {
151
return res;
152
}
153
154
-// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
155
-// values. We mostly follow the strategy in the link below, with some small
156
-// changes:
154
+template <typename scalar_t, vllm::ScalarTypeId w_type_id>
155
+__device__ inline typename ScalarType<scalar_t>::FragB dequant(int q);
156
+
157
+//
158
+// Efficiently dequantize 4bit values packed in an int32 value into a full
159
+// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
160
+// with some small changes:
161
// - FP16:
162
// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
163
// - BF16:
164
// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
161
-template <typename scalar_t>
162
-__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit(int q) {
163
- STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
164
-}
165
-
165
+//
166
template <>
167
-__device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
167
+__device__ inline typename ScalarType<half>::FragB
168
+dequant<half, vllm::kU4B8.id()>(int q) {
169
const int LO = 0x000f000f;
170
const int HI = 0x00f000f0;
171
const int EX = 0x64006400;
@@ -187,7 +188,7 @@ __device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
188
189
template <>
190
__device__ inline typename ScalarType<nv_bfloat16>::FragB
190
-dequant_4bit<nv_bfloat16>(int q) {
191
+dequant<nv_bfloat16, vllm::kU4B8.id()>(int q) {
192
static constexpr uint32_t MASK = 0x000f000f;
193
static constexpr uint32_t EX = 0x43004300;
194
@@ -210,19 +211,64 @@ dequant_4bit<nv_bfloat16>(int q) {
211
return frag_b;
212
}
213
214
+template <>
215
+__device__ inline typename ScalarType<half>::FragB
216
+dequant<half, vllm::kU4.id()>(int q) {
217
+ const int LO = 0x000f000f;
218
+ const int HI = 0x00f000f0;
219
+ const int EX = 0x64006400;
220
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
221
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
222
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
223
+
224
+ const int SUB = 0x64006400;
225
+ const int MUL = 0x2c002c00;
226
+ const int ADD = 0xd400d400;
227
+ typename ScalarType<half>::FragB frag_b;
228
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
229
+ *reinterpret_cast<const half2*>(&SUB));
230
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
231
+ *reinterpret_cast<const half2*>(&MUL),
232
+ *reinterpret_cast<const half2*>(&ADD));
233
+ return frag_b;
234
+}
235
+
236
+template <>
237
+__device__ inline typename ScalarType<nv_bfloat16>::FragB
238
+dequant<nv_bfloat16, vllm::kU4.id()>(int q) {
239
+ static constexpr uint32_t MASK = 0x000f000f;
240
+ static constexpr uint32_t EX = 0x43004300;
241
+
242
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
243
+
244
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
245
+ q >>= 4;
246
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
247
+
248
+ typename ScalarType<nv_bfloat16>::FragB frag_b;
249
+ static constexpr uint32_t MUL = 0x3F803F80;
250
+ static constexpr uint32_t ADD = 0xC300C300;
251
+
252
+ frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
253
+ *reinterpret_cast<const nv_bfloat162*>(&MUL),
254
+ *reinterpret_cast<const nv_bfloat162*>(&ADD));
255
+ frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
256
+ *reinterpret_cast<const nv_bfloat162*>(&MUL),
257
+ *reinterpret_cast<const nv_bfloat162*>(&ADD));
258
+ return frag_b;
259
+}
260
+
261
+//
262
// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or
263
// bf16 Reference:
264
// - FP16:
265
// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
266
// - BF16:
267
// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
219
-template <typename scalar_t>
220
-__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
221
- STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
222
-}
223
-
268
+//
269
template <>
225
-__device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
270
+__device__ inline typename ScalarType<half>::FragB
271
+dequant<half, vllm::kU8B128.id()>(int q) {
272
static constexpr uint32_t mask_for_elt_01 = 0x5250;
273
static constexpr uint32_t mask_for_elt_23 = 0x5351;
274
static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
@@ -242,7 +288,7 @@ __device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
288
289
template <>
290
__device__ inline typename ScalarType<nv_bfloat16>::FragB
245
-dequant_8bit<nv_bfloat16>(int q) {
291
+dequant<nv_bfloat16, vllm::kU8B128.id()>(int q) {
292
typename ScalarType<nv_bfloat16>::FragB frag_b;
293
294
float fp32_intermediates[4];
@@ -269,68 +315,9 @@ dequant_8bit<nv_bfloat16>(int q) {
315
return frag_b;
316
}
317
272
-// Zero-point dequantizers
273
-
274
-template <typename scalar_t>
275
-__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit_zp(int q) {
276
- STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
277
-}
278
-
279
-template <>
280
-__device__ inline typename ScalarType<half>::FragB dequant_4bit_zp<half>(
281
- int q) {
282
- const int LO = 0x000f000f;
283
- const int HI = 0x00f000f0;
284
- const int EX = 0x64006400;
285
- // Guarantee that the `(a & b) | c` operations are LOP3s.
286
- int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
287
- int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
288
-
289
- const int SUB = 0x64006400;
290
- const int MUL = 0x2c002c00;
291
- const int ADD = 0xd400d400;
292
- typename ScalarType<half>::FragB frag_b;
293
- frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
294
- *reinterpret_cast<const half2*>(&SUB));
295
- frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
296
- *reinterpret_cast<const half2*>(&MUL),
297
- *reinterpret_cast<const half2*>(&ADD));
298
- return frag_b;
299
-}
300
-
301
-template <>
302
-__device__ inline typename ScalarType<nv_bfloat16>::FragB
303
-dequant_4bit_zp<nv_bfloat16>(int q) {
304
- static constexpr uint32_t MASK = 0x000f000f;
305
- static constexpr uint32_t EX = 0x43004300;
306
-
307
- // Guarantee that the `(a & b) | c` operations are LOP3s.
308
-
309
- int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
310
- q >>= 4;
311
- int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
312
-
313
- typename ScalarType<nv_bfloat16>::FragB frag_b;
314
- static constexpr uint32_t MUL = 0x3F803F80;
315
- static constexpr uint32_t ADD = 0xC300C300;
316
-
317
- frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
318
- *reinterpret_cast<const nv_bfloat162*>(&MUL),
319
- *reinterpret_cast<const nv_bfloat162*>(&ADD));
320
- frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
321
- *reinterpret_cast<const nv_bfloat162*>(&MUL),
322
- *reinterpret_cast<const nv_bfloat162*>(&ADD));
323
- return frag_b;
324
-}
325
-
326
-template <typename scalar_t>
327
-__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit_zp(int q) {
328
- STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
329
-}
330
-
318
template <>
332
-__device__ inline typename ScalarType<half>::FragB dequant_8bit_zp<half>(
333
- int q) {
319
+__device__ inline typename ScalarType<half>::FragB
320
+dequant<half, vllm::kU8.id()>(int q) {
321
static constexpr uint32_t mask_for_elt_01 = 0x5250;
322
static constexpr uint32_t mask_for_elt_23 = 0x5351;
323
static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
@@ -350,7 +337,7 @@ __device__ inline typename ScalarType<half>::FragB dequant_8bit_zp<half>(
337
338
template <>
339
__device__ inline typename ScalarType<nv_bfloat16>::FragB
353
-dequant_8bit_zp<nv_bfloat16>(int q) {
340
+dequant<nv_bfloat16, vllm::kU8.id()>(int q) {
341
typename ScalarType<nv_bfloat16>::FragB frag_b;
342
343
float fp32_intermediates[4];
@@ -517,8 +504,8 @@ __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
504
}
505
}
506
520
-template <typename scalar_t, // compute dtype, half or nv_float16
521
- const int num_bits, // number of bits used for weights
507
+template <typename scalar_t, // compute dtype, half or nv_float16
508
+ const vllm::ScalarTypeId w_type_id, // weight ScalarType id
509
const int threads, // number of threads in a threadblock
510
const int thread_m_blocks, // number of 16x16 blocks in the m
511
// dimension (batchsize) of the
@@ -568,7 +555,9 @@ __global__ void Marlin(
555
using FragS = typename ScalarType<scalar_t>::FragS;
556
using FragZP = typename ScalarType<scalar_t>::FragZP;
557
571
- constexpr int pack_factor = 32 / num_bits;
558
+ static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
559
+
560
+ constexpr int pack_factor = 32 / w_type.size_bits();
561
562
// For larger GEMMs we run multiple batchsize 64 versions in parallel for a
563
// better partitioning with less reductions
@@ -670,7 +659,7 @@ __global__ void Marlin(
659
// B sizes/strides
660
int b_gl_stride = 16 * prob_n / (pack_factor * 4);
661
constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
673
- constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
662
+ constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
663
constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
664
665
int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
@@ -1186,19 +1175,20 @@ __global__ void Marlin(
1175
if constexpr (has_zp) {
1176
FragB frag_zp_0;
1177
FragB frag_zp_1;
1189
- if constexpr (num_bits == 4) {
1190
- int zp_quant = frag_qzp[k % 2][0];
1191
- int zp_quant_shift = zp_quant >> 8;
1192
- frag_zp_0 = dequant_4bit_zp<scalar_t>(zp_quant);
1193
- frag_zp_1 = dequant_4bit_zp<scalar_t>(zp_quant_shift);
1178
+ int zp_quant_0, zp_quant_1;
1179
1180
+ if constexpr (w_type.size_bits() == 4) {
1181
+ zp_quant_0 = frag_qzp[k % 2][0];
1182
+ zp_quant_1 = zp_quant_0 >> 8;
1183
} else {
1196
- int zp_quant_0 = frag_qzp[k % 2][0];
1197
- int zp_quant_1 = frag_qzp[k % 2][1];
1198
- frag_zp_0 = dequant_8bit_zp<scalar_t>(zp_quant_0);
1199
- frag_zp_1 = dequant_8bit_zp<scalar_t>(zp_quant_1);
1184
+ static_assert(w_type.size_bits() == 8);
1185
+ zp_quant_0 = frag_qzp[k % 2][0];
1186
+ zp_quant_1 = frag_qzp[k % 2][1];
1187
}
1188
1189
+ frag_zp_0 = dequant<scalar_t, w_type_id>(zp_quant_0);
1190
+ frag_zp_1 = dequant<scalar_t, w_type_id>(zp_quant_1);
1191
+
1192
frag_zp[0] = frag_zp_0[0];
1193
frag_zp[1] = frag_zp_0[1];
1194
frag_zp[2] = frag_zp_1[0];
@@ -1211,33 +1201,21 @@ __global__ void Marlin(
1201
for (int j = 0; j < 4; j++) {
1202
FragB frag_b0;
1203
FragB frag_b1;
1214
- if constexpr (num_bits == 4) {
1215
- int b_quant = frag_b_quant[k % 2][0][j];
1216
- int b_quant_shift = b_quant >> 8;
1217
-
1218
- if constexpr (has_zp) {
1219
- frag_b0 = dequant_4bit_zp<scalar_t>(b_quant);
1220
- frag_b1 = dequant_4bit_zp<scalar_t>(b_quant_shift);
1221
-
1222
- } else {
1223
- frag_b0 = dequant_4bit<scalar_t>(b_quant);
1224
- frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
1225
- }
1204
+ int b_quant_0, b_quant_1;
1205
1206
+ if constexpr (w_type.size_bits() == 4) {
1207
+ b_quant_0 = frag_b_quant[k % 2][0][j];
1208
+ b_quant_1 = b_quant_0 >> 8;
1209
} else {
1210
+ static_assert(w_type.size_bits() == 8);
1211
int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
1229
- int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
1230
- int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
1231
-
1232
- if constexpr (has_zp) {
1233
- frag_b0 = dequant_8bit_zp<scalar_t>(b_quant_0);
1234
- frag_b1 = dequant_8bit_zp<scalar_t>(b_quant_1);
1235
- } else {
1236
- frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
1237
- frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
1238
- }
1212
+ b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
1213
+ b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
1214
}
1215
1216
+ frag_b0 = dequant<scalar_t, w_type_id>(b_quant_0);
1217
+ frag_b1 = dequant<scalar_t, w_type_id>(b_quant_1);
1218
+
1219
// Apply zero-point to frag_b0
1220
if constexpr (has_zp) {
1221
sub_zp<scalar_t>(frag_b0, frag_zp[j], 0);
@@ -1477,7 +1455,8 @@ __global__ void Marlin(
1455
1456
// For per-column quantization we finally apply the scale here (only for
1457
// 4-bit)
1480
- if constexpr (!has_act_order && group_blocks == -1 && num_bits == 4) {
1458
+ if constexpr (!has_act_order && group_blocks == -1 &&
1459
+ w_type.size_bits() == 4) {
1460
res = __hmul2(res, s[0]);
1461
}
1462
@@ -1605,7 +1584,7 @@ __global__ void Marlin(
1584
// For per-column scales, we only fetch them here in the final step before
1585
// write-out
1586
if constexpr (!has_act_order && group_blocks == -1) {
1608
- if constexpr (num_bits == 8) {
1587
+ if constexpr (w_type.size_bits() == 8) {
1588
if (s_sh_wr_pred) {
1589
cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1590
}
@@ -1622,7 +1601,7 @@ __global__ void Marlin(
1601
1602
thread_block_reduce();
1603
if constexpr (!has_act_order && group_blocks == -1) {
1625
- if constexpr (num_bits == 8) {
1604
+ if constexpr (w_type.size_bits() == 8) {
1605
cp_async_wait<0>();
1606
__syncthreads();
1607
if (threadIdx.x / 32 < thread_n_blocks / 4) {
@@ -1645,7 +1624,8 @@ __global__ void Marlin(
1624
// For 8-bit channelwise, we apply the scale before the global reduction
1625
// that converts the fp32 results to fp16 (so that we avoid possible
1626
// overflow in fp16)
1648
- if constexpr (!has_act_order && group_blocks == -1 && num_bits == 8) {
1627
+ if constexpr (!has_act_order && group_blocks == -1 &&
1628
+ w_type.size_bits() == 8) {
1629
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1630
#pragma unroll
1631
for (int i = 0; i < thread_m_blocks; i++) {
@@ -1714,20 +1694,19 @@ __global__ void Marlin(
1694
}
1695
}
1696
1717
- #define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1718
- THREAD_K_BLOCKS, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, \
1719
- NUM_THREADS) \
1720
- else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1697
+ #define __CALL_IF(W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1698
+ HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, NUM_THREADS) \
1699
+ else if (q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && \
1700
thread_n_blocks == THREAD_N_BLOCKS && \
1701
thread_k_blocks == THREAD_K_BLOCKS && \
1702
has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1703
group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1704
cudaFuncSetAttribute( \
1726
- Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1705
+ Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1706
THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1707
HAS_ZP, GROUP_BLOCKS>, \
1708
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1730
- Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1709
+ Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1710
THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1711
HAS_ZP, GROUP_BLOCKS> \
1712
<<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
@@ -1923,52 +1902,52 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1902
return exec_config_t{0, {-1, -1, -1}};
1903
}
1904
1926
- #define GPTQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1927
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1928
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1929
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1930
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1931
- \
1932
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1933
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1934
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1935
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1936
- \
1937
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1938
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1939
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1940
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1941
- \
1942
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1943
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1944
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1945
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1946
- \
1947
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1948
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1949
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1950
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1951
-
1952
- #define AWQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1953
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1954
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1955
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1956
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1957
- \
1958
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1959
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1960
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1961
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1962
- \
1963
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1964
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1965
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1966
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1967
- \
1968
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1969
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1970
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1971
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1905
+ #define GPTQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1906
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1907
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1908
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1909
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1910
+ \
1911
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1912
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1913
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1914
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1915
+ \
1916
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1917
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1918
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1919
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1920
+ \
1921
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1922
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1923
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1924
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1925
+ \
1926
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1927
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1928
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1929
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1930
+
1931
+ #define AWQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1932
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1933
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1934
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1935
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1936
+ \
1937
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1938
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1939
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1940
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1941
+ \
1942
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1943
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1944
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1945
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1946
+ \
1947
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1948
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1949
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1950
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1951
1952
template <typename scalar_t>
1953
void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
@@ -2113,23 +2092,23 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
2092
2093
if (false) {
2094
}
2116
- GPTQ_CALL_IF(4, 16, 4, 256)
2117
- GPTQ_CALL_IF(4, 8, 8, 256)
2118
- GPTQ_CALL_IF(4, 8, 4, 128)
2119
- GPTQ_CALL_IF(4, 4, 8, 128)
2120
- GPTQ_CALL_IF(8, 16, 4, 256)
2121
- GPTQ_CALL_IF(8, 8, 8, 256)
2122
- GPTQ_CALL_IF(8, 8, 4, 128)
2123
- GPTQ_CALL_IF(8, 4, 8, 128)
2124
-
2125
- AWQ_CALL_IF(4, 16, 4, 256)
2126
- AWQ_CALL_IF(4, 8, 8, 256)
2127
- AWQ_CALL_IF(4, 8, 4, 128)
2128
- AWQ_CALL_IF(4, 4, 8, 128)
2129
- AWQ_CALL_IF(8, 16, 4, 256)
2130
- AWQ_CALL_IF(8, 8, 8, 256)
2131
- AWQ_CALL_IF(8, 8, 4, 128)
2132
- AWQ_CALL_IF(8, 4, 8, 128)
2095
+ GPTQ_CALL_IF(vllm::kU4B8, 16, 4, 256)
2096
+ GPTQ_CALL_IF(vllm::kU4B8, 8, 8, 256)
2097
+ GPTQ_CALL_IF(vllm::kU4B8, 8, 4, 128)
2098
+ GPTQ_CALL_IF(vllm::kU4B8, 4, 8, 128)
2099
+ GPTQ_CALL_IF(vllm::kU8B128, 16, 4, 256)
2100
+ GPTQ_CALL_IF(vllm::kU8B128, 8, 8, 256)
2101
+ GPTQ_CALL_IF(vllm::kU8B128, 8, 4, 128)
2102
+ GPTQ_CALL_IF(vllm::kU8B128, 4, 8, 128)
2103
+
2104
+ AWQ_CALL_IF(vllm::kU4, 16, 4, 256)
2105
+ AWQ_CALL_IF(vllm::kU4, 8, 8, 256)
2106
+ AWQ_CALL_IF(vllm::kU4, 8, 4, 128)
2107
+ AWQ_CALL_IF(vllm::kU4, 4, 8, 128)
2108
+ AWQ_CALL_IF(vllm::kU8, 16, 4, 256)
2109
+ AWQ_CALL_IF(vllm::kU8, 8, 8, 256)
2110
+ AWQ_CALL_IF(vllm::kU8, 8, 4, 128)
2111
+ AWQ_CALL_IF(vllm::kU8, 4, 8, 128)
2112
else {
2113
TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
2114
", ", prob_k, "]", ", has_act_order = ", has_act_order,