@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6aa33cb2dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Use scalar type to dispatch to different `gptq_marlin` kernels (#7323)

Lucas Wilkinson committed Aug 12, 2024 at 14:40 UTC 6aa33cb2ddd769e764a3312627cab5bffaa383cc
2 files changed +334 -220
csrc/core/scalar_type.hpp
+166 -31
@@ -20,7 +20,7 @@ namespace vllm {
20 //
21 class ScalarType {
22 public:
23 - enum NanRepr : int64_t {
23 + enum NanRepr : uint8_t {
24 NAN_NONE = 0, // nans are not supported
25 NAN_IEEE_754 = 1, // nans are: exp all 1s, mantissa not all 0s
26 NAN_EXTD_RANGE_MAX_MIN = 2, // nans are: exp all 1s, mantissa all 1s
@@ -28,33 +28,33 @@ class ScalarType {
28 NAN_REPR_ID_MAX
29 };
30
31 - constexpr ScalarType(bool signed_, int64_t exponent, int64_t mantissa,
32 - int64_t bias, bool finite_values_only = false,
31 + constexpr ScalarType(uint8_t exponent, uint8_t mantissa, bool signed_,
32 + int32_t bias, bool finite_values_only = false,
33 NanRepr nan_repr = NAN_IEEE_754)
34 : exponent(exponent),
35 mantissa(mantissa),
36 - bias(bias),
36 signed_(signed_),
37 + bias(bias),
38 finite_values_only(finite_values_only),
39 nan_repr(nan_repr){};
40
41 - static constexpr ScalarType int_(int64_t size_bits, int64_t bias = 0) {
42 - return ScalarType(true, 0, size_bits - 1, bias);
41 + static constexpr ScalarType int_(uint8_t size_bits, int32_t bias = 0) {
42 + return ScalarType(0, size_bits - 1, true, bias);
43 }
44
45 - static constexpr ScalarType uint(int64_t size_bits, int64_t bias = 0) {
46 - return ScalarType(false, 0, size_bits, bias);
45 + static constexpr ScalarType uint(uint8_t size_bits, int32_t bias = 0) {
46 + return ScalarType(0, size_bits, false, bias);
47 }
48
49 // IEEE 754 compliant floating point type
50 - static constexpr ScalarType float_IEEE754(int64_t exponent,
51 - int64_t mantissa) {
50 + static constexpr ScalarType float_IEEE754(uint8_t exponent,
51 + uint8_t mantissa) {
52 TORCH_CHECK(mantissa > 0 && exponent > 0);
53 - return ScalarType(true, exponent, mantissa, 0, false, NAN_IEEE_754);
53 + return ScalarType(exponent, mantissa, true, 0, false, NAN_IEEE_754);
54 }
55
56 // IEEE 754 non-compliant floating point type
57 - static constexpr ScalarType float_(int64_t exponent, int64_t mantissa,
57 + static constexpr ScalarType float_(uint8_t exponent, uint8_t mantissa,
58 bool finite_values_only,
59 NanRepr nan_repr) {
60 TORCH_CHECK(nan_repr < NAN_REPR_ID_MAX, "Invalid NanRepr");
@@ -62,36 +62,121 @@ class ScalarType {
62 TORCH_CHECK(nan_repr != NAN_IEEE_754,
63 "use `float_IEEE754` constructor for floating point types that "
64 "follow IEEE 754 conventions");
65 - return ScalarType(true, exponent, mantissa, 0, finite_values_only,
65 + return ScalarType(exponent, mantissa, true, 0, finite_values_only,
66 nan_repr);
67 }
68
69 - int64_t const exponent; // size of the exponent field (0 for integer types)
70 - int64_t const mantissa; // size of the mantissa field (size of the integer
69 + uint8_t const exponent; // size of the exponent field (0 for integer types)
70 + uint8_t const mantissa; // size of the mantissa field (size of the integer
71 // excluding the sign bit for integer types)
72 - int64_t const bias; // stored values equal value + bias,
73 - // used for quantized type
72 bool const signed_; // flag if the type supports negative numbers (i.e. has a
73 // sign bit)
74 + int32_t const bias; // stored values equal value + bias,
75 + // used for quantized type
76
77 // Extra Floating point info
78 bool const finite_values_only; // i.e. no +/-inf if true
79 NanRepr const nan_repr; // how NaNs are represented
80 // (not applicable for integer types)
81
82 - int64_t size_bits() const { return mantissa + exponent + is_signed(); }
83 - bool is_signed() const { return signed_; }
84 - bool is_integer() const { return exponent == 0; }
85 - bool is_floating_point() const { return exponent > 0; }
86 - bool is_ieee_754() const {
82 + using Id = int64_t;
83 +
84 + private:
85 + // Field size in id
86 + template <typename T_>
87 + static constexpr size_t member_id_field_width() {
88 + using T = std::decay_t<T_>;
89 + return std::is_same_v<T, bool> ? 1 : sizeof(T) * 8;
90 + }
91 +
92 + template <typename Fn, typename Init, typename Member, typename... Rest>
93 + static constexpr auto reduce_members_helper(Fn f, Init val, Member member,
94 + Rest... rest) {
95 + auto new_val = f(val, member);
96 + if constexpr (sizeof...(rest) > 0) {
97 + return reduce_members_helper(f, new_val, rest...);
98 + } else {
99 + return new_val;
100 + };
101 + }
102 +
103 + template <typename Fn, typename Init>
104 + constexpr auto reduce_members(Fn f, Init init) const {
105 + // Should be in constructor order for `from_id`
106 + return reduce_members_helper(f, init, exponent, mantissa, signed_, bias,
107 + finite_values_only, nan_repr);
108 + };
109 +
110 + template <typename Fn, typename Init>
111 + static constexpr auto reduce_member_types(Fn f, Init init) {
112 + constexpr auto dummy_type = ScalarType(0, 0, false, 0, false, NAN_NONE);
113 + return dummy_type.reduce_members(f, init);
114 + };
115 +
116 + static constexpr auto id_size_bits() {
117 + return reduce_member_types(
118 + [](int acc, auto member) -> int {
119 + return acc + member_id_field_width<decltype(member)>();
120 + },
121 + 0);
122 + }
123 +
124 + public:
125 + // unique id for this scalar type that can be computed at compile time for
126 + // c++17 template specialization this is not needed once we migrate to
127 + // c++20 and can pass literal classes as template parameters
128 + constexpr Id id() const {
129 + static_assert(id_size_bits() <= sizeof(Id) * 8,
130 + "ScalarType id is too large to be stored");
131 +
132 + auto or_and_advance = [](std::pair<Id, uint32_t> result,
133 + auto member) -> std::pair<Id, uint32_t> {
134 + auto [id, bit_offset] = result;
135 + auto constexpr bits = member_id_field_width<decltype(member)>();
136 + return {id | (int64_t(member) & ((uint64_t(1) << bits) - 1))
137 + << bit_offset,
138 + bit_offset + bits};
139 + };
140 + return reduce_members(or_and_advance, std::pair<Id, uint32_t>{}).first;
141 + }
142 +
143 + // create a ScalarType from an id, for c++17 template specialization,
144 + // this is not needed once we migrate to c++20 and can pass literal
145 + // classes as template parameters
146 + static constexpr ScalarType from_id(Id id) {
147 + auto extract_and_advance = [id](auto result, auto member) {
148 + using T = decltype(member);
149 + auto [tuple, bit_offset] = result;
150 + auto constexpr bits = member_id_field_width<T>();
151 + auto extracted_val = static_cast<T>((int64_t(id) >> bit_offset) &
152 + ((uint64_t(1) << bits) - 1));
153 + auto new_tuple = std::tuple_cat(tuple, std::make_tuple(extracted_val));
154 + return std::pair<decltype(new_tuple), int>{new_tuple, bit_offset + bits};
155 + };
156 +
157 + auto [tuple_args, _] = reduce_member_types(extract_and_advance,
158 + std::pair<std::tuple<>, int>{});
159 + return std::apply([](auto... args) { return ScalarType(args...); },
160 + tuple_args);
161 + }
162 +
163 + constexpr int64_t size_bits() const {
164 + return mantissa + exponent + is_signed();
165 + }
166 + constexpr bool is_signed() const { return signed_; }
167 + constexpr bool is_integer() const { return exponent == 0; }
168 + constexpr bool is_floating_point() const { return exponent > 0; }
169 + constexpr bool is_ieee_754() const {
170 return is_floating_point() && finite_values_only == false &&
171 nan_repr == NAN_IEEE_754;
172 }
90 - bool has_nans() const { return is_floating_point() && nan_repr != NAN_NONE; }
91 - bool has_infs() const {
173 + constexpr bool has_nans() const {
174 + return is_floating_point() && nan_repr != NAN_NONE;
175 + }
176 + constexpr bool has_infs() const {
177 return is_floating_point() && finite_values_only == false;
178 }
94 - bool has_bias() const { return bias != 0; }
179 + constexpr bool has_bias() const { return bias != 0; }
180
181 private:
182 double _floating_point_max() const {
@@ -131,7 +216,7 @@ class ScalarType {
216 return *reinterpret_cast<double*>(&double_raw);
217 }
218
134 - std::variant<int64_t, double> _raw_max() const {
219 + constexpr std::variant<int64_t, double> _raw_max() const {
220 if (is_floating_point()) {
221 return {_floating_point_max()};
222 } else {
@@ -141,7 +226,7 @@ class ScalarType {
226 }
227 }
228
144 - std::variant<int64_t, double> _raw_min() const {
229 + constexpr std::variant<int64_t, double> _raw_min() const {
230 if (is_floating_point()) {
231 TORCH_CHECK(is_signed(),
232 "We currently assume all floating point types are signed");
@@ -168,7 +253,7 @@ class ScalarType {
253 public:
254 // Max representable value for this scalar type.
255 // (accounting for bias if there is one)
171 - std::variant<int64_t, double> max() const {
256 + constexpr std::variant<int64_t, double> max() const {
257 return std::visit(
258 [this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
259 _raw_max());
@@ -176,7 +261,7 @@ class ScalarType {
261
262 // Min representable value for this scalar type.
263 // (accounting for bias if there is one)
179 - std::variant<int64_t, double> min() const {
264 + constexpr std::variant<int64_t, double> min() const {
265 return std::visit(
266 [this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
267 _raw_min());
@@ -215,7 +300,7 @@ class ScalarType {
300 }
301 }
302
218 - bool operator==(ScalarType const& other) const {
303 + constexpr bool operator==(ScalarType const& other) const {
304 return mantissa == other.mantissa && exponent == other.exponent &&
305 bias == other.bias && signed_ == other.signed_ &&
306 finite_values_only == other.finite_values_only &&
@@ -240,23 +325,59 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
325 using Self = ScalarTypeTorch;
326 using SelfPtr = c10::intrusive_ptr<Self>;
327
328 + static void check_size_bits(int64_t size_bits, bool signed_) {
329 + TORCH_CHECK(
330 + size_bits <=
331 + std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
332 + "size_bits bit width is too large to be represented");
333 + }
334 +
335 + static void check_bias(int64_t bias) {
336 + using Bias = decltype(std::declval<Self>().bias);
337 + TORCH_CHECK(bias <= std::numeric_limits<Bias>::max() &&
338 + bias >= std::numeric_limits<Bias>::min(),
339 + "bias too large or small to be represented");
340 + }
341 +
342 + static void check_exponent(int64_t exponent) {
343 + TORCH_CHECK(
344 + exponent <=
345 + std::numeric_limits<decltype(std::declval<Self>().exponent)>::max(),
346 + "exponent bit width is too large to be represented");
347 + }
348 +
349 + static void check_mantissa(int64_t mantissa) {
350 + TORCH_CHECK(
351 + mantissa <=
352 + std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
353 + "mantissa bit width is too large to be represented");
354 + }
355 +
356 static SelfPtr int_(int64_t size_bits, c10::optional<int64_t> bias) {
357 + check_size_bits(size_bits, true);
358 + check_bias(bias.value_or(0));
359 return c10::make_intrusive<Self>(
360 ScalarType::int_(size_bits, bias.value_or(0)));
361 }
362
363 static SelfPtr uint(int64_t size_bits, c10::optional<int64_t> bias) {
364 + check_size_bits(size_bits, true);
365 + check_bias(bias.value_or(0));
366 return c10::make_intrusive<Self>(
367 ScalarType::uint(size_bits, bias.value_or(0)));
368 }
369
370 static SelfPtr float_IEEE754(int64_t exponent, int64_t mantissa) {
371 + check_mantissa(mantissa);
372 + check_exponent(exponent);
373 return c10::make_intrusive<Self>(
374 ScalarType::float_IEEE754(exponent, mantissa));
375 }
376
377 static SelfPtr float_(int64_t exponent, int64_t mantissa,
378 bool finite_values_only, int64_t nan_repr) {
379 + check_mantissa(mantissa);
380 + check_exponent(exponent);
381 return c10::make_intrusive<Self>(ScalarType::float_(
382 exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
383 }
@@ -264,7 +385,7 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
385 template <typename T>
386 static void bind_readonly_property(torch::class_<Self>& cls,
387 std::string const& name, T Base::*field) {
267 - auto getter_func = [field = std::move(field)](SelfPtr const& self) {
388 + auto getter_func_helper = [field = std::move(field)](SelfPtr const& self) {
389 if constexpr (std::is_member_function_pointer_v<decltype(field)>) {
390 return (self.get()->*field)();
391 } else {
@@ -272,6 +393,18 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
393 }
394 };
395
396 + auto getter_func = [field = std::move(field),
397 + getter_func_helper = std::move(getter_func_helper)](
398 + SelfPtr const& self) {
399 + auto val = getter_func_helper(self);
400 + // upconvert uint8_t, int32_t etc. to int64_t for python
401 + if constexpr (std::is_integral_v<T>) {
402 + return static_cast<int64_t>(val);
403 + } else {
404 + return val;
405 + }
406 + };
407 +
408 cls.def_property(name, getter_func);
409 }
410
@@ -340,6 +473,7 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
473 }
474 };
475
476 +using ScalarTypeId = int64_t;
477 using ScalarTypeTorchPtr = c10::intrusive_ptr<ScalarTypeTorch>;
478
479 // "rust style" names generally following:
@@ -379,4 +513,5 @@ static inline constexpr auto kHalf = kFE5M10;
513 static inline constexpr auto kFloat16 = kHalf;
514 static inline constexpr auto kBFloat16 = kFE8M7;
515
516 +static inline constexpr auto kFloat16Id = kFloat16.id();
517 }; // namespace vllm
csrc/quantization/gptq_marlin/gptq_marlin.cu
+168 -189
@@ -42,8 +42,8 @@ __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
42 int4* __restrict__ out_int4_ptr, int size_m,
43 int size_k, int block_rows) {}
44
45 -template <typename scalar_t, // compute dtype, half or nv_float16
46 - const int num_bits, // number of bits used for weights
45 +template <typename scalar_t, // compute dtype, half or nv_float16
46 + const vllm::ScalarTypeId w_type_id, // weight ScalarType id
47 const int threads, // number of threads in a threadblock
48 const int thread_m_blocks, // number of 16x16 blocks in the m
49 // dimension (batchsize) of the
@@ -151,20 +151,21 @@ __device__ inline uint32_t prmt(uint32_t a) {
151 return res;
152 }
153
154 -// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
155 -// values. We mostly follow the strategy in the link below, with some small
156 -// changes:
154 +template <typename scalar_t, vllm::ScalarTypeId w_type_id>
155 +__device__ inline typename ScalarType<scalar_t>::FragB dequant(int q);
156 +
157 +//
158 +// Efficiently dequantize 4bit values packed in an int32 value into a full
159 +// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
160 +// with some small changes:
161 // - FP16:
162 // https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
163 // - BF16:
164 // https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
161 -template <typename scalar_t>
162 -__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit(int q) {
163 - STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
164 -}
165 -
165 +//
166 template <>
167 -__device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
167 +__device__ inline typename ScalarType<half>::FragB
168 +dequant<half, vllm::kU4B8.id()>(int q) {
169 const int LO = 0x000f000f;
170 const int HI = 0x00f000f0;
171 const int EX = 0x64006400;
@@ -187,7 +188,7 @@ __device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
188
189 template <>
190 __device__ inline typename ScalarType<nv_bfloat16>::FragB
190 -dequant_4bit<nv_bfloat16>(int q) {
191 +dequant<nv_bfloat16, vllm::kU4B8.id()>(int q) {
192 static constexpr uint32_t MASK = 0x000f000f;
193 static constexpr uint32_t EX = 0x43004300;
194
@@ -210,19 +211,64 @@ dequant_4bit<nv_bfloat16>(int q) {
211 return frag_b;
212 }
213
214 +template <>
215 +__device__ inline typename ScalarType<half>::FragB
216 +dequant<half, vllm::kU4.id()>(int q) {
217 + const int LO = 0x000f000f;
218 + const int HI = 0x00f000f0;
219 + const int EX = 0x64006400;
220 + // Guarantee that the `(a & b) | c` operations are LOP3s.
221 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
222 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
223 +
224 + const int SUB = 0x64006400;
225 + const int MUL = 0x2c002c00;
226 + const int ADD = 0xd400d400;
227 + typename ScalarType<half>::FragB frag_b;
228 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
229 + *reinterpret_cast<const half2*>(&SUB));
230 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
231 + *reinterpret_cast<const half2*>(&MUL),
232 + *reinterpret_cast<const half2*>(&ADD));
233 + return frag_b;
234 +}
235 +
236 +template <>
237 +__device__ inline typename ScalarType<nv_bfloat16>::FragB
238 +dequant<nv_bfloat16, vllm::kU4.id()>(int q) {
239 + static constexpr uint32_t MASK = 0x000f000f;
240 + static constexpr uint32_t EX = 0x43004300;
241 +
242 + // Guarantee that the `(a & b) | c` operations are LOP3s.
243 +
244 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
245 + q >>= 4;
246 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
247 +
248 + typename ScalarType<nv_bfloat16>::FragB frag_b;
249 + static constexpr uint32_t MUL = 0x3F803F80;
250 + static constexpr uint32_t ADD = 0xC300C300;
251 +
252 + frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
253 + *reinterpret_cast<const nv_bfloat162*>(&MUL),
254 + *reinterpret_cast<const nv_bfloat162*>(&ADD));
255 + frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
256 + *reinterpret_cast<const nv_bfloat162*>(&MUL),
257 + *reinterpret_cast<const nv_bfloat162*>(&ADD));
258 + return frag_b;
259 +}
260 +
261 +//
262 // Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or
263 // bf16 Reference:
264 // - FP16:
265 // https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
266 // - BF16:
267 // https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
219 -template <typename scalar_t>
220 -__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
221 - STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
222 -}
223 -
268 +//
269 template <>
225 -__device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
270 +__device__ inline typename ScalarType<half>::FragB
271 +dequant<half, vllm::kU8B128.id()>(int q) {
272 static constexpr uint32_t mask_for_elt_01 = 0x5250;
273 static constexpr uint32_t mask_for_elt_23 = 0x5351;
274 static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
@@ -242,7 +288,7 @@ __device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
288
289 template <>
290 __device__ inline typename ScalarType<nv_bfloat16>::FragB
245 -dequant_8bit<nv_bfloat16>(int q) {
291 +dequant<nv_bfloat16, vllm::kU8B128.id()>(int q) {
292 typename ScalarType<nv_bfloat16>::FragB frag_b;
293
294 float fp32_intermediates[4];
@@ -269,68 +315,9 @@ dequant_8bit<nv_bfloat16>(int q) {
315 return frag_b;
316 }
317
272 -// Zero-point dequantizers
273 -
274 -template <typename scalar_t>
275 -__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit_zp(int q) {
276 - STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
277 -}
278 -
279 -template <>
280 -__device__ inline typename ScalarType<half>::FragB dequant_4bit_zp<half>(
281 - int q) {
282 - const int LO = 0x000f000f;
283 - const int HI = 0x00f000f0;
284 - const int EX = 0x64006400;
285 - // Guarantee that the `(a & b) | c` operations are LOP3s.
286 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
287 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
288 -
289 - const int SUB = 0x64006400;
290 - const int MUL = 0x2c002c00;
291 - const int ADD = 0xd400d400;
292 - typename ScalarType<half>::FragB frag_b;
293 - frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
294 - *reinterpret_cast<const half2*>(&SUB));
295 - frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
296 - *reinterpret_cast<const half2*>(&MUL),
297 - *reinterpret_cast<const half2*>(&ADD));
298 - return frag_b;
299 -}
300 -
301 -template <>
302 -__device__ inline typename ScalarType<nv_bfloat16>::FragB
303 -dequant_4bit_zp<nv_bfloat16>(int q) {
304 - static constexpr uint32_t MASK = 0x000f000f;
305 - static constexpr uint32_t EX = 0x43004300;
306 -
307 - // Guarantee that the `(a & b) | c` operations are LOP3s.
308 -
309 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
310 - q >>= 4;
311 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX);
312 -
313 - typename ScalarType<nv_bfloat16>::FragB frag_b;
314 - static constexpr uint32_t MUL = 0x3F803F80;
315 - static constexpr uint32_t ADD = 0xC300C300;
316 -
317 - frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
318 - *reinterpret_cast<const nv_bfloat162*>(&MUL),
319 - *reinterpret_cast<const nv_bfloat162*>(&ADD));
320 - frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
321 - *reinterpret_cast<const nv_bfloat162*>(&MUL),
322 - *reinterpret_cast<const nv_bfloat162*>(&ADD));
323 - return frag_b;
324 -}
325 -
326 -template <typename scalar_t>
327 -__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit_zp(int q) {
328 - STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
329 -}
330 -
318 template <>
332 -__device__ inline typename ScalarType<half>::FragB dequant_8bit_zp<half>(
333 - int q) {
319 +__device__ inline typename ScalarType<half>::FragB
320 +dequant<half, vllm::kU8.id()>(int q) {
321 static constexpr uint32_t mask_for_elt_01 = 0x5250;
322 static constexpr uint32_t mask_for_elt_23 = 0x5351;
323 static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
@@ -350,7 +337,7 @@ __device__ inline typename ScalarType<half>::FragB dequant_8bit_zp<half>(
337
338 template <>
339 __device__ inline typename ScalarType<nv_bfloat16>::FragB
353 -dequant_8bit_zp<nv_bfloat16>(int q) {
340 +dequant<nv_bfloat16, vllm::kU8.id()>(int q) {
341 typename ScalarType<nv_bfloat16>::FragB frag_b;
342
343 float fp32_intermediates[4];
@@ -517,8 +504,8 @@ __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
504 }
505 }
506
520 -template <typename scalar_t, // compute dtype, half or nv_float16
521 - const int num_bits, // number of bits used for weights
507 +template <typename scalar_t, // compute dtype, half or nv_float16
508 + const vllm::ScalarTypeId w_type_id, // weight ScalarType id
509 const int threads, // number of threads in a threadblock
510 const int thread_m_blocks, // number of 16x16 blocks in the m
511 // dimension (batchsize) of the
@@ -568,7 +555,9 @@ __global__ void Marlin(
555 using FragS = typename ScalarType<scalar_t>::FragS;
556 using FragZP = typename ScalarType<scalar_t>::FragZP;
557
571 - constexpr int pack_factor = 32 / num_bits;
558 + static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
559 +
560 + constexpr int pack_factor = 32 / w_type.size_bits();
561
562 // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
563 // better partitioning with less reductions
@@ -670,7 +659,7 @@ __global__ void Marlin(
659 // B sizes/strides
660 int b_gl_stride = 16 * prob_n / (pack_factor * 4);
661 constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
673 - constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
662 + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
663 constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
664
665 int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
@@ -1186,19 +1175,20 @@ __global__ void Marlin(
1175 if constexpr (has_zp) {
1176 FragB frag_zp_0;
1177 FragB frag_zp_1;
1189 - if constexpr (num_bits == 4) {
1190 - int zp_quant = frag_qzp[k % 2][0];
1191 - int zp_quant_shift = zp_quant >> 8;
1192 - frag_zp_0 = dequant_4bit_zp<scalar_t>(zp_quant);
1193 - frag_zp_1 = dequant_4bit_zp<scalar_t>(zp_quant_shift);
1178 + int zp_quant_0, zp_quant_1;
1179
1180 + if constexpr (w_type.size_bits() == 4) {
1181 + zp_quant_0 = frag_qzp[k % 2][0];
1182 + zp_quant_1 = zp_quant_0 >> 8;
1183 } else {
1196 - int zp_quant_0 = frag_qzp[k % 2][0];
1197 - int zp_quant_1 = frag_qzp[k % 2][1];
1198 - frag_zp_0 = dequant_8bit_zp<scalar_t>(zp_quant_0);
1199 - frag_zp_1 = dequant_8bit_zp<scalar_t>(zp_quant_1);
1184 + static_assert(w_type.size_bits() == 8);
1185 + zp_quant_0 = frag_qzp[k % 2][0];
1186 + zp_quant_1 = frag_qzp[k % 2][1];
1187 }
1188
1189 + frag_zp_0 = dequant<scalar_t, w_type_id>(zp_quant_0);
1190 + frag_zp_1 = dequant<scalar_t, w_type_id>(zp_quant_1);
1191 +
1192 frag_zp[0] = frag_zp_0[0];
1193 frag_zp[1] = frag_zp_0[1];
1194 frag_zp[2] = frag_zp_1[0];
@@ -1211,33 +1201,21 @@ __global__ void Marlin(
1201 for (int j = 0; j < 4; j++) {
1202 FragB frag_b0;
1203 FragB frag_b1;
1214 - if constexpr (num_bits == 4) {
1215 - int b_quant = frag_b_quant[k % 2][0][j];
1216 - int b_quant_shift = b_quant >> 8;
1217 -
1218 - if constexpr (has_zp) {
1219 - frag_b0 = dequant_4bit_zp<scalar_t>(b_quant);
1220 - frag_b1 = dequant_4bit_zp<scalar_t>(b_quant_shift);
1221 -
1222 - } else {
1223 - frag_b0 = dequant_4bit<scalar_t>(b_quant);
1224 - frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
1225 - }
1204 + int b_quant_0, b_quant_1;
1205
1206 + if constexpr (w_type.size_bits() == 4) {
1207 + b_quant_0 = frag_b_quant[k % 2][0][j];
1208 + b_quant_1 = b_quant_0 >> 8;
1209 } else {
1210 + static_assert(w_type.size_bits() == 8);
1211 int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
1229 - int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
1230 - int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
1231 -
1232 - if constexpr (has_zp) {
1233 - frag_b0 = dequant_8bit_zp<scalar_t>(b_quant_0);
1234 - frag_b1 = dequant_8bit_zp<scalar_t>(b_quant_1);
1235 - } else {
1236 - frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
1237 - frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
1238 - }
1212 + b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
1213 + b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
1214 }
1215
1216 + frag_b0 = dequant<scalar_t, w_type_id>(b_quant_0);
1217 + frag_b1 = dequant<scalar_t, w_type_id>(b_quant_1);
1218 +
1219 // Apply zero-point to frag_b0
1220 if constexpr (has_zp) {
1221 sub_zp<scalar_t>(frag_b0, frag_zp[j], 0);
@@ -1477,7 +1455,8 @@ __global__ void Marlin(
1455
1456 // For per-column quantization we finally apply the scale here (only for
1457 // 4-bit)
1480 - if constexpr (!has_act_order && group_blocks == -1 && num_bits == 4) {
1458 + if constexpr (!has_act_order && group_blocks == -1 &&
1459 + w_type.size_bits() == 4) {
1460 res = __hmul2(res, s[0]);
1461 }
1462
@@ -1605,7 +1584,7 @@ __global__ void Marlin(
1584 // For per-column scales, we only fetch them here in the final step before
1585 // write-out
1586 if constexpr (!has_act_order && group_blocks == -1) {
1608 - if constexpr (num_bits == 8) {
1587 + if constexpr (w_type.size_bits() == 8) {
1588 if (s_sh_wr_pred) {
1589 cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1590 }
@@ -1622,7 +1601,7 @@ __global__ void Marlin(
1601
1602 thread_block_reduce();
1603 if constexpr (!has_act_order && group_blocks == -1) {
1625 - if constexpr (num_bits == 8) {
1604 + if constexpr (w_type.size_bits() == 8) {
1605 cp_async_wait<0>();
1606 __syncthreads();
1607 if (threadIdx.x / 32 < thread_n_blocks / 4) {
@@ -1645,7 +1624,8 @@ __global__ void Marlin(
1624 // For 8-bit channelwise, we apply the scale before the global reduction
1625 // that converts the fp32 results to fp16 (so that we avoid possible
1626 // overflow in fp16)
1648 - if constexpr (!has_act_order && group_blocks == -1 && num_bits == 8) {
1627 + if constexpr (!has_act_order && group_blocks == -1 &&
1628 + w_type.size_bits() == 8) {
1629 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1630 #pragma unroll
1631 for (int i = 0; i < thread_m_blocks; i++) {
@@ -1714,20 +1694,19 @@ __global__ void Marlin(
1694 }
1695 }
1696
1717 - #define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1718 - THREAD_K_BLOCKS, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, \
1719 - NUM_THREADS) \
1720 - else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1697 + #define __CALL_IF(W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1698 + HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, NUM_THREADS) \
1699 + else if (q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && \
1700 thread_n_blocks == THREAD_N_BLOCKS && \
1701 thread_k_blocks == THREAD_K_BLOCKS && \
1702 has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1703 group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1704 cudaFuncSetAttribute( \
1726 - Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1705 + Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1706 THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1707 HAS_ZP, GROUP_BLOCKS>, \
1708 cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1730 - Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1709 + Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1710 THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1711 HAS_ZP, GROUP_BLOCKS> \
1712 <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
@@ -1923,52 +1902,52 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1902 return exec_config_t{0, {-1, -1, -1}};
1903 }
1904
1926 - #define GPTQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1927 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1928 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1929 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1930 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1931 - \
1932 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1933 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1934 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1935 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1936 - \
1937 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1938 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1939 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1940 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1941 - \
1942 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1943 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1944 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1945 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1946 - \
1947 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1948 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1949 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1950 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1951 -
1952 - #define AWQ_CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1953 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1954 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1955 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1956 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1957 - \
1958 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1959 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1960 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1961 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1962 - \
1963 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1964 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1965 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1966 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1967 - \
1968 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1969 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1970 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1971 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1905 + #define GPTQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1906 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1907 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1908 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1909 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1910 + \
1911 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1912 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1913 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1914 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1915 + \
1916 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1917 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1918 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1919 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1920 + \
1921 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1922 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1923 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1924 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1925 + \
1926 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1927 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1928 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1929 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1930 +
1931 + #define AWQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1932 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1933 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1934 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1935 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1936 + \
1937 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1938 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1939 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1940 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1941 + \
1942 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1943 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1944 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1945 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
1946 + \
1947 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1948 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1949 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1950 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1951
1952 template <typename scalar_t>
1953 void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
@@ -2113,23 +2092,23 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
2092
2093 if (false) {
2094 }
2116 - GPTQ_CALL_IF(4, 16, 4, 256)
2117 - GPTQ_CALL_IF(4, 8, 8, 256)
2118 - GPTQ_CALL_IF(4, 8, 4, 128)
2119 - GPTQ_CALL_IF(4, 4, 8, 128)
2120 - GPTQ_CALL_IF(8, 16, 4, 256)
2121 - GPTQ_CALL_IF(8, 8, 8, 256)
2122 - GPTQ_CALL_IF(8, 8, 4, 128)
2123 - GPTQ_CALL_IF(8, 4, 8, 128)
2124 -
2125 - AWQ_CALL_IF(4, 16, 4, 256)
2126 - AWQ_CALL_IF(4, 8, 8, 256)
2127 - AWQ_CALL_IF(4, 8, 4, 128)
2128 - AWQ_CALL_IF(4, 4, 8, 128)
2129 - AWQ_CALL_IF(8, 16, 4, 256)
2130 - AWQ_CALL_IF(8, 8, 8, 256)
2131 - AWQ_CALL_IF(8, 8, 4, 128)
2132 - AWQ_CALL_IF(8, 4, 8, 128)
2095 + GPTQ_CALL_IF(vllm::kU4B8, 16, 4, 256)
2096 + GPTQ_CALL_IF(vllm::kU4B8, 8, 8, 256)
2097 + GPTQ_CALL_IF(vllm::kU4B8, 8, 4, 128)
2098 + GPTQ_CALL_IF(vllm::kU4B8, 4, 8, 128)
2099 + GPTQ_CALL_IF(vllm::kU8B128, 16, 4, 256)
2100 + GPTQ_CALL_IF(vllm::kU8B128, 8, 8, 256)
2101 + GPTQ_CALL_IF(vllm::kU8B128, 8, 4, 128)
2102 + GPTQ_CALL_IF(vllm::kU8B128, 4, 8, 128)
2103 +
2104 + AWQ_CALL_IF(vllm::kU4, 16, 4, 256)
2105 + AWQ_CALL_IF(vllm::kU4, 8, 8, 256)
2106 + AWQ_CALL_IF(vllm::kU4, 8, 4, 128)
2107 + AWQ_CALL_IF(vllm::kU4, 4, 8, 128)
2108 + AWQ_CALL_IF(vllm::kU8, 16, 4, 256)
2109 + AWQ_CALL_IF(vllm::kU8, 8, 8, 256)
2110 + AWQ_CALL_IF(vllm::kU8, 8, 4, 128)
2111 + AWQ_CALL_IF(vllm::kU8, 4, 8, 128)
2112 else {
2113 TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
2114 ", ", prob_k, "]", ", has_act_order = ", has_act_order,