A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Disambiguate quantized types via a new ScalarType (#6396)
Lucas Wilkinson committed
Aug 2, 2024 at 16:51 UTC
a8d604ca2a2912b3a5352821c53c080383580df1
29 files changed
+1107
-352
CMakeLists.txt
+35
-17
@@ -66,6 +66,39 @@ endif()
66
#
67
find_package(Torch REQUIRED)
68
69
+#
70
+# Add the `default` target which detects which extensions should be
71
+# built based on platform/architecture. This is the same logic that
72
+# setup.py uses to select which extensions should be built and should
73
+# be kept in sync.
74
+#
75
+# The `default` target makes direct use of cmake easier since knowledge
76
+# of which extensions are supported has been factored in, e.g.
77
+#
78
+# mkdir build && cd build
79
+# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
80
+# cmake --build . --target default
81
+#
82
+add_custom_target(default)
83
+message(STATUS "Enabling core extension.")
84
+
85
+# Define _core_C extension
86
+# built for (almost) every target platform, (excludes TPU and Neuron)
87
+
88
+set(VLLM_EXT_SRC
89
+ "csrc/core/torch_bindings.cpp")
90
+
91
+define_gpu_extension_target(
92
+ _core_C
93
+ DESTINATION vllm
94
+ LANGUAGE CXX
95
+ SOURCES ${VLLM_EXT_SRC}
96
+ COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
97
+ USE_SABI 3
98
+ WITH_SOABI)
99
+
100
+add_dependencies(default _core_C)
101
+
102
#
103
# Forward the non-CUDA device extensions to external CMake scripts.
104
#
@@ -74,7 +107,7 @@ if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND
107
if (VLLM_TARGET_DEVICE STREQUAL "cpu")
108
include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake)
109
else()
77
- message(FATAL_ERROR "Unsupported vLLM target device: ${VLLM_TARGET_DEVICE}")
110
+ return()
111
endif()
112
return()
113
endif()
@@ -132,7 +165,7 @@ if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
165
endif()
166
167
#
135
-# Define extension targets
168
+# Define other extension targets
169
#
170
171
#
@@ -228,21 +261,6 @@ define_gpu_extension_target(
261
262
263
231
-#
232
-# Add the `default` target which detects which extensions should be
233
-# built based on platform/architecture. This is the same logic that
234
-# setup.py uses to select which extensions should be built and should
235
-# be kept in sync.
236
-#
237
-# The `default` target makes direct use of cmake easier since knowledge
238
-# of which extensions are supported has been factored in, e.g.
239
-#
240
-# mkdir build && cd build
241
-# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
242
-# cmake --build . --target default
243
-#
244
-add_custom_target(default)
245
-
264
if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
265
message(STATUS "Enabling C extension.")
266
add_dependencies(default _C)
Dockerfile.openvino
+3
@@ -13,6 +13,9 @@ COPY requirements-common.txt /workspace/vllm/
13
COPY requirements-openvino.txt /workspace/vllm/
14
15
COPY vllm/ /workspace/vllm/vllm
16
+COPY csrc/core /workspace/vllm/csrc/core
17
+COPY cmake/utils.cmake /workspace/vllm/cmake/
18
+COPY CMakeLists.txt /workspace/vllm/
19
COPY setup.py /workspace/vllm/
20
21
# install build requirements
benchmarks/kernels/benchmark_marlin.py
+27
-23
@@ -7,16 +7,17 @@ from benchmark_shapes import WEIGHT_SHAPES
7
from vllm import _custom_ops as ops
8
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
9
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
10
- GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
10
+ GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES)
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13
- MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS)
13
+ MARLIN_SUPPORTED_GROUP_SIZES, query_marlin_supported_quant_types)
14
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15
MarlinWorkspace, marlin_quantize)
16
from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
17
marlin_24_quantize)
18
from vllm.model_executor.layers.quantization.utils.quant_utils import (
19
- gptq_pack, quantize_weights, sort_weights)
19
+ gptq_pack, gptq_quantize_weights, sort_weights)
20
+from vllm.scalar_type import ScalarType
21
from vllm.utils import FlexibleArgumentParser
22
23
DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
@@ -27,13 +28,14 @@ K_FULL_OPTS = [False, True]
28
29
30
def bench_run(results: List[benchmark.Measurement], model: str,
30
- act_order: bool, is_k_full: bool, num_bits: int, group_size: int,
31
- size_m: int, size_k: int, size_n: int):
31
+ act_order: bool, is_k_full: bool, quant_type: ScalarType,
32
+ group_size: int, size_m: int, size_k: int, size_n: int):
33
label = "Quant Matmul"
34
34
- sub_label = ("{}, act={} k_full={}, b={}, g={}, "
35
- "MKN=({}x{}x{})".format(model, act_order, is_k_full, num_bits,
36
- group_size, size_m, size_k, size_n))
35
+ sub_label = ("{}, act={} k_full={}, q={}, g={}, "
36
+ "MKN=({}x{}x{})".format(model, act_order, is_k_full,
37
+ str(quant_type), group_size, size_m,
38
+ size_k, size_n))
39
40
print(f"Testing: {sub_label}")
41
@@ -50,18 +52,18 @@ def bench_run(results: List[benchmark.Measurement], model: str,
52
marlin_g_idx,
53
marlin_sort_indices,
54
marlin_rand_perm,
53
- ) = marlin_quantize(b, num_bits, group_size, act_order)
55
+ ) = marlin_quantize(b, quant_type, group_size, act_order)
56
57
# Marlin_24 quant
58
(marlin_24_w_ref, marlin_24_q_w_comp, marlin_24_meta,
57
- marlin_24_s) = marlin_24_quantize(b, num_bits, group_size)
59
+ marlin_24_s) = marlin_24_quantize(b, quant_type, group_size)
60
61
marlin_zp = torch.empty(0, dtype=torch.int, device=b.device)
62
63
# GPTQ quant
64
(w_ref, q_w, s, g_idx,
63
- rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
64
- q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
65
+ rand_perm) = gptq_quantize_weights(b, quant_type, group_size, act_order)
66
+ q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
67
68
# For act_order, sort the "weights" and "g_idx"
69
# so that group ids are increasing
@@ -75,10 +77,11 @@ def bench_run(results: List[benchmark.Measurement], model: str,
77
78
marlin_24_workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
79
GPTQ_MARLIN_24_MAX_PARALLEL)
80
+ marlin_zp = torch.zeros_like(marlin_s, dtype=torch.int)
81
82
globals = {
83
# Gen params
81
- "num_bits": num_bits,
84
+ "quant_type": quant_type,
85
"group_size": group_size,
86
"size_m": size_m,
87
"size_n": size_n,
@@ -128,7 +131,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
131
results.append(
132
benchmark.Timer(
133
stmt=
131
- "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
134
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
135
globals=globals,
136
label=label,
137
sub_label=sub_label,
@@ -138,19 +141,19 @@ def bench_run(results: List[benchmark.Measurement], model: str,
141
results.append(
142
benchmark.Timer(
143
stmt=
141
- "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
144
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
145
globals=globals,
146
label=label,
147
sub_label=sub_label,
148
description="gptq_marlin_gemm_fp32",
149
).blocked_autorange(min_run_time=min_run_time))
150
148
- if (num_bits in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
151
+ if (quant_type in GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES
152
and group_size in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES):
153
results.append(
154
benchmark.Timer(
155
stmt=
153
- "output = gptq_marlin_24_gemm(a, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s, marlin_24_workspace.scratch, num_bits, size_m, size_n, size_k)", # noqa: E501
156
+ "output = gptq_marlin_24_gemm(a, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s, marlin_24_workspace.scratch, quant_type, size_m, size_n, size_k)", # noqa: E501
157
globals=globals,
158
label=label,
159
sub_label=sub_label,
@@ -160,7 +163,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
163
results.append(
164
benchmark.Timer(
165
stmt=
163
- "q_res = gptq_marlin_repack(q_w_gptq, repack_sort_indices, size_k, size_n, num_bits)", # noqa: E501
166
+ "q_res = gptq_marlin_repack(q_w_gptq, repack_sort_indices, size_k, size_n, quant_type.size_bits)", # noqa: E501
167
globals=globals,
168
label=label,
169
sub_label=sub_label,
@@ -196,9 +199,10 @@ def main(args):
199
) > 0 and is_k_full not in args.limit_k_full:
200
continue
201
199
- for num_bits in MARLIN_SUPPORTED_NUM_BITS:
200
- if len(args.limit_num_bits
201
- ) > 0 and num_bits not in args.limit_num_bits:
202
+ for quant_type in query_marlin_supported_quant_types(
203
+ False):
204
+ if len(args.limit_num_bits) > 0 and \
205
+ quant_type.size_bits not in args.limit_num_bits:
206
continue
207
208
for group_size in MARLIN_SUPPORTED_GROUP_SIZES:
@@ -215,8 +219,8 @@ def main(args):
219
220
for size_m in args.batch_sizes:
221
bench_run(results, model, act_order, is_k_full,
218
- num_bits, group_size, size_m, size_k,
219
- size_n)
222
+ quant_type, group_size, size_m,
223
+ size_k, size_n)
224
225
compare = benchmark.Compare(results)
226
compare.print()
cmake/cpu_extension.cmake
-1
@@ -113,6 +113,5 @@ define_gpu_extension_target(
113
WITH_SOABI
114
)
115
116
-add_custom_target(default)
116
message(STATUS "Enabling C extension.")
117
add_dependencies(default _C)
csrc/core/registration.h
renamed
csrc/core/scalar_type.hpp
new
+382
@@ -0,0 +1,382 @@
1
+#pragma once
2
+
3
+#include <torch/custom_class.h>
4
+
5
+namespace vllm {
6
+
7
+//
8
+// ScalarType can represent a wide range of floating point and integer types,
9
+// in particular it can be used to represent sub-byte data types (something
10
+// that torch.dtype currently does not support).
11
+//
12
+// ScalarTypeTorch is a subclass of ScalarType that is compatible with
13
+// TORCH_LIBRARY, making it accessible from Python as well meaning this class
14
+// can be used as a argument for custom operators, helping to simplify these
15
+// interfaces.
16
+//
17
+// The type definitions on the Python side can be found in: vllm/_core_ext.pyi
18
+// these type definitions should be kept up to date with any Python API changes
19
+// here.
20
+//
21
+class ScalarType {
22
+ public:
23
+ enum NanRepr : int64_t {
24
+ NAN_NONE = 0, // nans are not supported
25
+ NAN_IEEE_754 = 1, // nans are: exp all 1s, mantissa not all 0s
26
+ NAN_EXTD_RANGE_MAX_MIN = 2, // nans are: exp all 1s, mantissa all 1s
27
+
28
+ NAN_REPR_ID_MAX
29
+ };
30
+
31
+ constexpr ScalarType(bool signed_, int64_t exponent, int64_t mantissa,
32
+ int64_t bias, bool finite_values_only = false,
33
+ NanRepr nan_repr = NAN_IEEE_754)
34
+ : exponent(exponent),
35
+ mantissa(mantissa),
36
+ bias(bias),
37
+ signed_(signed_),
38
+ finite_values_only(finite_values_only),
39
+ nan_repr(nan_repr){};
40
+
41
+ static constexpr ScalarType int_(int64_t size_bits, int64_t bias = 0) {
42
+ return ScalarType(true, 0, size_bits - 1, bias);
43
+ }
44
+
45
+ static constexpr ScalarType uint(int64_t size_bits, int64_t bias = 0) {
46
+ return ScalarType(false, 0, size_bits, bias);
47
+ }
48
+
49
+ // IEEE 754 compliant floating point type
50
+ static constexpr ScalarType float_IEEE754(int64_t exponent,
51
+ int64_t mantissa) {
52
+ TORCH_CHECK(mantissa > 0 && exponent > 0);
53
+ return ScalarType(true, exponent, mantissa, 0, false, NAN_IEEE_754);
54
+ }
55
+
56
+ // IEEE 754 non-compliant floating point type
57
+ static constexpr ScalarType float_(int64_t exponent, int64_t mantissa,
58
+ bool finite_values_only,
59
+ NanRepr nan_repr) {
60
+ TORCH_CHECK(nan_repr < NAN_REPR_ID_MAX, "Invalid NanRepr");
61
+ TORCH_CHECK(mantissa > 0 && exponent > 0);
62
+ TORCH_CHECK(nan_repr != NAN_IEEE_754,
63
+ "use `float_IEEE754` constructor for floating point types that "
64
+ "follow IEEE 754 conventions");
65
+ return ScalarType(true, exponent, mantissa, 0, finite_values_only,
66
+ nan_repr);
67
+ }
68
+
69
+ int64_t const exponent; // size of the exponent field (0 for integer types)
70
+ int64_t const mantissa; // size of the mantissa field (size of the integer
71
+ // excluding the sign bit for integer types)
72
+ int64_t const bias; // stored values equal value + bias,
73
+ // used for quantized type
74
+ bool const signed_; // flag if the type supports negative numbers (i.e. has a
75
+ // sign bit)
76
+
77
+ // Extra Floating point info
78
+ bool const finite_values_only; // i.e. no +/-inf if true
79
+ NanRepr const nan_repr; // how NaNs are represented
80
+ // (not applicable for integer types)
81
+
82
+ int64_t size_bits() const { return mantissa + exponent + is_signed(); }
83
+ bool is_signed() const { return signed_; }
84
+ bool is_integer() const { return exponent == 0; }
85
+ bool is_floating_point() const { return exponent > 0; }
86
+ bool is_ieee_754() const {
87
+ return is_floating_point() && finite_values_only == false &&
88
+ nan_repr == NAN_IEEE_754;
89
+ }
90
+ bool has_nans() const { return is_floating_point() && nan_repr != NAN_NONE; }
91
+ bool has_infs() const {
92
+ return is_floating_point() && finite_values_only == false;
93
+ }
94
+ bool has_bias() const { return bias != 0; }
95
+
96
+ private:
97
+ double _floating_point_max() const {
98
+ TORCH_CHECK(mantissa <= 52 && exponent <= 11,
99
+ "Cannot represent max/min as a double for type ", str());
100
+
101
+ uint64_t max_mantissa = (uint64_t(1) << mantissa) - 1;
102
+ if (nan_repr == NAN_EXTD_RANGE_MAX_MIN) {
103
+ max_mantissa -= 1;
104
+ }
105
+
106
+ uint64_t max_exponent = (uint64_t(1) << exponent) - 2;
107
+ if (nan_repr == NAN_EXTD_RANGE_MAX_MIN || nan_repr == NAN_NONE) {
108
+ TORCH_CHECK(exponent < 11,
109
+ "Cannot represent max/min as a double for type ", str());
110
+ max_exponent += 1;
111
+ }
112
+
113
+ // adjust the exponent to match that of a double
114
+ // for now we assume the exponent bias is the standard 2^(e-1) -1, (where e
115
+ // is the exponent bits), there is some precedent for non-standard biases,
116
+ // example `float8_e4m3b11fnuz` here: https://github.com/jax-ml/ml_dtypes
117
+ // but to avoid premature over complication we are just assuming the
118
+ // standard exponent bias until there is a need to support non-standard
119
+ // biases
120
+ uint64_t exponent_bias = (uint64_t(1) << (exponent - 1)) - 1;
121
+ uint64_t exponent_bias_double = (uint64_t(1) << 10) - 1; // double e = 11
122
+
123
+ uint64_t max_exponent_double =
124
+ max_exponent - exponent_bias + exponent_bias_double;
125
+
126
+ // shift the mantissa into the position for a double and
127
+ // the exponent
128
+ uint64_t double_raw =
129
+ (max_mantissa << (52 - mantissa)) | (max_exponent_double << 52);
130
+
131
+ return *reinterpret_cast<double*>(&double_raw);
132
+ }
133
+
134
+ std::variant<int64_t, double> _raw_max() const {
135
+ if (is_floating_point()) {
136
+ return {_floating_point_max()};
137
+ } else {
138
+ TORCH_CHECK(size_bits() < 64 || size_bits() == 64 && is_signed(),
139
+ "Cannot represent max as a int64_t");
140
+ return {(int64_t(1) << mantissa) - 1};
141
+ }
142
+ }
143
+
144
+ std::variant<int64_t, double> _raw_min() const {
145
+ if (is_floating_point()) {
146
+ TORCH_CHECK(is_signed(),
147
+ "We currently assume all floating point types are signed");
148
+ constexpr uint64_t sign_bit_double = (uint64_t(1) << 63);
149
+
150
+ double max = _floating_point_max();
151
+ uint64_t max_raw = *reinterpret_cast<uint64_t*>(&max);
152
+ uint64_t min_raw = max_raw | sign_bit_double;
153
+ return {*reinterpret_cast<double*>(&min_raw)};
154
+ } else {
155
+ TORCH_CHECK(!is_signed() || size_bits() <= 64,
156
+ "Cannot represent min as a int64_t");
157
+ if (is_signed()) {
158
+ // set the top bit to 1 (i.e. INT64_MIN) and the rest to 0
159
+ // then perform an arithmetic shift right to set all the bits above
160
+ // (size_bits() - 1) to 1
161
+ return {INT64_MIN >> (64 - size_bits())};
162
+ } else {
163
+ return {int64_t(0)};
164
+ }
165
+ }
166
+ }
167
+
168
+ public:
169
+ // Max representable value for this scalar type.
170
+ // (accounting for bias if there is one)
171
+ std::variant<int64_t, double> max() const {
172
+ return std::visit(
173
+ [this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
174
+ _raw_max());
175
+ }
176
+
177
+ // Min representable value for this scalar type.
178
+ // (accounting for bias if there is one)
179
+ std::variant<int64_t, double> min() const {
180
+ return std::visit(
181
+ [this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
182
+ _raw_min());
183
+ }
184
+
185
+ std::string str() const {
186
+ /* naming generally follows: https://github.com/jax-ml/ml_dtypes
187
+ * for floating point types (leading f) the scheme is:
188
+ * `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
189
+ * flags:
190
+ * - no-flags: means it follows IEEE 754 conventions
191
+ * - f: means finite values only (no infinities)
192
+ * - n: means nans are supported (non-standard encoding)
193
+ * for integer types the scheme is:
194
+ * `[u]int<size_bits>[b<bias>]`
195
+ * - if bias is not present it means its zero
196
+ */
197
+ if (is_floating_point()) {
198
+ auto ret = "float" + std::to_string(size_bits()) + "_e" +
199
+ std::to_string(exponent) + "m" + std::to_string(mantissa);
200
+ if (!is_ieee_754()) {
201
+ if (finite_values_only) {
202
+ ret += "f";
203
+ }
204
+ if (nan_repr != NAN_NONE) {
205
+ ret += "n";
206
+ }
207
+ }
208
+ return ret;
209
+ } else {
210
+ auto ret = ((is_signed()) ? "int" : "uint") + std::to_string(size_bits());
211
+ if (has_bias()) {
212
+ ret += "b" + std::to_string(bias);
213
+ }
214
+ return ret;
215
+ }
216
+ }
217
+
218
+ bool operator==(ScalarType const& other) const {
219
+ return mantissa == other.mantissa && exponent == other.exponent &&
220
+ bias == other.bias && signed_ == other.signed_ &&
221
+ finite_values_only == other.finite_values_only &&
222
+ nan_repr == other.nan_repr;
223
+ }
224
+};
225
+
226
+// Create a TORCH_LIBRARY compatible version of ScalarType (i.e. inherit from
227
+// torch::CustomClassHolder), we use multiple inheritance here since we cannot
228
+// have ScalarType inherit from torch::CustomClassHolder and have a constexpr
229
+// constructor at the same time (torch::CustomClassHolder does not have a
230
+// constexpr destructor)
231
+class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
232
+ public:
233
+ ScalarTypeTorch(int64_t exponent, int64_t mantissa, int64_t bias,
234
+ bool _signed)
235
+ : ScalarType(exponent, mantissa, bias, _signed){};
236
+
237
+ ScalarTypeTorch(ScalarType type) : ScalarType(type){};
238
+
239
+ using Base = ScalarType;
240
+ using Self = ScalarTypeTorch;
241
+ using SelfPtr = c10::intrusive_ptr<Self>;
242
+
243
+ static SelfPtr int_(int64_t size_bits, c10::optional<int64_t> bias) {
244
+ return c10::make_intrusive<Self>(
245
+ ScalarType::int_(size_bits, bias.value_or(0)));
246
+ }
247
+
248
+ static SelfPtr uint(int64_t size_bits, c10::optional<int64_t> bias) {
249
+ return c10::make_intrusive<Self>(
250
+ ScalarType::uint(size_bits, bias.value_or(0)));
251
+ }
252
+
253
+ static SelfPtr float_IEEE754(int64_t exponent, int64_t mantissa) {
254
+ return c10::make_intrusive<Self>(
255
+ ScalarType::float_IEEE754(exponent, mantissa));
256
+ }
257
+
258
+ static SelfPtr float_(int64_t exponent, int64_t mantissa,
259
+ bool finite_values_only, int64_t nan_repr) {
260
+ return c10::make_intrusive<Self>(ScalarType::float_(
261
+ exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
262
+ }
263
+
264
+ template <typename T>
265
+ static void bind_readonly_property(torch::class_<Self>& cls,
266
+ std::string const& name, T Base::*field) {
267
+ auto getter_func = [field = std::move(field)](SelfPtr const& self) {
268
+ if constexpr (std::is_member_function_pointer_v<decltype(field)>) {
269
+ return (self.get()->*field)();
270
+ } else {
271
+ return self.get()->*field;
272
+ }
273
+ };
274
+
275
+ cls.def_property(name, getter_func);
276
+ }
277
+
278
+ template <typename MemberFunc, typename Cls>
279
+ static void bind_function(torch::class_<Self>& cls, const std::string& name,
280
+ MemberFunc Cls::*member) {
281
+ cls.def(name, [member = std::move(member)](SelfPtr const& self) {
282
+ return (self.get()->*member)();
283
+ });
284
+ }
285
+
286
+ template <typename Func>
287
+ static void bind_function(torch::class_<Self>& cls, const std::string& name,
288
+ Func func) {
289
+ cls.def(name, func);
290
+ }
291
+
292
+ template <typename Func>
293
+ static void bind_static_function(torch::class_<Self>& cls,
294
+ const std::string& name, Func func) {
295
+ cls.def_static(name, func);
296
+ }
297
+
298
+ static void bind_class(torch::Library& lib) {
299
+ auto cls = lib.class_<ScalarTypeTorch>("ScalarType")
300
+ .def(torch::init<int64_t, int64_t, int64_t, bool>());
301
+
302
+ // Bind Properties
303
+ bind_readonly_property(cls, "mantissa", &Base::mantissa);
304
+ bind_readonly_property(cls, "exponent", &Base::exponent);
305
+ bind_readonly_property(cls, "bias", &Base::bias);
306
+ bind_readonly_property(cls, "signed", &Base::is_signed);
307
+ bind_readonly_property(cls, "size_bits", &Base::size_bits);
308
+
309
+ // Bind member functions
310
+ bind_function(cls, "is_signed", &Base::is_signed);
311
+ bind_function(cls, "is_integer", &Base::is_integer);
312
+ bind_function(cls, "is_floating_point", &Base::is_floating_point);
313
+ bind_function(cls, "is_ieee_754", &Base::is_ieee_754);
314
+ bind_function(cls, "has_nans", &Base::has_nans);
315
+ bind_function(cls, "has_infs", &Base::has_infs);
316
+ bind_function(cls, "has_bias", &Base::has_bias);
317
+
318
+ bind_function(cls, "max", [](SelfPtr const& self) {
319
+ return std::visit([](auto arg) { return c10::IValue(arg); },
320
+ self.get()->max());
321
+ });
322
+ bind_function(cls, "min", [](SelfPtr const& self) {
323
+ return std::visit([](auto arg) { return c10::IValue(arg); },
324
+ self.get()->min());
325
+ });
326
+
327
+ bind_function(cls, "__str__", &Base::str);
328
+ bind_function(cls, "__eq__", [](SelfPtr const& self, SelfPtr const& other) {
329
+ return *self == *other;
330
+ });
331
+ bind_function(cls, "__repr__", [](SelfPtr const& self) {
332
+ return "ScalarType." + self.get()->str();
333
+ });
334
+
335
+ // Bind static functions (convenience constructors)
336
+ bind_static_function(cls, "int_", &ScalarTypeTorch::int_);
337
+ bind_static_function(cls, "uint", &ScalarTypeTorch::uint);
338
+ bind_static_function(cls, "float_IEEE754", &ScalarTypeTorch::float_IEEE754);
339
+ bind_static_function(cls, "float_", &ScalarTypeTorch::float_);
340
+ }
341
+};
342
+
343
+using ScalarTypeTorchPtr = c10::intrusive_ptr<ScalarTypeTorch>;
344
+
345
+// "rust style" names generally following:
346
+// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L60-L70
347
+static inline constexpr auto kS4 = ScalarType::int_(4);
348
+static inline constexpr auto kU4 = ScalarType::uint(4);
349
+static inline constexpr auto kU4B8 = ScalarType::uint(4, 8);
350
+static inline constexpr auto kS8 = ScalarType::int_(8);
351
+static inline constexpr auto kU8 = ScalarType::uint(8);
352
+static inline constexpr auto kU8B128 = ScalarType::uint(8, 128);
353
+
354
+static inline constexpr auto kFE3M2f =
355
+ ScalarType::float_(3, 2, true, ScalarType::NAN_NONE);
356
+static inline constexpr auto kFE4M3fn =
357
+ ScalarType::float_(4, 3, true, ScalarType::NAN_EXTD_RANGE_MAX_MIN);
358
+static inline constexpr auto kFE5M2 = ScalarType::float_IEEE754(5, 2);
359
+static inline constexpr auto kFE8M7 = ScalarType::float_IEEE754(8, 7);
360
+static inline constexpr auto kFE5M10 = ScalarType::float_IEEE754(5, 10);
361
+
362
+// Fixed width style names, generally following:
363
+// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L47-L57
364
+static inline constexpr auto kInt4 = kS4;
365
+static inline constexpr auto kUint4 = kU4;
366
+static inline constexpr auto kUint4b8 = kU4B8;
367
+static inline constexpr auto kInt8 = kS8;
368
+static inline constexpr auto kUint8 = kU8;
369
+static inline constexpr auto kUint8b128 = kU8B128;
370
+
371
+static inline constexpr auto kFloat6_e3m2f = kFE3M2f;
372
+static inline constexpr auto kFloat8_e4m3fn = kFE4M3fn;
373
+static inline constexpr auto kFloat8_e5m2 = kFE5M2;
374
+static inline constexpr auto kFloat16_e8m7 = kFE8M7;
375
+static inline constexpr auto kFloat16_e5m10 = kFE5M10;
376
+
377
+// colloquial names
378
+static inline constexpr auto kHalf = kFE5M10;
379
+static inline constexpr auto kFloat16 = kHalf;
380
+static inline constexpr auto kBFloat16 = kFE8M7;
381
+
382
+}; // namespace vllm
csrc/core/torch_bindings.cpp
new
+16
@@ -0,0 +1,16 @@
1
+#include <torch/library.h>
2
+
3
+#include "scalar_type.hpp"
4
+#include "registration.h"
5
+
6
+// Note the CORE exstension will be built for (almost) all hardware targets so
7
+// new additions must account for this. (currently not built for TPU and Neuron)
8
+
9
+TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, lib) {
10
+ // ScalarType, a custom class for representing data types that supports
11
+ // quantized types, declared here so it can be used when creating interfaces
12
+ // for custom ops.
13
+ vllm::ScalarTypeTorch::bind_class(lib);
14
+}
15
+
16
+REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cpu/torch_bindings.cpp
+1
-1
@@ -1,6 +1,6 @@
1
#include "cache.h"
2
#include "ops.h"
3
-#include "registration.h"
3
+#include "core/registration.h"
4
5
#include <torch/library.h>
6
csrc/moe/torch_bindings.cpp
+1
-1
@@ -1,4 +1,4 @@
1
-#include "registration.h"
1
+#include "core/registration.h"
2
#include "moe_ops.h"
3
4
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
csrc/ops.h
+6
-2
@@ -3,6 +3,8 @@
3
#include <optional>
4
#include <torch/library.h>
5
6
+#include "core/scalar_type.hpp"
7
+
8
void paged_attention_v1(
9
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
10
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
@@ -84,14 +86,16 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
86
torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
87
torch::Tensor& b_meta,
88
torch::Tensor& b_scales,
87
- torch::Tensor& workspace, int64_t num_bits,
89
+ torch::Tensor& workspace,
90
+ vllm::ScalarTypeTorchPtr const& b_q_type,
91
int64_t size_m, int64_t size_n,
92
int64_t size_k);
93
94
torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
95
torch::Tensor& b_scales, torch::Tensor& b_zeros,
96
torch::Tensor& g_idx, torch::Tensor& perm,
94
- torch::Tensor& workspace, int64_t num_bits,
97
+ torch::Tensor& workspace,
98
+ vllm::ScalarTypeTorchPtr const& b_q_type,
99
int64_t size_m, int64_t size_n, int64_t size_k,
100
bool is_k_full, bool has_zp,
101
bool use_fp32_reduce);
csrc/quantization/gptq_marlin/gptq_marlin.cu
+44
-22
@@ -21,6 +21,7 @@
21
22
#include "marlin.cuh"
23
#include "marlin_dtypes.cuh"
24
+#include "core/scalar_type.hpp"
25
26
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
27
static_assert(std::is_same<scalar_t, half>::value || \
@@ -71,14 +72,15 @@ __global__ void Marlin(
72
bool use_fp32_reduce // whether to use fp32 global reduce
73
) {}
74
74
-} // namespace gptq_marlin
75
+} // namespace marlin
76
77
torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
78
torch::Tensor& b_scales, torch::Tensor& b_zeros,
79
torch::Tensor& g_idx, torch::Tensor& perm,
79
- torch::Tensor& workspace, int64_t num_bits,
80
+ torch::Tensor& workspace,
81
+ vllm::ScalarTypeTorchPtr const& b_q_type,
82
int64_t size_m, int64_t size_n, int64_t size_k,
81
- bool is_k_full) {
83
+ bool is_k_full, bool has_zp) {
84
TORCH_CHECK_NOT_IMPLEMENTED(false,
85
"marlin_gemm(..) requires CUDA_ARCH >= 8.0");
86
return torch::empty({1, 1});
@@ -1963,18 +1965,29 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1965
__CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1966
1967
template <typename scalar_t>
1966
-void marlin_mm_f16i4(const void* A, const void* B, void* C, void* C_tmp,
1967
- void* s, void* zp, void* g_idx, void* perm, void* a_tmp,
1968
- int prob_m, int prob_n, int prob_k, void* workspace,
1969
- int num_bits, bool has_act_order, bool is_k_full,
1970
- bool has_zp, int num_groups, int group_size, int dev,
1971
- cudaStream_t stream, int thread_k, int thread_n, int sms,
1972
- int max_par, bool use_fp32_reduce) {
1973
- TORCH_CHECK(num_bits == 4 || num_bits == 8,
1974
- "num_bits must be 4 or 8. Got = ", num_bits);
1968
+void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
1969
+ void* zp, void* g_idx, void* perm, void* a_tmp, int prob_m,
1970
+ int prob_n, int prob_k, void* workspace,
1971
+ vllm::ScalarType const& q_type, bool has_act_order,
1972
+ bool is_k_full, bool has_zp, int num_groups, int group_size,
1973
+ int dev, cudaStream_t stream, int thread_k, int thread_n,
1974
+ int sms, int max_par, bool use_fp32_reduce) {
1975
+ if (has_zp) {
1976
+ TORCH_CHECK(
1977
+ q_type == vllm::kU4 || q_type == vllm::kU8,
1978
+ "q_type must be u4 or u8 when has_zp = True. Got = ", q_type.str());
1979
+ } else {
1980
+ TORCH_CHECK(
1981
+ q_type == vllm::kU4B8 || q_type == vllm::kU8B128,
1982
+ "q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ",
1983
+ q_type.str());
1984
+ }
1985
+
1986
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1987
", ", prob_n, ", ", prob_k, "]");
1988
1989
+ // TODO: remove alias when we start supporting other 8bit types
1990
+ int num_bits = q_type.size_bits();
1991
int tot_m = prob_m;
1992
int tot_m_blocks = div_ceil(tot_m, 16);
1993
int pad = 16 * tot_m_blocks - tot_m;
@@ -2126,19 +2139,28 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* C_tmp,
2139
}
2140
}
2141
2129
-} // namespace gptq_marlin
2142
+} // namespace marlin
2143
2144
torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2145
torch::Tensor& b_scales, torch::Tensor& b_zeros,
2146
torch::Tensor& g_idx, torch::Tensor& perm,
2134
- torch::Tensor& workspace, int64_t num_bits,
2147
+ torch::Tensor& workspace,
2148
+ vllm::ScalarTypeTorchPtr const& b_q_type,
2149
int64_t size_m, int64_t size_n, int64_t size_k,
2150
bool is_k_full, bool has_zp,
2151
bool use_fp32_reduce) {
2138
- // Verify num_bits
2139
- TORCH_CHECK(num_bits == 4 || num_bits == 8,
2140
- "num_bits must be 4 or 8. Got = ", num_bits);
2141
- int pack_factor = 32 / num_bits;
2152
+ if (has_zp) {
2153
+ TORCH_CHECK(*b_q_type == vllm::kU4 || *b_q_type == vllm::kU8,
2154
+ "b_q_type must be u4 or u8 when has_zp = True. Got = ",
2155
+ b_q_type->str());
2156
+ } else {
2157
+ TORCH_CHECK(
2158
+ *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
2159
+ "b_q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ",
2160
+ b_q_type->str());
2161
+ }
2162
+
2163
+ int pack_factor = 32 / b_q_type->size_bits();
2164
2165
// Verify A
2166
TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
@@ -2265,21 +2287,21 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2287
2288
int dev = a.get_device();
2289
if (a.scalar_type() == at::ScalarType::Half) {
2268
- marlin::marlin_mm_f16i4<half>(
2290
+ marlin::marlin_mm<half>(
2291
a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
2292
c_tmp.data_ptr<float>(), b_scales.data_ptr<at::Half>(),
2293
b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2294
a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2273
- workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2295
+ workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2296
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2297
thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2298
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
2277
- marlin::marlin_mm_f16i4<nv_bfloat16>(
2299
+ marlin::marlin_mm<nv_bfloat16>(
2300
a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
2301
c.data_ptr<at::BFloat16>(), c_tmp.data_ptr<float>(),
2302
b_scales.data_ptr<at::BFloat16>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2303
perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2282
- workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2304
+ workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2305
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2306
thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2307
} else {
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+10
-7
@@ -27,6 +27,7 @@
27
#include <iostream>
28
29
#include "common/base.h"
30
+#include "core/scalar_type.hpp"
31
32
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
33
@@ -86,7 +87,8 @@ __global__ void Marlin_24(
87
torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
88
torch::Tensor& b_meta,
89
torch::Tensor& b_scales,
89
- torch::Tensor& workspace, int64_t num_bits,
90
+ torch::Tensor& workspace,
91
+ vllm::ScalarTypeTorchPtr const& b_q_type,
92
int64_t size_m, int64_t size_n,
93
int64_t size_k) {
94
TORCH_CHECK_NOT_IMPLEMENTED(
@@ -1025,13 +1027,14 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
1027
torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1028
torch::Tensor& b_meta,
1029
torch::Tensor& b_scales,
1028
- torch::Tensor& workspace, int64_t num_bits,
1030
+ torch::Tensor& workspace,
1031
+ vllm::ScalarTypeTorchPtr const& b_q_type,
1032
int64_t size_m, int64_t size_n,
1033
int64_t size_k) {
1034
// Verify num_bits
1032
- TORCH_CHECK(num_bits == 4 || num_bits == 8,
1033
- "num_bits must be 4 or 8. Got = ", num_bits);
1034
- int pack_factor = 32 / num_bits;
1035
+ TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
1036
+ "num_bits must be uint4b8 or uint8b128. Got = ", b_q_type->str());
1037
+ int pack_factor = 32 / b_q_type->size_bits();
1038
1039
// Verify M
1040
TORCH_CHECK(size_m == a.size(0),
@@ -1126,8 +1129,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1129
marlin_24::marlin_cuda_2_4(
1130
a.data_ptr(), b_q_weight.data_ptr(), b_meta.data_ptr(), c.data_ptr(),
1131
b_scales.data_ptr(), size_n, size_m, size_k, workspace.data_ptr(),
1129
- num_bits, groupsize, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1130
- thread_m, sms, max_par);
1132
+ b_q_type->size_bits(), groupsize, dev,
1133
+ at::cuda::getCurrentCUDAStream(dev), thread_k, thread_m, sms, max_par);
1134
1135
return c;
1136
}
csrc/torch_bindings.cpp
+1
-1
@@ -1,7 +1,7 @@
1
#include "cache.h"
2
#include "cuda_utils.h"
3
#include "ops.h"
4
-#include "registration.h"
4
+#include "core/registration.h"
5
6
#include <torch/library.h>
7
setup.py
+8
-1
@@ -271,6 +271,10 @@ def _build_custom_ops() -> bool:
271
return _is_cuda() or _is_hip() or _is_cpu()
272
273
274
+def _build_core_ext() -> bool:
275
+ return not _is_neuron() and not _is_tpu()
276
+
277
+
278
def get_hipcc_rocm_version():
279
# Run the hipcc --version command
280
result = subprocess.run(['hipcc', '--version'],
@@ -433,6 +437,9 @@ def get_requirements() -> List[str]:
437
438
ext_modules = []
439
440
+if _build_core_ext():
441
+ ext_modules.append(CMakeExtension(name="vllm._core_C"))
442
+
443
if _is_cuda() or _is_hip():
444
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
445
@@ -477,7 +484,7 @@ setup(
484
extras_require={
485
"tensorizer": ["tensorizer>=2.9.0"],
486
},
480
- cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
487
+ cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
488
package_data=package_data,
489
entry_points={
490
"console_scripts": [
tests/kernels/test_int8_quant.py
-2
@@ -1,8 +1,6 @@
1
import pytest
2
import torch
3
4
-# ruff: noqa: F401
5
-import vllm._C
4
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
5
from vllm._custom_ops import scaled_int8_quant
6
tests/kernels/test_marlin_gemm.py
+41
-34
@@ -9,14 +9,14 @@ from tests.quantization.utils import is_quant_method_supported
9
from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
12
- GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
12
+ GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES)
13
from vllm.model_executor.layers.quantization.qqq import (
14
MARLIN_QQQ_MAX_PARALLEL, MARLIN_QQQ_MIN_THREAD_N,
15
MARLIN_QQQ_SUPPORTED_GROUP_SIZES, MARLIN_QQQ_SUPPORTED_NUM_BITS)
16
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
17
GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
18
- MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS,
19
- marlin_make_empty_g_idx, marlin_permute_scales)
18
+ MARLIN_SUPPORTED_GROUP_SIZES, marlin_make_empty_g_idx,
19
+ marlin_permute_scales, query_marlin_supported_quant_types)
20
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
21
pack_fp8_to_int32)
22
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
@@ -27,8 +27,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
27
from vllm.model_executor.layers.quantization.utils.marlin_utils_test_qqq import ( # noqa: E501
28
marlin_qqq_quantize)
29
from vllm.model_executor.layers.quantization.utils.quant_utils import (
30
- awq_pack, gptq_pack, quantize_weights, quantize_weights_with_zp,
31
- sort_weights)
30
+ awq_pack, gptq_pack, gptq_quantize_weights, quantize_weights, sort_weights)
31
32
ACT_ORDER_OPTS = [False, True]
33
K_FULL_OPTS = [False, True]
@@ -65,12 +64,13 @@ def rand_data(shape, dtype=torch.float16):
64
reason="Marlin is not supported on this GPU type.")
65
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
66
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
68
-@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
67
+@pytest.mark.parametrize("quant_type",
68
+ query_marlin_supported_quant_types(False))
69
@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
70
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
71
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
72
-def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
73
- mnk_factors):
72
+def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
73
+ act_order, mnk_factors):
74
m_factor, n_factor, k_factor = mnk_factors
75
76
size_m = m_factor
@@ -95,11 +95,11 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
95
b_weight = rand_data((size_k, size_n))
96
97
# Quantize (and apply act_order if provided)
98
- w_ref, q_w, s, g_idx, rand_perm = quantize_weights(b_weight, num_bits,
99
- group_size, act_order)
98
+ w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
99
+ b_weight, quant_type, group_size, act_order)
100
101
# Pack to GPTQ format
102
- q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
102
+ q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
103
104
# For act_order, sort the "weights" and "g_idx" so that group ids are
105
# increasing
@@ -108,8 +108,9 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
108
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
109
110
# Pack to Marlin format
111
- weight_perm = get_weight_perm(num_bits)
112
- marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
111
+ weight_perm = get_weight_perm(quant_type.size_bits)
112
+ marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
113
+ weight_perm)
114
115
# Run Marlin repack GPU kernel
116
marlin_q_w_2 = ops.gptq_marlin_repack(
@@ -117,7 +118,7 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
118
sort_indices,
119
size_k,
120
size_n,
120
- num_bits,
121
+ quant_type.size_bits,
122
)
123
torch.cuda.synchronize()
124
@@ -128,10 +129,11 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
129
reason="Marlin is not supported on this GPU type.")
130
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
131
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
131
-@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
132
+@pytest.mark.parametrize("quant_type",
133
+ query_marlin_supported_quant_types(False))
134
@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
135
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
134
-def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
136
+def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
137
mnk_factors):
138
m_factor, n_factor, k_factor = mnk_factors
139
@@ -150,22 +152,25 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
152
b_weight = rand_data((size_k, size_n))
153
154
# Quantize
153
- w_ref, q_w, s, zp = quantize_weights_with_zp(b_weight, num_bits,
154
- group_size)
155
+ w_ref, q_w, s, zp = quantize_weights(b_weight,
156
+ quant_type,
157
+ group_size,
158
+ zero_points=True)
159
160
# Pack to AWQ format
157
- q_w_awq = awq_pack(q_w, num_bits, size_k, size_n)
161
+ q_w_awq = awq_pack(q_w, quant_type.size_bits, size_k, size_n)
162
163
# Pack to Marlin format
160
- weight_perm = get_weight_perm(num_bits)
161
- marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
164
+ weight_perm = get_weight_perm(quant_type.size_bits)
165
+ marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
166
+ weight_perm)
167
168
# Run Marlin repack GPU kernel
169
marlin_q_w_2 = ops.awq_marlin_repack(
170
q_w_awq,
171
size_k,
172
size_n,
168
- num_bits,
173
+ quant_type.size_bits,
174
)
175
torch.cuda.synchronize()
176
@@ -176,7 +181,8 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
181
reason="Marlin is not supported on this GPU type.")
182
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
183
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
179
-@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
184
+@pytest.mark.parametrize("quant_type",
185
+ query_marlin_supported_quant_types(False))
186
@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
187
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
188
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
@@ -185,7 +191,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
191
def test_gptq_marlin_gemm(
192
k_chunk,
193
n_chunk,
188
- num_bits,
194
+ quant_type,
195
group_size,
196
mnk_factors,
197
act_order,
@@ -211,7 +217,7 @@ def test_gptq_marlin_gemm(
217
b_weight = rand_data((size_k, size_n))
218
219
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
214
- b_weight, num_bits, group_size, act_order)
220
+ b_weight, quant_type, group_size, act_order)
221
222
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
223
@@ -226,7 +232,7 @@ def test_gptq_marlin_gemm(
232
g_idx,
233
sort_indices,
234
workspace.scratch,
229
- num_bits,
235
+ quant_type,
236
a_input.shape[0],
237
b_weight.shape[1],
238
a_input.shape[1],
@@ -248,10 +254,10 @@ def test_gptq_marlin_gemm(
254
reason="Marlin is not supported on this GPU type.")
255
@pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
256
@pytest.mark.parametrize("n_chunk", MARLIN_24_N_CHUNKS)
251
-@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
257
+@pytest.mark.parametrize("quant_type", GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES)
258
@pytest.mark.parametrize("group_size", GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES)
259
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
254
-def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
260
+def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
261
mnk_factors):
262
m_factor, n_factor, k_factor = mnk_factors
263
@@ -266,7 +272,7 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
272
b_weight = rand_data((size_k, size_n))
273
274
(w_24_ref, marlin_24_q_w_comp, marlin_24_meta,
269
- marlin_24_s) = marlin_24_quantize(b_weight, num_bits, group_size)
275
+ marlin_24_s) = marlin_24_quantize(b_weight, quant_type, group_size)
276
277
workspace_24 = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
278
GPTQ_MARLIN_24_MAX_PARALLEL)
@@ -279,7 +285,7 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
285
marlin_24_meta,
286
marlin_24_s,
287
workspace_24.scratch,
282
- num_bits,
288
+ quant_type,
289
a_input.shape[0],
290
b_weight.shape[1],
291
a_input.shape[1],
@@ -371,14 +377,15 @@ def test_fp8_marlin_gemm(
377
reason="Marlin is not supported on this GPU type.")
378
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
379
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
374
-@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
380
+@pytest.mark.parametrize("quant_type",
381
+ query_marlin_supported_quant_types(True))
382
@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
383
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
384
@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
385
def test_awq_marlin_gemm(
386
k_chunk,
387
n_chunk,
381
- num_bits,
388
+ quant_type,
389
group_size,
390
mnk_factors,
391
use_fp32_reduce,
@@ -396,7 +403,7 @@ def test_awq_marlin_gemm(
403
b_weight = rand_data((size_k, size_n))
404
405
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
399
- b_weight, num_bits, group_size)
406
+ b_weight, quant_type, group_size)
407
408
g_idx = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
409
sort_indices = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
@@ -414,7 +421,7 @@ def test_awq_marlin_gemm(
421
g_idx,
422
sort_indices,
423
workspace.scratch,
417
- num_bits,
424
+ quant_type,
425
a_input.shape[0],
426
b_weight.shape[1],
427
a_input.shape[1],
tests/test_scalartype.py
new
+36
@@ -0,0 +1,36 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm.scalar_type import scalar_types
5
+
6
+
7
+@pytest.mark.parametrize("type_tuple", (
8
+ (-8, 7, scalar_types.int4),
9
+ (0, 15, scalar_types.uint4),
10
+ (-8, 7, scalar_types.uint4b8),
11
+ (-128, 127, scalar_types.uint8b128),
12
+ (-28., 28., scalar_types.float6_e3m2f),
13
+ (torch.int8, scalar_types.int8),
14
+ (torch.uint8, scalar_types.uint8),
15
+ (torch.float8_e5m2, scalar_types.float8_e5m2),
16
+ (torch.float8_e4m3fn, scalar_types.float8_e4m3fn),
17
+ (torch.bfloat16, scalar_types.float16_e8m7),
18
+ (torch.float16, scalar_types.float16_e5m10),
19
+),
20
+ ids=lambda x: str(x))
21
+def test_scalar_type_min_max(type_tuple):
22
+ print(type_tuple)
23
+ if len(type_tuple) == 3:
24
+ min, max, t = type_tuple
25
+ else:
26
+ torch_type, t = type_tuple
27
+ if torch_type.is_floating_point:
28
+ min = torch.finfo(torch_type).min
29
+ max = torch.finfo(torch_type).max
30
+ else:
31
+ min = torch.iinfo(torch_type).min
32
+ max = torch.iinfo(torch_type).max
33
+
34
+ print(t, min, max, t.min(), t.max())
35
+ assert min == t.min()
36
+ assert max == t.max()
vllm/_core_ext.py
new
+177
@@ -0,0 +1,177 @@
1
+import importlib.util
2
+from enum import Enum
3
+from typing import TYPE_CHECKING, Optional, Union
4
+
5
+import torch
6
+
7
+from vllm.logger import init_logger
8
+
9
+logger = init_logger(__name__)
10
+core_C_available = importlib.util.find_spec('._core_C', 'vllm') is not None
11
+
12
+
13
+# Mirrors enum in `core/scalar_type.hpp`
14
+class NanRepr(Enum):
15
+ NONE = 0 # nans are not supported
16
+ IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s
17
+ EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s
18
+
19
+
20
+if TYPE_CHECKING or not core_C_available:
21
+ # On platforms were we cannot use/build the C++ core extension (i.e. namely
22
+ # neuron and tpu), we define the mock ScalarType class here that partially
23
+ # mimics the C++ ScalarType class.
24
+ #
25
+ # We also use this provide type signatures to the Python LSP for the methods
26
+ # in the C++ ScalarType class. So these type signatures should be kept
27
+ # in sync with csrc/core/scalar_type.hpp
28
+
29
+ from dataclasses import dataclass
30
+
31
+ @dataclass(frozen=True)
32
+ class ScalarType:
33
+ """
34
+ ScalarType can represent a wide range of floating point and integer
35
+ types, in particular it can be used to represent sub-byte data types
36
+ (something that torch.dtype currently does not support). It is also
37
+ capable of representing types with a bias, i.e.:
38
+ `stored_value = value + bias`,
39
+ this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40
+ of 8). The implementation for this class can be found in
41
+ csrc/core/scalar_type.hpp, these type signatures should be kept in sync
42
+ with that file.
43
+ """
44
+
45
+ exponent: int
46
+ """
47
+ Number of bits in the exponent if this is a floating point type
48
+ (zero if this an integer type)
49
+ """
50
+
51
+ mantissa: int
52
+ """
53
+ Number of bits in the mantissa if this is a floating point type,
54
+ or the number bits representing an integer excluding the sign bit if
55
+ this an integer type.
56
+ """
57
+
58
+ bias: int
59
+ """
60
+ bias used to encode the values in this scalar type
61
+ (value = stored_value - bias, default 0) for example if we store the
62
+ type as an unsigned integer with a bias of 128 then the value 0 will be
63
+ stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
64
+ """
65
+
66
+ signed: bool
67
+ "If the type is signed (i.e. has a sign bit)"
68
+
69
+ _finite_values_only: bool = False
70
+ """
71
+ Private: if NANs are supported, used `has_infs()` instead.
72
+ """
73
+
74
+ nan_repr: int = NanRepr.IEEE_754.value
75
+ """
76
+ How NaNs are represent in this scalar type, returns NanRepr value.
77
+ (not applicable for integer types)
78
+ """
79
+
80
+ @property
81
+ def size_bits(self):
82
+ return self.exponent + self.mantissa + int(self.signed)
83
+
84
+ def min(self) -> Union[int, float]:
85
+ """
86
+ Min representable value for this scalar type.
87
+ (accounting for bias if there is one)
88
+ """
89
+ raise NotImplementedError
90
+
91
+ def max(self) -> Union[int, float]:
92
+ """
93
+ Max representable value for this scalar type.
94
+ (accounting for bias if there is one)
95
+ """
96
+ raise NotImplementedError
97
+
98
+ def is_signed(self) -> bool:
99
+ """
100
+ If the type is signed (i.e. has a sign bit), same as `signed`
101
+ added for consistency with:
102
+ https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
103
+ """
104
+ ...
105
+
106
+ def is_floating_point(self):
107
+ "If the type is a floating point type"
108
+ return self.exponent != 0
109
+
110
+ def is_integer(self):
111
+ "If the type is an integer type"
112
+ return self.exponent == 0
113
+
114
+ def has_bias(self):
115
+ "If the type has a non-zero bias"
116
+ return self.bias != 0
117
+
118
+ def has_infs(self):
119
+ "If the type is floating point and supports infinity"
120
+ return not self._finite_values_only
121
+
122
+ def has_nans(self):
123
+ return self.nan_repr != NanRepr.NONE.value
124
+
125
+ def is_ieee_754(self) -> bool:
126
+ """
127
+ If the type is a floating point type that follows IEEE 754
128
+ conventions
129
+ """
130
+ return self.nan_repr == NanRepr.IEEE_754.value and \
131
+ not self._finite_values_only
132
+
133
+ def __str__(self) -> str:
134
+ raise NotImplementedError
135
+
136
+ def __repr__(self) -> str:
137
+ raise NotImplementedError
138
+
139
+ #
140
+ # Convenience Constructors
141
+ #
142
+
143
+ @classmethod
144
+ def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
145
+ "Create a signed integer scalar type (size_bits includes sign-bit)."
146
+ return cls(size_bits - 1, size_bits, bias if bias else 0, True)
147
+
148
+ @classmethod
149
+ def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
150
+ """Create a unsigned integer scalar type."""
151
+ return cls(size_bits, size_bits, bias if bias else 0, False)
152
+
153
+ @classmethod
154
+ def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
155
+ """
156
+ Create a standard floating point type
157
+ (i.e. follows IEEE 754 conventions).
158
+ """
159
+ return cls(exponent, mantissa, 0, True)
160
+
161
+ @classmethod
162
+ def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
163
+ nan_repr: int):
164
+ """
165
+ Create a non-standard floating point type
166
+ (i.e. does not follow IEEE 754 conventions).
167
+ """
168
+ return cls(exponent, mantissa, 0, True, finite_values_only,
169
+ nan_repr)
170
+
171
+elif core_C_available:
172
+ try:
173
+ import vllm._core_C # noqa: F401
174
+ except ImportError as e:
175
+ logger.warning("Failed to import from vllm._core_C with %r", e)
176
+
177
+ ScalarType = torch.classes._core_C.ScalarType
vllm/_custom_ops.py
+19
-10
@@ -4,6 +4,7 @@ from typing import List, Optional, Tuple, Union
4
5
import torch
6
7
+from vllm._core_ext import ScalarType
8
from vllm.logger import init_logger
9
10
logger = init_logger(__name__)
@@ -220,10 +221,10 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
221
# marlin_24
222
def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
223
b_meta: torch.Tensor, b_scales: torch.Tensor,
223
- workspace: torch.Tensor, num_bits: int, size_m: int,
224
- size_n: int, size_k: int) -> torch.Tensor:
224
+ workspace: torch.Tensor, b_q_type: ScalarType,
225
+ size_m: int, size_n: int, size_k: int) -> torch.Tensor:
226
return torch.ops._C.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
226
- workspace, num_bits, size_m,
227
+ workspace, b_q_type, size_m,
228
size_n, size_k)
229
230
@@ -279,14 +280,22 @@ def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
280
return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
281
282
282
-def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
283
- b_scales: torch.Tensor, b_zeros: torch.Tensor,
284
- g_idx: torch.Tensor, perm: torch.Tensor,
285
- workspace: torch.Tensor, num_bits: int, size_m: int,
286
- size_n: int, size_k: int, is_k_full: bool, has_zp: bool,
287
- use_fp32_reduce: bool) -> torch.Tensor:
283
+def gptq_marlin_gemm(a: torch.Tensor,
284
+ b_q_weight: torch.Tensor,
285
+ b_scales: torch.Tensor,
286
+ b_zeros: torch.Tensor,
287
+ g_idx: torch.Tensor,
288
+ perm: torch.Tensor,
289
+ workspace: torch.Tensor,
290
+ b_q_type: ScalarType,
291
+ size_m: int,
292
+ size_n: int,
293
+ size_k: int,
294
+ is_k_full: bool,
295
+ has_zp: bool = False,
296
+ use_fp32_reduce: bool = False) -> torch.Tensor:
297
return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
289
- g_idx, perm, workspace, num_bits,
298
+ g_idx, perm, workspace, b_q_type,
299
size_m, size_n, size_k, is_k_full,
300
has_zp, use_fp32_reduce)
301
vllm/model_executor/layers/quantization/awq_marlin.py
+31
-18
@@ -10,11 +10,11 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig)
12
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13
- apply_awq_marlin_linear, awq_to_marlin_zero_points,
14
- check_awq_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
15
- marlin_permute_scales, replace_tensor, verify_awq_marlin_supported,
16
- verify_marlin_supports_shape)
13
+ apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
14
+ marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15
+ replace_tensor, verify_marlin_supported, verify_marlin_supports_shape)
16
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17
+from vllm.scalar_type import scalar_types
18
19
logger = init_logger(__name__)
20
@@ -22,20 +22,31 @@ logger = init_logger(__name__)
22
class AWQMarlinConfig(QuantizationConfig):
23
"""Config class for AWQ Marlin"""
24
25
+ # num_bits -> type
26
+ TYPE_MAP = {
27
+ 4: scalar_types.uint4,
28
+ 8: scalar_types.uint8,
29
+ }
30
+
31
def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
32
lm_head_quantized: bool) -> None:
27
- self.weight_bits = weight_bits
28
- self.pack_factor = 32 // self.weight_bits # packed into 32bits
33
+ self.pack_factor = 32 // weight_bits # packed into int32
34
self.group_size = group_size
35
self.has_zp = has_zp
36
self.lm_head_quantized = lm_head_quantized
37
33
- verify_awq_marlin_supported(num_bits=self.weight_bits,
34
- group_size=self.group_size,
35
- has_zp=self.has_zp)
38
+ if weight_bits not in self.TYPE_MAP:
39
+ raise ValueError(f"Unsupported num_bits = {weight_bits}. "
40
+ f"Supported num_bits = {self.TYPE_MAP.keys()}")
41
+
42
+ self.quant_type = self.TYPE_MAP[weight_bits]
43
+
44
+ verify_marlin_supported(self.quant_type,
45
+ group_size=self.group_size,
46
+ has_zp=self.has_zp)
47
48
def __repr__(self) -> str:
38
- return (f"AWQMarlinConfig(weight_bits={self.weight_bits}, "
49
+ return (f"AWQMarlinConfig(quant_type={self.quant_type}, "
50
f"group_size={self.group_size}, "
51
f"has_zp={self.has_zp}, "
52
f"lm_head_quantized={self.lm_head_quantized})")
@@ -110,11 +121,13 @@ class AWQMarlinConfig(QuantizationConfig):
121
if (num_bits is None or group_size is None or has_zp is None):
122
return False
123
113
- return check_awq_marlin_supported(
114
- num_bits=num_bits,
115
- group_size=group_size,
116
- has_zp=has_zp,
117
- min_capability=cls.get_min_capability())
124
+ if num_bits not in cls.TYPE_MAP:
125
+ return False
126
+
127
+ return check_marlin_supported(quant_type=cls.TYPE_MAP[num_bits],
128
+ group_size=group_size,
129
+ has_zp=has_zp,
130
+ min_capability=cls.get_min_capability())
131
132
133
class AWQMarlinLinearMethod(LinearMethodBase):
@@ -226,7 +239,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
239
layer.qweight,
240
size_k=layer.input_size_per_partition,
241
size_n=layer.output_size_per_partition,
229
- num_bits=self.quant_config.weight_bits)
242
+ num_bits=self.quant_config.quant_type.size_bits)
243
replace_tensor(layer, "qweight", marlin_qweight)
244
245
# Permute scales from AWQ format to marlin format.
@@ -242,7 +255,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
255
layer.qzeros,
256
size_k=layer.num_groups,
257
size_n=layer.output_size_per_partition,
245
- num_bits=self.quant_config.weight_bits)
258
+ num_bits=self.quant_config.quant_type.size_bits)
259
replace_tensor(layer, "qzeros", marlin_zp)
260
261
# Not-used
@@ -263,7 +276,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
276
g_idx=layer.g_idx,
277
g_idx_sort_indices=layer.g_idx_sort_indices,
278
workspace=layer.workspace,
266
- num_bits=self.quant_config.weight_bits,
279
+ quant_type=self.quant_config.quant_type,
280
output_size_per_partition=layer.output_size_per_partition,
281
input_size_per_partition=layer.input_size_per_partition,
282
bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+14
-4
@@ -9,9 +9,13 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N)
11
from vllm.model_executor.utils import set_weight_attrs
12
+from vllm.scalar_type import scalar_types
13
14
__all__ = ["CompressedTensorsW4A16Sparse24"]
14
-W4A16SPARSE24_SUPPORTED_BITS = [4]
15
+W4A16SPARSE24_SUPPORTED_TYPES_MAP = {
16
+ 4: scalar_types.uint4b8,
17
+}
18
+W4A16SPARSE24_SUPPORTED_BITS = list(W4A16SPARSE24_SUPPORTED_TYPES_MAP.keys())
19
20
21
class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
@@ -22,9 +26,15 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
26
group_size: Optional[int] = None):
27
self.strategy = strategy
28
self.group_size = group_size
25
- self.num_bits = num_bits
29
self.tile_size = 16
30
31
+ if num_bits not in W4A16SPARSE24_SUPPORTED_TYPES_MAP:
32
+ raise ValueError(
33
+ f"Unsupported num_bits = {num_bits}. "
34
+ f"Supported num_bits = {W4A16SPARSE24_SUPPORTED_BITS}")
35
+
36
+ self.quant_type = W4A16SPARSE24_SUPPORTED_TYPES_MAP[num_bits]
37
+
38
if self.strategy == "group" and self.group_size is None:
39
raise ValueError(
40
"group_size must be given when using strategy group")
@@ -43,7 +53,7 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
53
params_dtype: torch.dtype, weight_loader: Callable,
54
**kwargs):
55
46
- pack_factor = 32 // self.num_bits
56
+ pack_factor = 32 // self.quant_type.size_bits
57
output_size_per_partition = sum(output_partition_sizes)
58
59
qweight = Parameter(
@@ -138,7 +148,7 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
148
size_n = scales.shape[1]
149
150
output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
141
- workspace, self.num_bits, size_m,
151
+ workspace, self.quant_type, size_m,
152
size_n, size_k)
153
154
output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+20
-9
@@ -8,12 +8,17 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
10
apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11
- marlin_permute_scales, replace_tensor, verify_gptq_marlin_supported,
11
+ marlin_permute_scales, replace_tensor, verify_marlin_supported,
12
verify_marlin_supports_shape)
13
from vllm.model_executor.utils import set_weight_attrs
14
+from vllm.scalar_type import scalar_types
15
16
__all__ = ["CompressedTensorsWNA16"]
16
-WNA16_SUPPORTED_BITS = [4, 8]
17
+WNA16_SUPPORTED_TYPES_MAP = {
18
+ 4: scalar_types.uint4b8,
19
+ 8: scalar_types.uint8b128,
20
+}
21
+WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
22
23
24
class CompressedTensorsWNA16(CompressedTensorsScheme):
@@ -22,8 +27,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
27
strategy: str,
28
num_bits: int,
29
group_size: Optional[int] = None):
25
- self.num_bits = num_bits
26
- self.pack_factor = 32 // self.num_bits
30
+
31
+ self.pack_factor = 32 // num_bits
32
self.strategy = strategy
33
34
self.group_size: int
@@ -37,10 +42,16 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
42
else:
43
self.group_size = group_size
44
45
+ if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
46
+ raise ValueError(
47
+ f"Unsupported num_bits = {num_bits}. "
48
+ f"Supported num_bits = {WNA16_SUPPORTED_TYPES_MAP.keys()}")
49
+
50
+ self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
51
+
52
# Verify supported on platform.
41
- verify_gptq_marlin_supported(num_bits=self.num_bits,
42
- group_size=self.group_size,
43
- is_sym=True)
53
+ verify_marlin_supported(quant_type=self.quant_type,
54
+ group_size=self.group_size)
55
56
@classmethod
57
def get_min_capability(cls) -> int:
@@ -150,7 +161,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
161
perm=layer.g_idx_sort_indices,
162
size_k=layer.input_size_per_partition,
163
size_n=layer.output_size_per_partition,
153
- num_bits=self.num_bits)
164
+ num_bits=self.quant_type.size_bits)
165
replace_tensor(layer, "weight_packed", marlin_qweight)
166
167
# Permute scales from compressed-tensors format to marlin format.
@@ -172,7 +183,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
183
g_idx=layer.g_idx,
184
g_idx_sort_indices=layer.g_idx_sort_indices,
185
workspace=layer.workspace,
175
- num_bits=self.num_bits,
186
+ wtype=self.quant_type,
187
output_size_per_partition=layer.output_size_per_partition,
188
input_size_per_partition=layer.input_size_per_partition,
189
is_k_full=True,
vllm/model_executor/layers/quantization/gptq_marlin.py
+27
-16
@@ -10,11 +10,12 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig)
12
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13
- apply_gptq_marlin_linear, check_gptq_marlin_supported, marlin_is_k_full,
13
+ apply_gptq_marlin_linear, check_marlin_supported, marlin_is_k_full,
14
marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15
marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
16
- verify_gptq_marlin_supported, verify_marlin_supports_shape)
16
+ verify_marlin_supported, verify_marlin_supports_shape)
17
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
+from vllm.scalar_type import scalar_types
19
20
logger = init_logger(__name__)
21
@@ -22,6 +23,12 @@ logger = init_logger(__name__)
23
class GPTQMarlinConfig(QuantizationConfig):
24
"""Config class for GPTQ Marlin"""
25
26
+ # (num_bits, is_sym) -> quant_type
27
+ TYPE_MAP = {
28
+ (4, True): scalar_types.uint4b8,
29
+ (8, True): scalar_types.uint8b128,
30
+ }
31
+
32
def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
33
is_sym: bool, lm_head_quantized: bool) -> None:
34
if desc_act and group_size == -1:
@@ -29,20 +36,23 @@ class GPTQMarlinConfig(QuantizationConfig):
36
# (since we have only one group per output channel)
37
desc_act = False
38
32
- self.weight_bits = weight_bits
33
- self.pack_factor = 32 // self.weight_bits # packed into int32
39
+ self.pack_factor = 32 // weight_bits # packed into int32
40
self.group_size = group_size
41
self.desc_act = desc_act
36
- self.is_sym = is_sym
42
self.lm_head_quantized = lm_head_quantized
43
44
+ if (weight_bits, is_sym) not in self.TYPE_MAP:
45
+ raise ValueError("Unsupported quantization config: "
46
+ f"bits={weight_bits}, sym={is_sym}")
47
+
48
+ self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
49
+
50
# Verify supported on platform.
40
- verify_gptq_marlin_supported(num_bits=self.weight_bits,
41
- group_size=self.group_size,
42
- is_sym=self.is_sym)
51
+ verify_marlin_supported(quant_type=self.quant_type,
52
+ group_size=self.group_size)
53
54
def __repr__(self) -> str:
45
- return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
55
+ return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
56
f"group_size={self.group_size}, "
57
f"desc_act={self.desc_act}, "
58
f"lm_head_quantized={self.lm_head_quantized})")
@@ -122,11 +132,12 @@ class GPTQMarlinConfig(QuantizationConfig):
132
or desc_act is None):
133
return False
134
125
- return check_gptq_marlin_supported(
126
- num_bits=num_bits,
127
- group_size=group_size,
128
- is_sym=sym,
129
- min_capability=cls.get_min_capability())
135
+ if (num_bits, sym) not in cls.TYPE_MAP:
136
+ return False
137
+
138
+ return check_marlin_supported(quant_type=cls.TYPE_MAP[(num_bits, sym)],
139
+ group_size=group_size,
140
+ min_capability=cls.get_min_capability())
141
142
143
class GPTQMarlinLinearMethod(LinearMethodBase):
@@ -293,7 +304,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
304
perm=layer.g_idx_sort_indices,
305
size_k=layer.input_size_per_partition,
306
size_n=layer.output_size_per_partition,
296
- num_bits=self.quant_config.weight_bits)
307
+ num_bits=self.quant_config.quant_type.size_bits)
308
replace_tensor(layer, "qweight", marlin_qweight)
309
310
# Permute scales from autogptq format to marlin format.
@@ -319,7 +330,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
330
g_idx=layer.g_idx,
331
g_idx_sort_indices=layer.g_idx_sort_indices,
332
workspace=layer.workspace,
322
- num_bits=self.quant_config.weight_bits,
333
+ wtype=self.quant_config.quant_type,
334
output_size_per_partition=layer.output_size_per_partition,
335
input_size_per_partition=layer.input_size_per_partition,
336
is_k_full=layer.is_k_full,
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+19
-10
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
from vllm.model_executor.utils import set_weight_attrs
12
+from vllm.scalar_type import scalar_types
13
14
logger = init_logger(__name__)
15
@@ -17,9 +18,10 @@ GPTQ_MARLIN_24_MIN_THREAD_N = 128
18
GPTQ_MARLIN_24_MIN_THREAD_K = 128
19
GPTQ_MARLIN_24_MAX_PARALLEL = 64
20
20
-GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
21
+GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES = [
22
+ scalar_types.uint4b8, scalar_types.uint8b128
23
+]
24
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
22
-GPTQ_MARLIN_24_SUPPORTED_SYM = [True]
25
26
27
class GPTQMarlin24Config(QuantizationConfig):
@@ -31,14 +33,19 @@ class GPTQMarlin24Config(QuantizationConfig):
33
weight_bits: int,
34
group_size: int,
35
) -> None:
34
- self.weight_bits = weight_bits
36
+ quant_type = {
37
+ 4: scalar_types.uint4b8,
38
+ 8: scalar_types.uint8b128,
39
+ }.get(weight_bits)
40
+
41
self.group_size = group_size
42
43
# Verify
38
- if self.weight_bits not in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS:
44
+ if quant_type is None or \
45
+ quant_type not in GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES:
46
raise ValueError(
40
- f"Marlin_24 does not support weight_bits = {self.weight_bits}. "
41
- f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_NUM_BITS} "
47
+ f"Marlin_24 does not support quant_type = {quant_type}. "
48
+ f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES} "
49
"are supported.")
50
if self.group_size not in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES:
51
raise ValueError(
@@ -46,8 +53,10 @@ class GPTQMarlin24Config(QuantizationConfig):
53
f"Only group_sizes = {GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES} "
54
"are supported.")
55
56
+ self.quant_type = quant_type
57
+
58
# 4 Bits packed into 32 bit datatype.
50
- self.pack_factor = 32 // self.weight_bits
59
+ self.pack_factor = 32 // self.quant_type.size_bits
60
61
# Tile size used by marlin kernels.
62
self.tile_size = 16
@@ -66,8 +75,8 @@ class GPTQMarlin24Config(QuantizationConfig):
75
self.perm_len = 1024
76
77
def __repr__(self) -> str:
69
- return "Marlin24Config(weight_bits={}, group_size={})".format(
70
- self.weight_bits, self.group_size)
78
+ return "Marlin24Config(quant_type={}, group_size={})".format(
79
+ self.quant_type, self.group_size)
80
81
@classmethod
82
def get_name(cls) -> str:
@@ -279,7 +288,7 @@ class GPTQMarlin24LinearMethod(LinearMethodBase):
288
289
output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
290
workspace,
282
- self.quant_config.weight_bits,
291
+ self.quant_config.quant_type,
292
size_m, size_n, size_k)
293
294
output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+57
-63
@@ -5,6 +5,7 @@ import torch
5
6
from vllm import _custom_ops as ops
7
from vllm.platforms import current_platform
8
+from vllm.scalar_type import ScalarType, scalar_types
9
10
from .quant_utils import pack_cols, unpack_cols
11
@@ -13,7 +14,6 @@ GPTQ_MARLIN_MIN_THREAD_N = 64
14
GPTQ_MARLIN_MIN_THREAD_K = 128
15
GPTQ_MARLIN_MAX_PARALLEL = 16
16
16
-MARLIN_SUPPORTED_NUM_BITS = [4, 8]
17
MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
18
19
# In case there is a performance issue with Marlin, the variable below can be
@@ -22,76 +22,70 @@ MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
22
USE_FP32_REDUCE_DEFAULT = True
23
24
25
-def _check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
26
- min_capability: Optional[int],
27
- has_zp: bool) -> Tuple[bool, Optional[str]]:
28
- if min_capability is not None:
25
+# For binary size and compile time, we don't support the same types for with and
26
+# without runtime zero-point. We support common cases, i.e. AWQ and GPTQ.
27
+# TODO: we may want to move this into the C++ so its closer to the actual impl
28
+def query_marlin_supported_quant_types(has_zp: bool,
29
+ min_capability: Optional[int] = None):
30
+ if min_capability is None:
31
major, minor = current_platform.get_device_capability()
30
- device_capability = major * 10 + minor
31
- if device_capability < min_capability:
32
- return (False, "Marlin does not support device_capability = {}"
33
- ", the min_capability required is {}".format(
34
- device_capability, min_capability))
35
-
36
- if num_bits not in MARLIN_SUPPORTED_NUM_BITS:
37
- return (False, "Marlin does not support weight_bits = {}. "
38
- "Only weight_bits = {} are supported.".format(
39
- num_bits, MARLIN_SUPPORTED_NUM_BITS))
40
-
41
- if group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
42
- return (False, "Marlin does not support group_size = {}. Only "
43
- "group_sizes = {} are supported.".format(
44
- group_size, MARLIN_SUPPORTED_GROUP_SIZES))
45
-
46
- if not has_zp and not is_sym:
47
- return (False,
48
- "Marlin without zero_points must have symmetric quantization")
32
+ min_capability = major * 10 + minor
33
50
- return True, None
34
+ if min_capability < 80:
35
+ return []
36
37
+ if has_zp:
38
+ # AWQ style, unsigned + runtime zero-point
39
+ return [scalar_types.uint4, scalar_types.uint8]
40
+ else:
41
+ # GPTQ style, unsigned + symmetric bias
42
+ # TODO: once fp8_marlin is merged into "gptq_marlin" we should be able
43
+ # to add `scalar_types.float8_e4m3fn` here
44
+ return [scalar_types.uint4b8, scalar_types.uint8b128]
45
53
-def check_gptq_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
54
- min_capability: int) -> bool:
55
- cond, _ = _check_marlin_supported(num_bits,
56
- group_size,
57
- is_sym,
58
- min_capability,
59
- has_zp=False)
60
- return cond
46
47
+def _check_marlin_supported(
48
+ quant_type: ScalarType,
49
+ group_size: Optional[int],
50
+ has_zp: bool,
51
+ min_capability: Optional[int] = None) -> Tuple[bool, Optional[str]]:
52
63
-def check_awq_marlin_supported(num_bits: int, group_size: int, has_zp: bool,
64
- min_capability: int) -> bool:
65
- cond, _ = _check_marlin_supported(num_bits,
66
- group_size,
67
- False,
68
- min_capability,
69
- has_zp=has_zp)
70
- return cond
53
+ if min_capability is None:
54
+ major, minor = current_platform.get_device_capability()
55
+ min_capability = major * 10 + minor
56
57
+ supported_types = query_marlin_supported_quant_types(
58
+ has_zp, min_capability)
59
73
-def verify_gptq_marlin_supported(num_bits: int, group_size: int,
74
- is_sym: bool) -> None:
75
- cond, err_msg = _check_marlin_supported(num_bits,
76
- group_size,
77
- is_sym,
78
- min_capability=None,
79
- has_zp=False)
80
- if not cond:
81
- assert err_msg is not None
82
- raise ValueError("GPTQ" + err_msg)
60
+ if quant_type not in supported_types:
61
+ return (False, f"Marlin does not support weight_bits = {quant_type}. "
62
+ f"Only types = {supported_types} "
63
+ f"are supported (for group_size = {group_size}, "
64
+ f"min_capability = {min_capability}, zp = {has_zp}).")
65
+ if (group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES):
66
+ return (False, f"Marlin does not support group_size = {group_size}. "
67
+ f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} "
68
+ "are supported.")
69
+
70
+ return True, None
71
+
72
+
73
+def check_marlin_supported(quant_type: ScalarType,
74
+ group_size: int,
75
+ has_zp: bool = False,
76
+ min_capability: Optional[int] = None) -> bool:
77
+ cond, _ = _check_marlin_supported(quant_type, group_size, has_zp,
78
+ min_capability)
79
+ return cond
80
81
85
-def verify_awq_marlin_supported(num_bits: int, group_size: int,
86
- has_zp: bool) -> None:
87
- cond, err_msg = _check_marlin_supported(num_bits,
88
- group_size,
89
- False,
90
- min_capability=None,
91
- has_zp=has_zp)
82
+def verify_marlin_supported(quant_type: ScalarType,
83
+ group_size: int,
84
+ has_zp: bool = False) -> None:
85
+ cond, err_msg = _check_marlin_supported(quant_type, group_size, has_zp)
86
if not cond:
87
assert err_msg is not None
94
- raise ValueError("AWQ" + err_msg)
88
+ raise ValueError(err_msg)
89
90
91
def verify_marlin_supports_shape(output_size_per_partition: int,
@@ -245,7 +239,7 @@ def apply_gptq_marlin_linear(
239
g_idx: torch.Tensor,
240
g_idx_sort_indices: torch.Tensor,
241
workspace: torch.Tensor,
248
- num_bits: int,
242
+ wtype: ScalarType,
243
output_size_per_partition: int,
244
input_size_per_partition: int,
245
is_k_full: bool,
@@ -261,7 +255,7 @@ def apply_gptq_marlin_linear(
255
g_idx,
256
g_idx_sort_indices,
257
workspace,
264
- num_bits,
258
+ wtype,
259
size_m=reshaped_x.shape[0],
260
size_n=output_size_per_partition,
261
size_k=input_size_per_partition,
@@ -283,7 +277,7 @@ def apply_awq_marlin_linear(
277
g_idx: torch.Tensor,
278
g_idx_sort_indices: torch.Tensor,
279
workspace: torch.Tensor,
286
- num_bits: int,
280
+ quant_type: ScalarType,
281
output_size_per_partition: int,
282
input_size_per_partition: int,
283
bias: Optional[torch.Tensor] = None,
@@ -298,7 +292,7 @@ def apply_awq_marlin_linear(
292
g_idx,
293
g_idx_sort_indices,
294
workspace,
301
- num_bits,
295
+ quant_type,
296
size_m=reshaped_x.shape[0],
297
size_n=output_size_per_partition,
298
size_k=input_size_per_partition,
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
+19
-10
@@ -5,10 +5,12 @@ from typing import List
5
import numpy as np
6
import torch
7
8
+from vllm.scalar_type import ScalarType
9
+
10
from .marlin_utils import (GPTQ_MARLIN_TILE, marlin_permute_scales,
11
marlin_zero_points)
10
-from .quant_utils import (get_pack_factor, quantize_weights,
11
- quantize_weights_with_zp, sort_weights)
12
+from .quant_utils import (get_pack_factor, gptq_quantize_weights,
13
+ quantize_weights, sort_weights)
14
15
16
class MarlinWorkspace:
@@ -90,9 +92,10 @@ def get_weight_perm(num_bits: int):
92
return perm
93
94
93
-def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
95
+def marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int,
96
act_order: bool):
97
size_k, size_n = w.shape
98
+ num_bits = quant_type.size_bits
99
100
# Normalize group_size
101
if group_size == -1:
@@ -100,8 +103,8 @@ def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
103
assert group_size <= size_k
104
105
# Quantize (and apply act_order if provided)
103
- w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
104
- act_order)
106
+ w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
107
+ w, quant_type, group_size, act_order)
108
109
# For act_order, sort the "weights" and "g_idx" so that group ids are
110
# increasing
@@ -122,7 +125,8 @@ def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
125
return res_list
126
127
125
-def awq_marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int):
128
+def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType,
129
+ group_size: int):
130
size_k, size_n = w.shape
131
132
# Normalize group_size
@@ -135,13 +139,18 @@ def awq_marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int):
139
num_groups = size_k // group_size
140
141
# Quantize with zp
138
- w_ref, q_w, s, zp = quantize_weights_with_zp(w, num_bits, group_size)
142
+ w_ref, q_w, s, zp = quantize_weights(w,
143
+ quant_type,
144
+ group_size,
145
+ zero_points=True)
146
147
# Reformat to marlin
141
- weight_perm = get_weight_perm(num_bits)
142
- marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
148
+ weight_perm = get_weight_perm(quant_type.size_bits)
149
+ marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
150
+ weight_perm)
151
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
144
- marlin_zp = marlin_zero_points(zp, num_groups, size_n, num_bits)
152
+ marlin_zp = marlin_zero_points(zp, num_groups, size_n,
153
+ quant_type.size_bits)
154
155
# Create result
156
res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py
+14
-16
@@ -6,8 +6,10 @@ from typing import List
6
import numpy
7
import torch
8
9
+from vllm.scalar_type import ScalarType
10
+
11
from .marlin_utils_test import marlin_weights
10
-from .quant_utils import quantize_weights
12
+from .quant_utils import gptq_quantize_weights
13
14
15
# This is PyTorch implementation of main part of reorder_meta()
@@ -348,13 +350,11 @@ def check_24(w, num_rows_to_sample=50, _verbose=False):
350
print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
351
352
351
-def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
353
+def compress_quantized_24_weight(q_24, size_k, size_n, wtype: ScalarType):
354
assert q_24.shape == (size_k, size_n)
355
354
- # Remove zp to normalize over 0
355
- max_q_val = (1 << num_bits) - 1
356
- zp = (max_q_val + 1) // 2
357
- q_24_no_zp = q_24 - zp
356
+ # Remove bias to normalize over 0
357
+ q_24_no_zp = q_24 - wtype.bias
358
359
# Compress
360
q_24_no_zp = q_24_no_zp.t().contiguous()
@@ -362,8 +362,8 @@ def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
362
q_24_no_zp)
363
q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
364
365
- # Restore zp
366
- q_24_comp = q_24_no_zp_comp + zp
365
+ # Restore bias
366
+ q_24_comp = q_24_no_zp_comp + wtype.bias
367
368
# Resize meta to its actual shape (without moving any data)
369
meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
@@ -427,7 +427,7 @@ def marlin_permute_scales_24(s: torch.Tensor, size_k: int, size_n: int,
427
428
def marlin_24_quantize(
429
w: torch.Tensor,
430
- num_bits: int,
430
+ quant_type: ScalarType,
431
group_size: int,
432
):
433
size_k, size_n = w.shape
@@ -441,20 +441,18 @@ def marlin_24_quantize(
441
w_24, mask_24 = inject_24(w, size_k, size_n)
442
443
# Quantize
444
- w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
445
- num_bits,
446
- group_size,
447
- act_order=False)
444
+ w_24_ref, q_w_24, s, g_idx, rand_perm = gptq_quantize_weights(
445
+ w_24, quant_type, group_size, act_order=False)
446
447
# Compress quantized weight
448
q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
451
- num_bits)
449
+ quant_type)
450
size_k_comp = size_k // 2
451
452
# Reformat to marlin
455
- weight_perm = get_weight_perm_24(num_bits)
453
+ weight_perm = get_weight_perm_24(quant_type.size_bits)
454
marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
457
- num_bits, weight_perm)
455
+ quant_type.size_bits, weight_perm)
456
marlin_24_s = marlin_permute_scales_24(s, size_k, size_n, group_size)
457
458
# Create result
vllm/model_executor/layers/quantization/utils/quant_utils.py
+64
-84
@@ -4,7 +4,11 @@ from typing import List
4
import numpy
5
import torch
6
7
-SUPPORTED_NUM_BITS = [4, 8]
7
+from vllm.model_executor.layers.quantization.qqq import (
8
+ MARLIN_QQQ_SUPPORTED_NUM_BITS)
9
+from vllm.scalar_type import ScalarType, scalar_types
10
+
11
+SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128]
12
SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
13
14
# Note: this is a hack. We should update each model to register the
@@ -45,7 +49,7 @@ def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
49
50
51
def get_pack_factor(num_bits):
48
- assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
52
+ assert 32 % num_bits == 0, f"Unsupported num_bits = {num_bits}"
53
return 32 // num_bits
54
55
@@ -74,24 +78,23 @@ def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
78
)
79
80
77
-def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
78
- act_order: bool):
81
+def quantize_weights(w: torch.Tensor,
82
+ quant_type: ScalarType,
83
+ group_size: int,
84
+ zero_points: bool = False):
85
+ assert quant_type.is_integer(), \
86
+ "Floating point quantization may work but has not been tested"
87
+
88
orig_device = w.device
89
+ orig_type = w.dtype
90
size_k, size_n = w.shape
91
92
assert w.is_floating_point(), "w must be float"
83
- assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
84
- assert group_size in SUPPORTED_GROUP_SIZES + [
85
- size_k
86
- ], f"Unsupported groupsize = {group_size}"
93
94
if group_size == -1:
95
group_size = size_k
96
assert group_size <= size_k
97
92
- max_q_val = 2**num_bits - 1
93
- half_q_val = (max_q_val + 1) // 2
94
-
98
# Reshape to [groupsize, -1]
99
if group_size < size_k:
100
w = w.reshape((-1, group_size, size_n))
@@ -99,16 +102,34 @@ def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
102
w = w.reshape((group_size, -1))
103
104
# Compute scale for each group
102
- s = torch.max(torch.abs(w), 0, keepdim=True)[0]
103
- s *= 2 / max_q_val # 2 => symmetric
105
+ max_val = torch.max(w, 0, keepdim=True).values
106
+ min_val = torch.min(w, 0, keepdim=True).values
107
+
108
+ max_q_val = quant_type.max()
109
+ min_q_val = quant_type.min()
110
+
111
+ if zero_points:
112
+ assert not quant_type.is_signed() and quant_type.max() > 0
113
+ w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
114
+ maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
115
+ .clamp(min_q_val, max_q_val).int()
116
+ else:
117
+ # If the bias is such that there are no possible negative/positive
118
+ # values, set the max value to inf to avoid divide by 0
119
+ w_s = torch.max(
120
+ abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
121
+ abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
122
+ maybe_w_zp = None
123
124
# Quantize
106
- q_w = torch.round(w / s).int()
107
- q_w += half_q_val
108
- q_w = torch.clamp(q_w, 0, max_q_val)
125
+ w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0)
126
+ w_q = torch.clamp(w_q, min_q_val, max_q_val)
127
128
# Compute ref (dequantized)
111
- w_ref = (q_w - half_q_val).half() * s
129
+ w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
130
+
131
+ if quant_type.has_bias():
132
+ w_q += quant_type.bias
133
134
# Restore original shapes
135
if group_size < size_k:
@@ -119,90 +140,48 @@ def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
140
w = w.reshape((size_k, size_n)).contiguous()
141
return w
142
122
- q_w = reshape_w(q_w)
143
+ w_q = reshape_w(w_q)
144
w_ref = reshape_w(w_ref)
145
125
- s = s.reshape((-1, size_n)).contiguous()
146
+ w_s = w_s.reshape((-1, size_n)).contiguous()
147
127
- # Apply act_order
128
- g_idx = torch.empty(0, dtype=torch.int, device=w.device)
129
- rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
130
- if act_order:
131
- assert (
132
- group_size < size_k
133
- ), "For act_order, groupsize = {} must be less than size_k = {}".format(
134
- group_size, size_k)
135
-
136
- w_ref, q_w, g_idx, rand_perm = permute_rows(q_w, w_ref, group_size)
148
+ if zero_points:
149
+ maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous()
150
+ maybe_w_zp = maybe_w_zp.to(device=orig_device)
151
152
return (
153
w_ref.to(device=orig_device),
140
- q_w.to(device=orig_device),
141
- s.to(device=orig_device),
142
- g_idx.to(device=orig_device),
143
- rand_perm.to(device=orig_device),
154
+ w_q.to(device=orig_device),
155
+ w_s.to(device=orig_device),
156
+ maybe_w_zp,
157
)
158
159
147
-def quantize_weights_with_zp(w: torch.Tensor, num_bits: int, group_size: int):
148
- orig_device = w.device
149
- size_k, size_n = w.shape
160
+def gptq_quantize_weights(w: torch.Tensor, quant_type: ScalarType,
161
+ group_size: int, act_order: bool):
162
+ size_k, _ = w.shape
163
164
assert w.is_floating_point(), "w must be float"
152
- assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
165
+ assert quant_type in SUPPORTED_GPTQ_QUANT_TYPES, \
166
+ f"Unsupported gptq type = {quant_type}"
167
assert group_size in SUPPORTED_GROUP_SIZES + [
168
size_k
169
], f"Unsupported groupsize = {group_size}"
170
157
- if group_size == -1:
158
- group_size = size_k
159
- assert group_size <= size_k
160
-
161
- max_q_val = 2**num_bits - 1
162
- min_q_val = 0
171
+ w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
172
164
- # Reshape to [groupsize, -1]
165
- if group_size < size_k:
166
- w = w.reshape((-1, group_size, size_n))
167
- w = w.permute(1, 0, 2)
168
- w = w.reshape((group_size, -1))
169
-
170
- # Compute scale for each group
171
- max = torch.max(w, 0, keepdim=True)[0]
172
- min = torch.min(w, 0, keepdim=True)[0]
173
- s = (max - min).clamp(min=1e-5) / max_q_val
174
-
175
- # Compute zero-point for each group
176
- zp = (-torch.round(min / s)).clamp(min_q_val, max_q_val).int()
177
-
178
- # Quantize
179
- q_w = torch.round(w / s).int() + zp
180
- q_w = torch.clamp(q_w, min_q_val, max_q_val)
181
-
182
- # Compute ref (dequantized)
183
- w_ref = (q_w - zp).half() * s
184
-
185
- # Restore original shapes
186
- if group_size < size_k:
187
-
188
- def reshape_w(w):
189
- w = w.reshape((group_size, -1, size_n))
190
- w = w.permute(1, 0, 2)
191
- w = w.reshape((size_k, size_n)).contiguous()
192
- return w
193
-
194
- q_w = reshape_w(q_w)
195
- w_ref = reshape_w(w_ref)
173
+ # Apply act_order
174
+ g_idx = torch.empty(0, dtype=torch.int, device=w.device)
175
+ rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
176
+ if act_order:
177
+ assert (
178
+ group_size < size_k
179
+ ), "For act_order, groupsize = {} must be less than size_k = {}".format(
180
+ group_size, size_k)
181
197
- s = s.reshape((-1, size_n)).contiguous()
198
- zp = zp.reshape((-1, size_n)).contiguous()
182
+ w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size)
183
200
- return (
201
- w_ref.to(device=orig_device),
202
- q_w.to(device=orig_device),
203
- s.to(device=orig_device),
204
- zp.to(device=orig_device),
205
- )
184
+ return w_ref, w_q, w_s, g_idx, rand_perm
185
186
187
# QQQ employs different quant schemes for per-group and
@@ -212,7 +191,8 @@ def qqq_quantize_weights(w: torch.Tensor, num_bits: int, group_size: int):
191
size_k, size_n = w.shape
192
193
assert w.is_floating_point(), "w must be float"
215
- assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
194
+ assert num_bits in MARLIN_QQQ_SUPPORTED_NUM_BITS, \
195
+ f"Unsupported num_bits = {num_bits}"
196
assert group_size in SUPPORTED_GROUP_SIZES + [
197
size_k
198
], f"Unsupported groupsize = {group_size}"
vllm/scalar_type.py
new
+35
@@ -0,0 +1,35 @@
1
+from ._core_ext import NanRepr, ScalarType
2
+
3
+# naming generally follows: https://github.com/jax-ml/ml_dtypes
4
+# for floating point types (leading f) the scheme is:
5
+# `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
6
+# flags:
7
+# - no-flags: means it follows IEEE 754 conventions
8
+# - f: means finite values only (no infinities)
9
+# - n: means nans are supported (non-standard encoding)
10
+# for integer types the scheme is:
11
+# `[u]int<size_bits>[b<bias>]`
12
+# - if bias is not present it means its zero
13
+
14
+
15
+class scalar_types:
16
+ int4 = ScalarType.int_(4, None)
17
+ uint4 = ScalarType.uint(4, None)
18
+ int8 = ScalarType.int_(8, None)
19
+ uint8 = ScalarType.uint(8, None)
20
+ float8_e4m3fn = ScalarType.float_(4, 3, True,
21
+ NanRepr.EXTD_RANGE_MAX_MIN.value)
22
+ float8_e5m2 = ScalarType.float_IEEE754(5, 2)
23
+ float16_e8m7 = ScalarType.float_IEEE754(8, 7)
24
+ float16_e5m10 = ScalarType.float_IEEE754(5, 10)
25
+
26
+ # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
27
+ float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE.value)
28
+
29
+ # "gptq" types
30
+ uint4b8 = ScalarType.uint(4, 8)
31
+ uint8b128 = ScalarType.uint(8, 128)
32
+
33
+ # colloquial names
34
+ bfloat16 = float16_e8m7
35
+ float16 = float16_e5m10