@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a8d604ca2a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Disambiguate quantized types via a new ScalarType (#6396)

Lucas Wilkinson committed Aug 2, 2024 at 16:51 UTC a8d604ca2a2912b3a5352821c53c080383580df1
29 files changed +1107 -352
CMakeLists.txt
+35 -17
@@ -66,6 +66,39 @@ endif()
66 #
67 find_package(Torch REQUIRED)
68
69 +#
70 +# Add the `default` target which detects which extensions should be
71 +# built based on platform/architecture. This is the same logic that
72 +# setup.py uses to select which extensions should be built and should
73 +# be kept in sync.
74 +#
75 +# The `default` target makes direct use of cmake easier since knowledge
76 +# of which extensions are supported has been factored in, e.g.
77 +#
78 +# mkdir build && cd build
79 +# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
80 +# cmake --build . --target default
81 +#
82 +add_custom_target(default)
83 +message(STATUS "Enabling core extension.")
84 +
85 +# Define _core_C extension
86 +# built for (almost) every target platform, (excludes TPU and Neuron)
87 +
88 +set(VLLM_EXT_SRC
89 + "csrc/core/torch_bindings.cpp")
90 +
91 +define_gpu_extension_target(
92 + _core_C
93 + DESTINATION vllm
94 + LANGUAGE CXX
95 + SOURCES ${VLLM_EXT_SRC}
96 + COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
97 + USE_SABI 3
98 + WITH_SOABI)
99 +
100 +add_dependencies(default _core_C)
101 +
102 #
103 # Forward the non-CUDA device extensions to external CMake scripts.
104 #
@@ -74,7 +107,7 @@ if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND
107 if (VLLM_TARGET_DEVICE STREQUAL "cpu")
108 include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake)
109 else()
77 - message(FATAL_ERROR "Unsupported vLLM target device: ${VLLM_TARGET_DEVICE}")
110 + return()
111 endif()
112 return()
113 endif()
@@ -132,7 +165,7 @@ if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
165 endif()
166
167 #
135 -# Define extension targets
168 +# Define other extension targets
169 #
170
171 #
@@ -228,21 +261,6 @@ define_gpu_extension_target(
261
262
263
231 -#
232 -# Add the `default` target which detects which extensions should be
233 -# built based on platform/architecture. This is the same logic that
234 -# setup.py uses to select which extensions should be built and should
235 -# be kept in sync.
236 -#
237 -# The `default` target makes direct use of cmake easier since knowledge
238 -# of which extensions are supported has been factored in, e.g.
239 -#
240 -# mkdir build && cd build
241 -# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
242 -# cmake --build . --target default
243 -#
244 -add_custom_target(default)
245 -
264 if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
265 message(STATUS "Enabling C extension.")
266 add_dependencies(default _C)
Dockerfile.openvino
+3
@@ -13,6 +13,9 @@ COPY requirements-common.txt /workspace/vllm/
13 COPY requirements-openvino.txt /workspace/vllm/
14
15 COPY vllm/ /workspace/vllm/vllm
16 +COPY csrc/core /workspace/vllm/csrc/core
17 +COPY cmake/utils.cmake /workspace/vllm/cmake/
18 +COPY CMakeLists.txt /workspace/vllm/
19 COPY setup.py /workspace/vllm/
20
21 # install build requirements
benchmarks/kernels/benchmark_marlin.py
+27 -23
@@ -7,16 +7,17 @@ from benchmark_shapes import WEIGHT_SHAPES
7 from vllm import _custom_ops as ops
8 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
9 GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
10 - GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
10 + GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES)
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13 - MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS)
13 + MARLIN_SUPPORTED_GROUP_SIZES, query_marlin_supported_quant_types)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15 MarlinWorkspace, marlin_quantize)
16 from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
17 marlin_24_quantize)
18 from vllm.model_executor.layers.quantization.utils.quant_utils import (
19 - gptq_pack, quantize_weights, sort_weights)
19 + gptq_pack, gptq_quantize_weights, sort_weights)
20 +from vllm.scalar_type import ScalarType
21 from vllm.utils import FlexibleArgumentParser
22
23 DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
@@ -27,13 +28,14 @@ K_FULL_OPTS = [False, True]
28
29
30 def bench_run(results: List[benchmark.Measurement], model: str,
30 - act_order: bool, is_k_full: bool, num_bits: int, group_size: int,
31 - size_m: int, size_k: int, size_n: int):
31 + act_order: bool, is_k_full: bool, quant_type: ScalarType,
32 + group_size: int, size_m: int, size_k: int, size_n: int):
33 label = "Quant Matmul"
34
34 - sub_label = ("{}, act={} k_full={}, b={}, g={}, "
35 - "MKN=({}x{}x{})".format(model, act_order, is_k_full, num_bits,
36 - group_size, size_m, size_k, size_n))
35 + sub_label = ("{}, act={} k_full={}, q={}, g={}, "
36 + "MKN=({}x{}x{})".format(model, act_order, is_k_full,
37 + str(quant_type), group_size, size_m,
38 + size_k, size_n))
39
40 print(f"Testing: {sub_label}")
41
@@ -50,18 +52,18 @@ def bench_run(results: List[benchmark.Measurement], model: str,
52 marlin_g_idx,
53 marlin_sort_indices,
54 marlin_rand_perm,
53 - ) = marlin_quantize(b, num_bits, group_size, act_order)
55 + ) = marlin_quantize(b, quant_type, group_size, act_order)
56
57 # Marlin_24 quant
58 (marlin_24_w_ref, marlin_24_q_w_comp, marlin_24_meta,
57 - marlin_24_s) = marlin_24_quantize(b, num_bits, group_size)
59 + marlin_24_s) = marlin_24_quantize(b, quant_type, group_size)
60
61 marlin_zp = torch.empty(0, dtype=torch.int, device=b.device)
62
63 # GPTQ quant
64 (w_ref, q_w, s, g_idx,
63 - rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
64 - q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
65 + rand_perm) = gptq_quantize_weights(b, quant_type, group_size, act_order)
66 + q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
67
68 # For act_order, sort the "weights" and "g_idx"
69 # so that group ids are increasing
@@ -75,10 +77,11 @@ def bench_run(results: List[benchmark.Measurement], model: str,
77
78 marlin_24_workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
79 GPTQ_MARLIN_24_MAX_PARALLEL)
80 + marlin_zp = torch.zeros_like(marlin_s, dtype=torch.int)
81
82 globals = {
83 # Gen params
81 - "num_bits": num_bits,
84 + "quant_type": quant_type,
85 "group_size": group_size,
86 "size_m": size_m,
87 "size_n": size_n,
@@ -128,7 +131,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
131 results.append(
132 benchmark.Timer(
133 stmt=
131 - "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
134 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
135 globals=globals,
136 label=label,
137 sub_label=sub_label,
@@ -138,19 +141,19 @@ def bench_run(results: List[benchmark.Measurement], model: str,
141 results.append(
142 benchmark.Timer(
143 stmt=
141 - "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
144 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
145 globals=globals,
146 label=label,
147 sub_label=sub_label,
148 description="gptq_marlin_gemm_fp32",
149 ).blocked_autorange(min_run_time=min_run_time))
150
148 - if (num_bits in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
151 + if (quant_type in GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES
152 and group_size in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES):
153 results.append(
154 benchmark.Timer(
155 stmt=
153 - "output = gptq_marlin_24_gemm(a, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s, marlin_24_workspace.scratch, num_bits, size_m, size_n, size_k)", # noqa: E501
156 + "output = gptq_marlin_24_gemm(a, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s, marlin_24_workspace.scratch, quant_type, size_m, size_n, size_k)", # noqa: E501
157 globals=globals,
158 label=label,
159 sub_label=sub_label,
@@ -160,7 +163,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
163 results.append(
164 benchmark.Timer(
165 stmt=
163 - "q_res = gptq_marlin_repack(q_w_gptq, repack_sort_indices, size_k, size_n, num_bits)", # noqa: E501
166 + "q_res = gptq_marlin_repack(q_w_gptq, repack_sort_indices, size_k, size_n, quant_type.size_bits)", # noqa: E501
167 globals=globals,
168 label=label,
169 sub_label=sub_label,
@@ -196,9 +199,10 @@ def main(args):
199 ) > 0 and is_k_full not in args.limit_k_full:
200 continue
201
199 - for num_bits in MARLIN_SUPPORTED_NUM_BITS:
200 - if len(args.limit_num_bits
201 - ) > 0 and num_bits not in args.limit_num_bits:
202 + for quant_type in query_marlin_supported_quant_types(
203 + False):
204 + if len(args.limit_num_bits) > 0 and \
205 + quant_type.size_bits not in args.limit_num_bits:
206 continue
207
208 for group_size in MARLIN_SUPPORTED_GROUP_SIZES:
@@ -215,8 +219,8 @@ def main(args):
219
220 for size_m in args.batch_sizes:
221 bench_run(results, model, act_order, is_k_full,
218 - num_bits, group_size, size_m, size_k,
219 - size_n)
222 + quant_type, group_size, size_m,
223 + size_k, size_n)
224
225 compare = benchmark.Compare(results)
226 compare.print()
cmake/cpu_extension.cmake
-1
@@ -113,6 +113,5 @@ define_gpu_extension_target(
113 WITH_SOABI
114 )
115
116 -add_custom_target(default)
116 message(STATUS "Enabling C extension.")
117 add_dependencies(default _C)
csrc/core/registration.h renamed
csrc/core/scalar_type.hpp new
+382
@@ -0,0 +1,382 @@
1 +#pragma once
2 +
3 +#include <torch/custom_class.h>
4 +
5 +namespace vllm {
6 +
7 +//
8 +// ScalarType can represent a wide range of floating point and integer types,
9 +// in particular it can be used to represent sub-byte data types (something
10 +// that torch.dtype currently does not support).
11 +//
12 +// ScalarTypeTorch is a subclass of ScalarType that is compatible with
13 +// TORCH_LIBRARY, making it accessible from Python as well meaning this class
14 +// can be used as a argument for custom operators, helping to simplify these
15 +// interfaces.
16 +//
17 +// The type definitions on the Python side can be found in: vllm/_core_ext.pyi
18 +// these type definitions should be kept up to date with any Python API changes
19 +// here.
20 +//
21 +class ScalarType {
22 + public:
23 + enum NanRepr : int64_t {
24 + NAN_NONE = 0, // nans are not supported
25 + NAN_IEEE_754 = 1, // nans are: exp all 1s, mantissa not all 0s
26 + NAN_EXTD_RANGE_MAX_MIN = 2, // nans are: exp all 1s, mantissa all 1s
27 +
28 + NAN_REPR_ID_MAX
29 + };
30 +
31 + constexpr ScalarType(bool signed_, int64_t exponent, int64_t mantissa,
32 + int64_t bias, bool finite_values_only = false,
33 + NanRepr nan_repr = NAN_IEEE_754)
34 + : exponent(exponent),
35 + mantissa(mantissa),
36 + bias(bias),
37 + signed_(signed_),
38 + finite_values_only(finite_values_only),
39 + nan_repr(nan_repr){};
40 +
41 + static constexpr ScalarType int_(int64_t size_bits, int64_t bias = 0) {
42 + return ScalarType(true, 0, size_bits - 1, bias);
43 + }
44 +
45 + static constexpr ScalarType uint(int64_t size_bits, int64_t bias = 0) {
46 + return ScalarType(false, 0, size_bits, bias);
47 + }
48 +
49 + // IEEE 754 compliant floating point type
50 + static constexpr ScalarType float_IEEE754(int64_t exponent,
51 + int64_t mantissa) {
52 + TORCH_CHECK(mantissa > 0 && exponent > 0);
53 + return ScalarType(true, exponent, mantissa, 0, false, NAN_IEEE_754);
54 + }
55 +
56 + // IEEE 754 non-compliant floating point type
57 + static constexpr ScalarType float_(int64_t exponent, int64_t mantissa,
58 + bool finite_values_only,
59 + NanRepr nan_repr) {
60 + TORCH_CHECK(nan_repr < NAN_REPR_ID_MAX, "Invalid NanRepr");
61 + TORCH_CHECK(mantissa > 0 && exponent > 0);
62 + TORCH_CHECK(nan_repr != NAN_IEEE_754,
63 + "use `float_IEEE754` constructor for floating point types that "
64 + "follow IEEE 754 conventions");
65 + return ScalarType(true, exponent, mantissa, 0, finite_values_only,
66 + nan_repr);
67 + }
68 +
69 + int64_t const exponent; // size of the exponent field (0 for integer types)
70 + int64_t const mantissa; // size of the mantissa field (size of the integer
71 + // excluding the sign bit for integer types)
72 + int64_t const bias; // stored values equal value + bias,
73 + // used for quantized type
74 + bool const signed_; // flag if the type supports negative numbers (i.e. has a
75 + // sign bit)
76 +
77 + // Extra Floating point info
78 + bool const finite_values_only; // i.e. no +/-inf if true
79 + NanRepr const nan_repr; // how NaNs are represented
80 + // (not applicable for integer types)
81 +
82 + int64_t size_bits() const { return mantissa + exponent + is_signed(); }
83 + bool is_signed() const { return signed_; }
84 + bool is_integer() const { return exponent == 0; }
85 + bool is_floating_point() const { return exponent > 0; }
86 + bool is_ieee_754() const {
87 + return is_floating_point() && finite_values_only == false &&
88 + nan_repr == NAN_IEEE_754;
89 + }
90 + bool has_nans() const { return is_floating_point() && nan_repr != NAN_NONE; }
91 + bool has_infs() const {
92 + return is_floating_point() && finite_values_only == false;
93 + }
94 + bool has_bias() const { return bias != 0; }
95 +
96 + private:
97 + double _floating_point_max() const {
98 + TORCH_CHECK(mantissa <= 52 && exponent <= 11,
99 + "Cannot represent max/min as a double for type ", str());
100 +
101 + uint64_t max_mantissa = (uint64_t(1) << mantissa) - 1;
102 + if (nan_repr == NAN_EXTD_RANGE_MAX_MIN) {
103 + max_mantissa -= 1;
104 + }
105 +
106 + uint64_t max_exponent = (uint64_t(1) << exponent) - 2;
107 + if (nan_repr == NAN_EXTD_RANGE_MAX_MIN || nan_repr == NAN_NONE) {
108 + TORCH_CHECK(exponent < 11,
109 + "Cannot represent max/min as a double for type ", str());
110 + max_exponent += 1;
111 + }
112 +
113 + // adjust the exponent to match that of a double
114 + // for now we assume the exponent bias is the standard 2^(e-1) -1, (where e
115 + // is the exponent bits), there is some precedent for non-standard biases,
116 + // example `float8_e4m3b11fnuz` here: https://github.com/jax-ml/ml_dtypes
117 + // but to avoid premature over complication we are just assuming the
118 + // standard exponent bias until there is a need to support non-standard
119 + // biases
120 + uint64_t exponent_bias = (uint64_t(1) << (exponent - 1)) - 1;
121 + uint64_t exponent_bias_double = (uint64_t(1) << 10) - 1; // double e = 11
122 +
123 + uint64_t max_exponent_double =
124 + max_exponent - exponent_bias + exponent_bias_double;
125 +
126 + // shift the mantissa into the position for a double and
127 + // the exponent
128 + uint64_t double_raw =
129 + (max_mantissa << (52 - mantissa)) | (max_exponent_double << 52);
130 +
131 + return *reinterpret_cast<double*>(&double_raw);
132 + }
133 +
134 + std::variant<int64_t, double> _raw_max() const {
135 + if (is_floating_point()) {
136 + return {_floating_point_max()};
137 + } else {
138 + TORCH_CHECK(size_bits() < 64 || size_bits() == 64 && is_signed(),
139 + "Cannot represent max as a int64_t");
140 + return {(int64_t(1) << mantissa) - 1};
141 + }
142 + }
143 +
144 + std::variant<int64_t, double> _raw_min() const {
145 + if (is_floating_point()) {
146 + TORCH_CHECK(is_signed(),
147 + "We currently assume all floating point types are signed");
148 + constexpr uint64_t sign_bit_double = (uint64_t(1) << 63);
149 +
150 + double max = _floating_point_max();
151 + uint64_t max_raw = *reinterpret_cast<uint64_t*>(&max);
152 + uint64_t min_raw = max_raw | sign_bit_double;
153 + return {*reinterpret_cast<double*>(&min_raw)};
154 + } else {
155 + TORCH_CHECK(!is_signed() || size_bits() <= 64,
156 + "Cannot represent min as a int64_t");
157 + if (is_signed()) {
158 + // set the top bit to 1 (i.e. INT64_MIN) and the rest to 0
159 + // then perform an arithmetic shift right to set all the bits above
160 + // (size_bits() - 1) to 1
161 + return {INT64_MIN >> (64 - size_bits())};
162 + } else {
163 + return {int64_t(0)};
164 + }
165 + }
166 + }
167 +
168 + public:
169 + // Max representable value for this scalar type.
170 + // (accounting for bias if there is one)
171 + std::variant<int64_t, double> max() const {
172 + return std::visit(
173 + [this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
174 + _raw_max());
175 + }
176 +
177 + // Min representable value for this scalar type.
178 + // (accounting for bias if there is one)
179 + std::variant<int64_t, double> min() const {
180 + return std::visit(
181 + [this](auto x) -> std::variant<int64_t, double> { return {x - bias}; },
182 + _raw_min());
183 + }
184 +
185 + std::string str() const {
186 + /* naming generally follows: https://github.com/jax-ml/ml_dtypes
187 + * for floating point types (leading f) the scheme is:
188 + * `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
189 + * flags:
190 + * - no-flags: means it follows IEEE 754 conventions
191 + * - f: means finite values only (no infinities)
192 + * - n: means nans are supported (non-standard encoding)
193 + * for integer types the scheme is:
194 + * `[u]int<size_bits>[b<bias>]`
195 + * - if bias is not present it means its zero
196 + */
197 + if (is_floating_point()) {
198 + auto ret = "float" + std::to_string(size_bits()) + "_e" +
199 + std::to_string(exponent) + "m" + std::to_string(mantissa);
200 + if (!is_ieee_754()) {
201 + if (finite_values_only) {
202 + ret += "f";
203 + }
204 + if (nan_repr != NAN_NONE) {
205 + ret += "n";
206 + }
207 + }
208 + return ret;
209 + } else {
210 + auto ret = ((is_signed()) ? "int" : "uint") + std::to_string(size_bits());
211 + if (has_bias()) {
212 + ret += "b" + std::to_string(bias);
213 + }
214 + return ret;
215 + }
216 + }
217 +
218 + bool operator==(ScalarType const& other) const {
219 + return mantissa == other.mantissa && exponent == other.exponent &&
220 + bias == other.bias && signed_ == other.signed_ &&
221 + finite_values_only == other.finite_values_only &&
222 + nan_repr == other.nan_repr;
223 + }
224 +};
225 +
226 +// Create a TORCH_LIBRARY compatible version of ScalarType (i.e. inherit from
227 +// torch::CustomClassHolder), we use multiple inheritance here since we cannot
228 +// have ScalarType inherit from torch::CustomClassHolder and have a constexpr
229 +// constructor at the same time (torch::CustomClassHolder does not have a
230 +// constexpr destructor)
231 +class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
232 + public:
233 + ScalarTypeTorch(int64_t exponent, int64_t mantissa, int64_t bias,
234 + bool _signed)
235 + : ScalarType(exponent, mantissa, bias, _signed){};
236 +
237 + ScalarTypeTorch(ScalarType type) : ScalarType(type){};
238 +
239 + using Base = ScalarType;
240 + using Self = ScalarTypeTorch;
241 + using SelfPtr = c10::intrusive_ptr<Self>;
242 +
243 + static SelfPtr int_(int64_t size_bits, c10::optional<int64_t> bias) {
244 + return c10::make_intrusive<Self>(
245 + ScalarType::int_(size_bits, bias.value_or(0)));
246 + }
247 +
248 + static SelfPtr uint(int64_t size_bits, c10::optional<int64_t> bias) {
249 + return c10::make_intrusive<Self>(
250 + ScalarType::uint(size_bits, bias.value_or(0)));
251 + }
252 +
253 + static SelfPtr float_IEEE754(int64_t exponent, int64_t mantissa) {
254 + return c10::make_intrusive<Self>(
255 + ScalarType::float_IEEE754(exponent, mantissa));
256 + }
257 +
258 + static SelfPtr float_(int64_t exponent, int64_t mantissa,
259 + bool finite_values_only, int64_t nan_repr) {
260 + return c10::make_intrusive<Self>(ScalarType::float_(
261 + exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
262 + }
263 +
264 + template <typename T>
265 + static void bind_readonly_property(torch::class_<Self>& cls,
266 + std::string const& name, T Base::*field) {
267 + auto getter_func = [field = std::move(field)](SelfPtr const& self) {
268 + if constexpr (std::is_member_function_pointer_v<decltype(field)>) {
269 + return (self.get()->*field)();
270 + } else {
271 + return self.get()->*field;
272 + }
273 + };
274 +
275 + cls.def_property(name, getter_func);
276 + }
277 +
278 + template <typename MemberFunc, typename Cls>
279 + static void bind_function(torch::class_<Self>& cls, const std::string& name,
280 + MemberFunc Cls::*member) {
281 + cls.def(name, [member = std::move(member)](SelfPtr const& self) {
282 + return (self.get()->*member)();
283 + });
284 + }
285 +
286 + template <typename Func>
287 + static void bind_function(torch::class_<Self>& cls, const std::string& name,
288 + Func func) {
289 + cls.def(name, func);
290 + }
291 +
292 + template <typename Func>
293 + static void bind_static_function(torch::class_<Self>& cls,
294 + const std::string& name, Func func) {
295 + cls.def_static(name, func);
296 + }
297 +
298 + static void bind_class(torch::Library& lib) {
299 + auto cls = lib.class_<ScalarTypeTorch>("ScalarType")
300 + .def(torch::init<int64_t, int64_t, int64_t, bool>());
301 +
302 + // Bind Properties
303 + bind_readonly_property(cls, "mantissa", &Base::mantissa);
304 + bind_readonly_property(cls, "exponent", &Base::exponent);
305 + bind_readonly_property(cls, "bias", &Base::bias);
306 + bind_readonly_property(cls, "signed", &Base::is_signed);
307 + bind_readonly_property(cls, "size_bits", &Base::size_bits);
308 +
309 + // Bind member functions
310 + bind_function(cls, "is_signed", &Base::is_signed);
311 + bind_function(cls, "is_integer", &Base::is_integer);
312 + bind_function(cls, "is_floating_point", &Base::is_floating_point);
313 + bind_function(cls, "is_ieee_754", &Base::is_ieee_754);
314 + bind_function(cls, "has_nans", &Base::has_nans);
315 + bind_function(cls, "has_infs", &Base::has_infs);
316 + bind_function(cls, "has_bias", &Base::has_bias);
317 +
318 + bind_function(cls, "max", [](SelfPtr const& self) {
319 + return std::visit([](auto arg) { return c10::IValue(arg); },
320 + self.get()->max());
321 + });
322 + bind_function(cls, "min", [](SelfPtr const& self) {
323 + return std::visit([](auto arg) { return c10::IValue(arg); },
324 + self.get()->min());
325 + });
326 +
327 + bind_function(cls, "__str__", &Base::str);
328 + bind_function(cls, "__eq__", [](SelfPtr const& self, SelfPtr const& other) {
329 + return *self == *other;
330 + });
331 + bind_function(cls, "__repr__", [](SelfPtr const& self) {
332 + return "ScalarType." + self.get()->str();
333 + });
334 +
335 + // Bind static functions (convenience constructors)
336 + bind_static_function(cls, "int_", &ScalarTypeTorch::int_);
337 + bind_static_function(cls, "uint", &ScalarTypeTorch::uint);
338 + bind_static_function(cls, "float_IEEE754", &ScalarTypeTorch::float_IEEE754);
339 + bind_static_function(cls, "float_", &ScalarTypeTorch::float_);
340 + }
341 +};
342 +
343 +using ScalarTypeTorchPtr = c10::intrusive_ptr<ScalarTypeTorch>;
344 +
345 +// "rust style" names generally following:
346 +// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L60-L70
347 +static inline constexpr auto kS4 = ScalarType::int_(4);
348 +static inline constexpr auto kU4 = ScalarType::uint(4);
349 +static inline constexpr auto kU4B8 = ScalarType::uint(4, 8);
350 +static inline constexpr auto kS8 = ScalarType::int_(8);
351 +static inline constexpr auto kU8 = ScalarType::uint(8);
352 +static inline constexpr auto kU8B128 = ScalarType::uint(8, 128);
353 +
354 +static inline constexpr auto kFE3M2f =
355 + ScalarType::float_(3, 2, true, ScalarType::NAN_NONE);
356 +static inline constexpr auto kFE4M3fn =
357 + ScalarType::float_(4, 3, true, ScalarType::NAN_EXTD_RANGE_MAX_MIN);
358 +static inline constexpr auto kFE5M2 = ScalarType::float_IEEE754(5, 2);
359 +static inline constexpr auto kFE8M7 = ScalarType::float_IEEE754(8, 7);
360 +static inline constexpr auto kFE5M10 = ScalarType::float_IEEE754(5, 10);
361 +
362 +// Fixed width style names, generally following:
363 +// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L47-L57
364 +static inline constexpr auto kInt4 = kS4;
365 +static inline constexpr auto kUint4 = kU4;
366 +static inline constexpr auto kUint4b8 = kU4B8;
367 +static inline constexpr auto kInt8 = kS8;
368 +static inline constexpr auto kUint8 = kU8;
369 +static inline constexpr auto kUint8b128 = kU8B128;
370 +
371 +static inline constexpr auto kFloat6_e3m2f = kFE3M2f;
372 +static inline constexpr auto kFloat8_e4m3fn = kFE4M3fn;
373 +static inline constexpr auto kFloat8_e5m2 = kFE5M2;
374 +static inline constexpr auto kFloat16_e8m7 = kFE8M7;
375 +static inline constexpr auto kFloat16_e5m10 = kFE5M10;
376 +
377 +// colloquial names
378 +static inline constexpr auto kHalf = kFE5M10;
379 +static inline constexpr auto kFloat16 = kHalf;
380 +static inline constexpr auto kBFloat16 = kFE8M7;
381 +
382 +}; // namespace vllm
csrc/core/torch_bindings.cpp new
+16
@@ -0,0 +1,16 @@
1 +#include <torch/library.h>
2 +
3 +#include "scalar_type.hpp"
4 +#include "registration.h"
5 +
6 +// Note the CORE exstension will be built for (almost) all hardware targets so
7 +// new additions must account for this. (currently not built for TPU and Neuron)
8 +
9 +TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, lib) {
10 + // ScalarType, a custom class for representing data types that supports
11 + // quantized types, declared here so it can be used when creating interfaces
12 + // for custom ops.
13 + vllm::ScalarTypeTorch::bind_class(lib);
14 +}
15 +
16 +REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cpu/torch_bindings.cpp
+1 -1
@@ -1,6 +1,6 @@
1 #include "cache.h"
2 #include "ops.h"
3 -#include "registration.h"
3 +#include "core/registration.h"
4
5 #include <torch/library.h>
6
csrc/moe/torch_bindings.cpp
+1 -1
@@ -1,4 +1,4 @@
1 -#include "registration.h"
1 +#include "core/registration.h"
2 #include "moe_ops.h"
3
4 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
csrc/ops.h
+6 -2
@@ -3,6 +3,8 @@
3 #include <optional>
4 #include <torch/library.h>
5
6 +#include "core/scalar_type.hpp"
7 +
8 void paged_attention_v1(
9 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
10 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
@@ -84,14 +86,16 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
86 torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
87 torch::Tensor& b_meta,
88 torch::Tensor& b_scales,
87 - torch::Tensor& workspace, int64_t num_bits,
89 + torch::Tensor& workspace,
90 + vllm::ScalarTypeTorchPtr const& b_q_type,
91 int64_t size_m, int64_t size_n,
92 int64_t size_k);
93
94 torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
95 torch::Tensor& b_scales, torch::Tensor& b_zeros,
96 torch::Tensor& g_idx, torch::Tensor& perm,
94 - torch::Tensor& workspace, int64_t num_bits,
97 + torch::Tensor& workspace,
98 + vllm::ScalarTypeTorchPtr const& b_q_type,
99 int64_t size_m, int64_t size_n, int64_t size_k,
100 bool is_k_full, bool has_zp,
101 bool use_fp32_reduce);
csrc/quantization/gptq_marlin/gptq_marlin.cu
+44 -22
@@ -21,6 +21,7 @@
21
22 #include "marlin.cuh"
23 #include "marlin_dtypes.cuh"
24 +#include "core/scalar_type.hpp"
25
26 #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
27 static_assert(std::is_same<scalar_t, half>::value || \
@@ -71,14 +72,15 @@ __global__ void Marlin(
72 bool use_fp32_reduce // whether to use fp32 global reduce
73 ) {}
74
74 -} // namespace gptq_marlin
75 +} // namespace marlin
76
77 torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
78 torch::Tensor& b_scales, torch::Tensor& b_zeros,
79 torch::Tensor& g_idx, torch::Tensor& perm,
79 - torch::Tensor& workspace, int64_t num_bits,
80 + torch::Tensor& workspace,
81 + vllm::ScalarTypeTorchPtr const& b_q_type,
82 int64_t size_m, int64_t size_n, int64_t size_k,
81 - bool is_k_full) {
83 + bool is_k_full, bool has_zp) {
84 TORCH_CHECK_NOT_IMPLEMENTED(false,
85 "marlin_gemm(..) requires CUDA_ARCH >= 8.0");
86 return torch::empty({1, 1});
@@ -1963,18 +1965,29 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1965 __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1966
1967 template <typename scalar_t>
1966 -void marlin_mm_f16i4(const void* A, const void* B, void* C, void* C_tmp,
1967 - void* s, void* zp, void* g_idx, void* perm, void* a_tmp,
1968 - int prob_m, int prob_n, int prob_k, void* workspace,
1969 - int num_bits, bool has_act_order, bool is_k_full,
1970 - bool has_zp, int num_groups, int group_size, int dev,
1971 - cudaStream_t stream, int thread_k, int thread_n, int sms,
1972 - int max_par, bool use_fp32_reduce) {
1973 - TORCH_CHECK(num_bits == 4 || num_bits == 8,
1974 - "num_bits must be 4 or 8. Got = ", num_bits);
1968 +void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
1969 + void* zp, void* g_idx, void* perm, void* a_tmp, int prob_m,
1970 + int prob_n, int prob_k, void* workspace,
1971 + vllm::ScalarType const& q_type, bool has_act_order,
1972 + bool is_k_full, bool has_zp, int num_groups, int group_size,
1973 + int dev, cudaStream_t stream, int thread_k, int thread_n,
1974 + int sms, int max_par, bool use_fp32_reduce) {
1975 + if (has_zp) {
1976 + TORCH_CHECK(
1977 + q_type == vllm::kU4 || q_type == vllm::kU8,
1978 + "q_type must be u4 or u8 when has_zp = True. Got = ", q_type.str());
1979 + } else {
1980 + TORCH_CHECK(
1981 + q_type == vllm::kU4B8 || q_type == vllm::kU8B128,
1982 + "q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ",
1983 + q_type.str());
1984 + }
1985 +
1986 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1987 ", ", prob_n, ", ", prob_k, "]");
1988
1989 + // TODO: remove alias when we start supporting other 8bit types
1990 + int num_bits = q_type.size_bits();
1991 int tot_m = prob_m;
1992 int tot_m_blocks = div_ceil(tot_m, 16);
1993 int pad = 16 * tot_m_blocks - tot_m;
@@ -2126,19 +2139,28 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* C_tmp,
2139 }
2140 }
2141
2129 -} // namespace gptq_marlin
2142 +} // namespace marlin
2143
2144 torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2145 torch::Tensor& b_scales, torch::Tensor& b_zeros,
2146 torch::Tensor& g_idx, torch::Tensor& perm,
2134 - torch::Tensor& workspace, int64_t num_bits,
2147 + torch::Tensor& workspace,
2148 + vllm::ScalarTypeTorchPtr const& b_q_type,
2149 int64_t size_m, int64_t size_n, int64_t size_k,
2150 bool is_k_full, bool has_zp,
2151 bool use_fp32_reduce) {
2138 - // Verify num_bits
2139 - TORCH_CHECK(num_bits == 4 || num_bits == 8,
2140 - "num_bits must be 4 or 8. Got = ", num_bits);
2141 - int pack_factor = 32 / num_bits;
2152 + if (has_zp) {
2153 + TORCH_CHECK(*b_q_type == vllm::kU4 || *b_q_type == vllm::kU8,
2154 + "b_q_type must be u4 or u8 when has_zp = True. Got = ",
2155 + b_q_type->str());
2156 + } else {
2157 + TORCH_CHECK(
2158 + *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
2159 + "b_q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ",
2160 + b_q_type->str());
2161 + }
2162 +
2163 + int pack_factor = 32 / b_q_type->size_bits();
2164
2165 // Verify A
2166 TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
@@ -2265,21 +2287,21 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2287
2288 int dev = a.get_device();
2289 if (a.scalar_type() == at::ScalarType::Half) {
2268 - marlin::marlin_mm_f16i4<half>(
2290 + marlin::marlin_mm<half>(
2291 a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
2292 c_tmp.data_ptr<float>(), b_scales.data_ptr<at::Half>(),
2293 b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2294 a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2273 - workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2295 + workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2296 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2297 thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2298 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
2277 - marlin::marlin_mm_f16i4<nv_bfloat16>(
2299 + marlin::marlin_mm<nv_bfloat16>(
2300 a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
2301 c.data_ptr<at::BFloat16>(), c_tmp.data_ptr<float>(),
2302 b_scales.data_ptr<at::BFloat16>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2303 perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2282 - workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2304 + workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2305 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2306 thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2307 } else {
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+10 -7
@@ -27,6 +27,7 @@
27 #include <iostream>
28
29 #include "common/base.h"
30 +#include "core/scalar_type.hpp"
31
32 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
33
@@ -86,7 +87,8 @@ __global__ void Marlin_24(
87 torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
88 torch::Tensor& b_meta,
89 torch::Tensor& b_scales,
89 - torch::Tensor& workspace, int64_t num_bits,
90 + torch::Tensor& workspace,
91 + vllm::ScalarTypeTorchPtr const& b_q_type,
92 int64_t size_m, int64_t size_n,
93 int64_t size_k) {
94 TORCH_CHECK_NOT_IMPLEMENTED(
@@ -1025,13 +1027,14 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
1027 torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1028 torch::Tensor& b_meta,
1029 torch::Tensor& b_scales,
1028 - torch::Tensor& workspace, int64_t num_bits,
1030 + torch::Tensor& workspace,
1031 + vllm::ScalarTypeTorchPtr const& b_q_type,
1032 int64_t size_m, int64_t size_n,
1033 int64_t size_k) {
1034 // Verify num_bits
1032 - TORCH_CHECK(num_bits == 4 || num_bits == 8,
1033 - "num_bits must be 4 or 8. Got = ", num_bits);
1034 - int pack_factor = 32 / num_bits;
1035 + TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
1036 + "num_bits must be uint4b8 or uint8b128. Got = ", b_q_type->str());
1037 + int pack_factor = 32 / b_q_type->size_bits();
1038
1039 // Verify M
1040 TORCH_CHECK(size_m == a.size(0),
@@ -1126,8 +1129,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1129 marlin_24::marlin_cuda_2_4(
1130 a.data_ptr(), b_q_weight.data_ptr(), b_meta.data_ptr(), c.data_ptr(),
1131 b_scales.data_ptr(), size_n, size_m, size_k, workspace.data_ptr(),
1129 - num_bits, groupsize, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1130 - thread_m, sms, max_par);
1132 + b_q_type->size_bits(), groupsize, dev,
1133 + at::cuda::getCurrentCUDAStream(dev), thread_k, thread_m, sms, max_par);
1134
1135 return c;
1136 }
csrc/torch_bindings.cpp
+1 -1
@@ -1,7 +1,7 @@
1 #include "cache.h"
2 #include "cuda_utils.h"
3 #include "ops.h"
4 -#include "registration.h"
4 +#include "core/registration.h"
5
6 #include <torch/library.h>
7
setup.py
+8 -1
@@ -271,6 +271,10 @@ def _build_custom_ops() -> bool:
271 return _is_cuda() or _is_hip() or _is_cpu()
272
273
274 +def _build_core_ext() -> bool:
275 + return not _is_neuron() and not _is_tpu()
276 +
277 +
278 def get_hipcc_rocm_version():
279 # Run the hipcc --version command
280 result = subprocess.run(['hipcc', '--version'],
@@ -433,6 +437,9 @@ def get_requirements() -> List[str]:
437
438 ext_modules = []
439
440 +if _build_core_ext():
441 + ext_modules.append(CMakeExtension(name="vllm._core_C"))
442 +
443 if _is_cuda() or _is_hip():
444 ext_modules.append(CMakeExtension(name="vllm._moe_C"))
445
@@ -477,7 +484,7 @@ setup(
484 extras_require={
485 "tensorizer": ["tensorizer>=2.9.0"],
486 },
480 - cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
487 + cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
488 package_data=package_data,
489 entry_points={
490 "console_scripts": [
tests/kernels/test_int8_quant.py
-2
@@ -1,8 +1,6 @@
1 import pytest
2 import torch
3
4 -# ruff: noqa: F401
5 -import vllm._C
4 from tests.kernels.quant_utils import ref_dynamic_per_token_quant
5 from vllm._custom_ops import scaled_int8_quant
6
tests/kernels/test_marlin_gemm.py
+41 -34
@@ -9,14 +9,14 @@ from tests.quantization.utils import is_quant_method_supported
9 from vllm import _custom_ops as ops
10 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11 GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
12 - GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
12 + GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES)
13 from vllm.model_executor.layers.quantization.qqq import (
14 MARLIN_QQQ_MAX_PARALLEL, MARLIN_QQQ_MIN_THREAD_N,
15 MARLIN_QQQ_SUPPORTED_GROUP_SIZES, MARLIN_QQQ_SUPPORTED_NUM_BITS)
16 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
17 GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
18 - MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS,
19 - marlin_make_empty_g_idx, marlin_permute_scales)
18 + MARLIN_SUPPORTED_GROUP_SIZES, marlin_make_empty_g_idx,
19 + marlin_permute_scales, query_marlin_supported_quant_types)
20 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
21 pack_fp8_to_int32)
22 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
@@ -27,8 +27,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
27 from vllm.model_executor.layers.quantization.utils.marlin_utils_test_qqq import ( # noqa: E501
28 marlin_qqq_quantize)
29 from vllm.model_executor.layers.quantization.utils.quant_utils import (
30 - awq_pack, gptq_pack, quantize_weights, quantize_weights_with_zp,
31 - sort_weights)
30 + awq_pack, gptq_pack, gptq_quantize_weights, quantize_weights, sort_weights)
31
32 ACT_ORDER_OPTS = [False, True]
33 K_FULL_OPTS = [False, True]
@@ -65,12 +64,13 @@ def rand_data(shape, dtype=torch.float16):
64 reason="Marlin is not supported on this GPU type.")
65 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
66 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
68 -@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
67 +@pytest.mark.parametrize("quant_type",
68 + query_marlin_supported_quant_types(False))
69 @pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
70 @pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
71 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
72 -def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
73 - mnk_factors):
72 +def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
73 + act_order, mnk_factors):
74 m_factor, n_factor, k_factor = mnk_factors
75
76 size_m = m_factor
@@ -95,11 +95,11 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
95 b_weight = rand_data((size_k, size_n))
96
97 # Quantize (and apply act_order if provided)
98 - w_ref, q_w, s, g_idx, rand_perm = quantize_weights(b_weight, num_bits,
99 - group_size, act_order)
98 + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
99 + b_weight, quant_type, group_size, act_order)
100
101 # Pack to GPTQ format
102 - q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
102 + q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
103
104 # For act_order, sort the "weights" and "g_idx" so that group ids are
105 # increasing
@@ -108,8 +108,9 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
108 q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
109
110 # Pack to Marlin format
111 - weight_perm = get_weight_perm(num_bits)
112 - marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
111 + weight_perm = get_weight_perm(quant_type.size_bits)
112 + marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
113 + weight_perm)
114
115 # Run Marlin repack GPU kernel
116 marlin_q_w_2 = ops.gptq_marlin_repack(
@@ -117,7 +118,7 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
118 sort_indices,
119 size_k,
120 size_n,
120 - num_bits,
121 + quant_type.size_bits,
122 )
123 torch.cuda.synchronize()
124
@@ -128,10 +129,11 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
129 reason="Marlin is not supported on this GPU type.")
130 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
131 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
131 -@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
132 +@pytest.mark.parametrize("quant_type",
133 + query_marlin_supported_quant_types(False))
134 @pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
135 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
134 -def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
136 +def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
137 mnk_factors):
138 m_factor, n_factor, k_factor = mnk_factors
139
@@ -150,22 +152,25 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
152 b_weight = rand_data((size_k, size_n))
153
154 # Quantize
153 - w_ref, q_w, s, zp = quantize_weights_with_zp(b_weight, num_bits,
154 - group_size)
155 + w_ref, q_w, s, zp = quantize_weights(b_weight,
156 + quant_type,
157 + group_size,
158 + zero_points=True)
159
160 # Pack to AWQ format
157 - q_w_awq = awq_pack(q_w, num_bits, size_k, size_n)
161 + q_w_awq = awq_pack(q_w, quant_type.size_bits, size_k, size_n)
162
163 # Pack to Marlin format
160 - weight_perm = get_weight_perm(num_bits)
161 - marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
164 + weight_perm = get_weight_perm(quant_type.size_bits)
165 + marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
166 + weight_perm)
167
168 # Run Marlin repack GPU kernel
169 marlin_q_w_2 = ops.awq_marlin_repack(
170 q_w_awq,
171 size_k,
172 size_n,
168 - num_bits,
173 + quant_type.size_bits,
174 )
175 torch.cuda.synchronize()
176
@@ -176,7 +181,8 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
181 reason="Marlin is not supported on this GPU type.")
182 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
183 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
179 -@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
184 +@pytest.mark.parametrize("quant_type",
185 + query_marlin_supported_quant_types(False))
186 @pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
187 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
188 @pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
@@ -185,7 +191,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
191 def test_gptq_marlin_gemm(
192 k_chunk,
193 n_chunk,
188 - num_bits,
194 + quant_type,
195 group_size,
196 mnk_factors,
197 act_order,
@@ -211,7 +217,7 @@ def test_gptq_marlin_gemm(
217 b_weight = rand_data((size_k, size_n))
218
219 w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
214 - b_weight, num_bits, group_size, act_order)
220 + b_weight, quant_type, group_size, act_order)
221
222 marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
223
@@ -226,7 +232,7 @@ def test_gptq_marlin_gemm(
232 g_idx,
233 sort_indices,
234 workspace.scratch,
229 - num_bits,
235 + quant_type,
236 a_input.shape[0],
237 b_weight.shape[1],
238 a_input.shape[1],
@@ -248,10 +254,10 @@ def test_gptq_marlin_gemm(
254 reason="Marlin is not supported on this GPU type.")
255 @pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
256 @pytest.mark.parametrize("n_chunk", MARLIN_24_N_CHUNKS)
251 -@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
257 +@pytest.mark.parametrize("quant_type", GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES)
258 @pytest.mark.parametrize("group_size", GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES)
259 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
254 -def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
260 +def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
261 mnk_factors):
262 m_factor, n_factor, k_factor = mnk_factors
263
@@ -266,7 +272,7 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
272 b_weight = rand_data((size_k, size_n))
273
274 (w_24_ref, marlin_24_q_w_comp, marlin_24_meta,
269 - marlin_24_s) = marlin_24_quantize(b_weight, num_bits, group_size)
275 + marlin_24_s) = marlin_24_quantize(b_weight, quant_type, group_size)
276
277 workspace_24 = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
278 GPTQ_MARLIN_24_MAX_PARALLEL)
@@ -279,7 +285,7 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size,
285 marlin_24_meta,
286 marlin_24_s,
287 workspace_24.scratch,
282 - num_bits,
288 + quant_type,
289 a_input.shape[0],
290 b_weight.shape[1],
291 a_input.shape[1],
@@ -371,14 +377,15 @@ def test_fp8_marlin_gemm(
377 reason="Marlin is not supported on this GPU type.")
378 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
379 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
374 -@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
380 +@pytest.mark.parametrize("quant_type",
381 + query_marlin_supported_quant_types(True))
382 @pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
383 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
384 @pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
385 def test_awq_marlin_gemm(
386 k_chunk,
387 n_chunk,
381 - num_bits,
388 + quant_type,
389 group_size,
390 mnk_factors,
391 use_fp32_reduce,
@@ -396,7 +403,7 @@ def test_awq_marlin_gemm(
403 b_weight = rand_data((size_k, size_n))
404
405 w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
399 - b_weight, num_bits, group_size)
406 + b_weight, quant_type, group_size)
407
408 g_idx = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
409 sort_indices = torch.empty(0, dtype=torch.int, device=marlin_q_w.device)
@@ -414,7 +421,7 @@ def test_awq_marlin_gemm(
421 g_idx,
422 sort_indices,
423 workspace.scratch,
417 - num_bits,
424 + quant_type,
425 a_input.shape[0],
426 b_weight.shape[1],
427 a_input.shape[1],
tests/test_scalartype.py new
+36
@@ -0,0 +1,36 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm.scalar_type import scalar_types
5 +
6 +
7 +@pytest.mark.parametrize("type_tuple", (
8 + (-8, 7, scalar_types.int4),
9 + (0, 15, scalar_types.uint4),
10 + (-8, 7, scalar_types.uint4b8),
11 + (-128, 127, scalar_types.uint8b128),
12 + (-28., 28., scalar_types.float6_e3m2f),
13 + (torch.int8, scalar_types.int8),
14 + (torch.uint8, scalar_types.uint8),
15 + (torch.float8_e5m2, scalar_types.float8_e5m2),
16 + (torch.float8_e4m3fn, scalar_types.float8_e4m3fn),
17 + (torch.bfloat16, scalar_types.float16_e8m7),
18 + (torch.float16, scalar_types.float16_e5m10),
19 +),
20 + ids=lambda x: str(x))
21 +def test_scalar_type_min_max(type_tuple):
22 + print(type_tuple)
23 + if len(type_tuple) == 3:
24 + min, max, t = type_tuple
25 + else:
26 + torch_type, t = type_tuple
27 + if torch_type.is_floating_point:
28 + min = torch.finfo(torch_type).min
29 + max = torch.finfo(torch_type).max
30 + else:
31 + min = torch.iinfo(torch_type).min
32 + max = torch.iinfo(torch_type).max
33 +
34 + print(t, min, max, t.min(), t.max())
35 + assert min == t.min()
36 + assert max == t.max()
vllm/_core_ext.py new
+177
@@ -0,0 +1,177 @@
1 +import importlib.util
2 +from enum import Enum
3 +from typing import TYPE_CHECKING, Optional, Union
4 +
5 +import torch
6 +
7 +from vllm.logger import init_logger
8 +
9 +logger = init_logger(__name__)
10 +core_C_available = importlib.util.find_spec('._core_C', 'vllm') is not None
11 +
12 +
13 +# Mirrors enum in `core/scalar_type.hpp`
14 +class NanRepr(Enum):
15 + NONE = 0 # nans are not supported
16 + IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s
17 + EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s
18 +
19 +
20 +if TYPE_CHECKING or not core_C_available:
21 + # On platforms were we cannot use/build the C++ core extension (i.e. namely
22 + # neuron and tpu), we define the mock ScalarType class here that partially
23 + # mimics the C++ ScalarType class.
24 + #
25 + # We also use this provide type signatures to the Python LSP for the methods
26 + # in the C++ ScalarType class. So these type signatures should be kept
27 + # in sync with csrc/core/scalar_type.hpp
28 +
29 + from dataclasses import dataclass
30 +
31 + @dataclass(frozen=True)
32 + class ScalarType:
33 + """
34 + ScalarType can represent a wide range of floating point and integer
35 + types, in particular it can be used to represent sub-byte data types
36 + (something that torch.dtype currently does not support). It is also
37 + capable of representing types with a bias, i.e.:
38 + `stored_value = value + bias`,
39 + this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40 + of 8). The implementation for this class can be found in
41 + csrc/core/scalar_type.hpp, these type signatures should be kept in sync
42 + with that file.
43 + """
44 +
45 + exponent: int
46 + """
47 + Number of bits in the exponent if this is a floating point type
48 + (zero if this an integer type)
49 + """
50 +
51 + mantissa: int
52 + """
53 + Number of bits in the mantissa if this is a floating point type,
54 + or the number bits representing an integer excluding the sign bit if
55 + this an integer type.
56 + """
57 +
58 + bias: int
59 + """
60 + bias used to encode the values in this scalar type
61 + (value = stored_value - bias, default 0) for example if we store the
62 + type as an unsigned integer with a bias of 128 then the value 0 will be
63 + stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
64 + """
65 +
66 + signed: bool
67 + "If the type is signed (i.e. has a sign bit)"
68 +
69 + _finite_values_only: bool = False
70 + """
71 + Private: if NANs are supported, used `has_infs()` instead.
72 + """
73 +
74 + nan_repr: int = NanRepr.IEEE_754.value
75 + """
76 + How NaNs are represent in this scalar type, returns NanRepr value.
77 + (not applicable for integer types)
78 + """
79 +
80 + @property
81 + def size_bits(self):
82 + return self.exponent + self.mantissa + int(self.signed)
83 +
84 + def min(self) -> Union[int, float]:
85 + """
86 + Min representable value for this scalar type.
87 + (accounting for bias if there is one)
88 + """
89 + raise NotImplementedError
90 +
91 + def max(self) -> Union[int, float]:
92 + """
93 + Max representable value for this scalar type.
94 + (accounting for bias if there is one)
95 + """
96 + raise NotImplementedError
97 +
98 + def is_signed(self) -> bool:
99 + """
100 + If the type is signed (i.e. has a sign bit), same as `signed`
101 + added for consistency with:
102 + https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
103 + """
104 + ...
105 +
106 + def is_floating_point(self):
107 + "If the type is a floating point type"
108 + return self.exponent != 0
109 +
110 + def is_integer(self):
111 + "If the type is an integer type"
112 + return self.exponent == 0
113 +
114 + def has_bias(self):
115 + "If the type has a non-zero bias"
116 + return self.bias != 0
117 +
118 + def has_infs(self):
119 + "If the type is floating point and supports infinity"
120 + return not self._finite_values_only
121 +
122 + def has_nans(self):
123 + return self.nan_repr != NanRepr.NONE.value
124 +
125 + def is_ieee_754(self) -> bool:
126 + """
127 + If the type is a floating point type that follows IEEE 754
128 + conventions
129 + """
130 + return self.nan_repr == NanRepr.IEEE_754.value and \
131 + not self._finite_values_only
132 +
133 + def __str__(self) -> str:
134 + raise NotImplementedError
135 +
136 + def __repr__(self) -> str:
137 + raise NotImplementedError
138 +
139 + #
140 + # Convenience Constructors
141 + #
142 +
143 + @classmethod
144 + def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
145 + "Create a signed integer scalar type (size_bits includes sign-bit)."
146 + return cls(size_bits - 1, size_bits, bias if bias else 0, True)
147 +
148 + @classmethod
149 + def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
150 + """Create a unsigned integer scalar type."""
151 + return cls(size_bits, size_bits, bias if bias else 0, False)
152 +
153 + @classmethod
154 + def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
155 + """
156 + Create a standard floating point type
157 + (i.e. follows IEEE 754 conventions).
158 + """
159 + return cls(exponent, mantissa, 0, True)
160 +
161 + @classmethod
162 + def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
163 + nan_repr: int):
164 + """
165 + Create a non-standard floating point type
166 + (i.e. does not follow IEEE 754 conventions).
167 + """
168 + return cls(exponent, mantissa, 0, True, finite_values_only,
169 + nan_repr)
170 +
171 +elif core_C_available:
172 + try:
173 + import vllm._core_C # noqa: F401
174 + except ImportError as e:
175 + logger.warning("Failed to import from vllm._core_C with %r", e)
176 +
177 + ScalarType = torch.classes._core_C.ScalarType
vllm/_custom_ops.py
+19 -10
@@ -4,6 +4,7 @@ from typing import List, Optional, Tuple, Union
4
5 import torch
6
7 +from vllm._core_ext import ScalarType
8 from vllm.logger import init_logger
9
10 logger = init_logger(__name__)
@@ -220,10 +221,10 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
221 # marlin_24
222 def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
223 b_meta: torch.Tensor, b_scales: torch.Tensor,
223 - workspace: torch.Tensor, num_bits: int, size_m: int,
224 - size_n: int, size_k: int) -> torch.Tensor:
224 + workspace: torch.Tensor, b_q_type: ScalarType,
225 + size_m: int, size_n: int, size_k: int) -> torch.Tensor:
226 return torch.ops._C.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
226 - workspace, num_bits, size_m,
227 + workspace, b_q_type, size_m,
228 size_n, size_k)
229
230
@@ -279,14 +280,22 @@ def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
280 return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
281
282
282 -def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
283 - b_scales: torch.Tensor, b_zeros: torch.Tensor,
284 - g_idx: torch.Tensor, perm: torch.Tensor,
285 - workspace: torch.Tensor, num_bits: int, size_m: int,
286 - size_n: int, size_k: int, is_k_full: bool, has_zp: bool,
287 - use_fp32_reduce: bool) -> torch.Tensor:
283 +def gptq_marlin_gemm(a: torch.Tensor,
284 + b_q_weight: torch.Tensor,
285 + b_scales: torch.Tensor,
286 + b_zeros: torch.Tensor,
287 + g_idx: torch.Tensor,
288 + perm: torch.Tensor,
289 + workspace: torch.Tensor,
290 + b_q_type: ScalarType,
291 + size_m: int,
292 + size_n: int,
293 + size_k: int,
294 + is_k_full: bool,
295 + has_zp: bool = False,
296 + use_fp32_reduce: bool = False) -> torch.Tensor:
297 return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
289 - g_idx, perm, workspace, num_bits,
298 + g_idx, perm, workspace, b_q_type,
299 size_m, size_n, size_k, is_k_full,
300 has_zp, use_fp32_reduce)
301
vllm/model_executor/layers/quantization/awq_marlin.py
+31 -18
@@ -10,11 +10,11 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13 - apply_awq_marlin_linear, awq_to_marlin_zero_points,
14 - check_awq_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
15 - marlin_permute_scales, replace_tensor, verify_awq_marlin_supported,
16 - verify_marlin_supports_shape)
13 + apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
14 + marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15 + replace_tensor, verify_marlin_supported, verify_marlin_supports_shape)
16 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17 +from vllm.scalar_type import scalar_types
18
19 logger = init_logger(__name__)
20
@@ -22,20 +22,31 @@ logger = init_logger(__name__)
22 class AWQMarlinConfig(QuantizationConfig):
23 """Config class for AWQ Marlin"""
24
25 + # num_bits -> type
26 + TYPE_MAP = {
27 + 4: scalar_types.uint4,
28 + 8: scalar_types.uint8,
29 + }
30 +
31 def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
32 lm_head_quantized: bool) -> None:
27 - self.weight_bits = weight_bits
28 - self.pack_factor = 32 // self.weight_bits # packed into 32bits
33 + self.pack_factor = 32 // weight_bits # packed into int32
34 self.group_size = group_size
35 self.has_zp = has_zp
36 self.lm_head_quantized = lm_head_quantized
37
33 - verify_awq_marlin_supported(num_bits=self.weight_bits,
34 - group_size=self.group_size,
35 - has_zp=self.has_zp)
38 + if weight_bits not in self.TYPE_MAP:
39 + raise ValueError(f"Unsupported num_bits = {weight_bits}. "
40 + f"Supported num_bits = {self.TYPE_MAP.keys()}")
41 +
42 + self.quant_type = self.TYPE_MAP[weight_bits]
43 +
44 + verify_marlin_supported(self.quant_type,
45 + group_size=self.group_size,
46 + has_zp=self.has_zp)
47
48 def __repr__(self) -> str:
38 - return (f"AWQMarlinConfig(weight_bits={self.weight_bits}, "
49 + return (f"AWQMarlinConfig(quant_type={self.quant_type}, "
50 f"group_size={self.group_size}, "
51 f"has_zp={self.has_zp}, "
52 f"lm_head_quantized={self.lm_head_quantized})")
@@ -110,11 +121,13 @@ class AWQMarlinConfig(QuantizationConfig):
121 if (num_bits is None or group_size is None or has_zp is None):
122 return False
123
113 - return check_awq_marlin_supported(
114 - num_bits=num_bits,
115 - group_size=group_size,
116 - has_zp=has_zp,
117 - min_capability=cls.get_min_capability())
124 + if num_bits not in cls.TYPE_MAP:
125 + return False
126 +
127 + return check_marlin_supported(quant_type=cls.TYPE_MAP[num_bits],
128 + group_size=group_size,
129 + has_zp=has_zp,
130 + min_capability=cls.get_min_capability())
131
132
133 class AWQMarlinLinearMethod(LinearMethodBase):
@@ -226,7 +239,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
239 layer.qweight,
240 size_k=layer.input_size_per_partition,
241 size_n=layer.output_size_per_partition,
229 - num_bits=self.quant_config.weight_bits)
242 + num_bits=self.quant_config.quant_type.size_bits)
243 replace_tensor(layer, "qweight", marlin_qweight)
244
245 # Permute scales from AWQ format to marlin format.
@@ -242,7 +255,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
255 layer.qzeros,
256 size_k=layer.num_groups,
257 size_n=layer.output_size_per_partition,
245 - num_bits=self.quant_config.weight_bits)
258 + num_bits=self.quant_config.quant_type.size_bits)
259 replace_tensor(layer, "qzeros", marlin_zp)
260
261 # Not-used
@@ -263,7 +276,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
276 g_idx=layer.g_idx,
277 g_idx_sort_indices=layer.g_idx_sort_indices,
278 workspace=layer.workspace,
266 - num_bits=self.quant_config.weight_bits,
279 + quant_type=self.quant_config.quant_type,
280 output_size_per_partition=layer.output_size_per_partition,
281 input_size_per_partition=layer.input_size_per_partition,
282 bias=bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+14 -4
@@ -9,9 +9,13 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10 GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N)
11 from vllm.model_executor.utils import set_weight_attrs
12 +from vllm.scalar_type import scalar_types
13
14 __all__ = ["CompressedTensorsW4A16Sparse24"]
14 -W4A16SPARSE24_SUPPORTED_BITS = [4]
15 +W4A16SPARSE24_SUPPORTED_TYPES_MAP = {
16 + 4: scalar_types.uint4b8,
17 +}
18 +W4A16SPARSE24_SUPPORTED_BITS = list(W4A16SPARSE24_SUPPORTED_TYPES_MAP.keys())
19
20
21 class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
@@ -22,9 +26,15 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
26 group_size: Optional[int] = None):
27 self.strategy = strategy
28 self.group_size = group_size
25 - self.num_bits = num_bits
29 self.tile_size = 16
30
31 + if num_bits not in W4A16SPARSE24_SUPPORTED_TYPES_MAP:
32 + raise ValueError(
33 + f"Unsupported num_bits = {num_bits}. "
34 + f"Supported num_bits = {W4A16SPARSE24_SUPPORTED_BITS}")
35 +
36 + self.quant_type = W4A16SPARSE24_SUPPORTED_TYPES_MAP[num_bits]
37 +
38 if self.strategy == "group" and self.group_size is None:
39 raise ValueError(
40 "group_size must be given when using strategy group")
@@ -43,7 +53,7 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
53 params_dtype: torch.dtype, weight_loader: Callable,
54 **kwargs):
55
46 - pack_factor = 32 // self.num_bits
56 + pack_factor = 32 // self.quant_type.size_bits
57 output_size_per_partition = sum(output_partition_sizes)
58
59 qweight = Parameter(
@@ -138,7 +148,7 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
148 size_n = scales.shape[1]
149
150 output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
141 - workspace, self.num_bits, size_m,
151 + workspace, self.quant_type, size_m,
152 size_n, size_k)
153
154 output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+20 -9
@@ -8,12 +8,17 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
10 apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11 - marlin_permute_scales, replace_tensor, verify_gptq_marlin_supported,
11 + marlin_permute_scales, replace_tensor, verify_marlin_supported,
12 verify_marlin_supports_shape)
13 from vllm.model_executor.utils import set_weight_attrs
14 +from vllm.scalar_type import scalar_types
15
16 __all__ = ["CompressedTensorsWNA16"]
16 -WNA16_SUPPORTED_BITS = [4, 8]
17 +WNA16_SUPPORTED_TYPES_MAP = {
18 + 4: scalar_types.uint4b8,
19 + 8: scalar_types.uint8b128,
20 +}
21 +WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
22
23
24 class CompressedTensorsWNA16(CompressedTensorsScheme):
@@ -22,8 +27,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
27 strategy: str,
28 num_bits: int,
29 group_size: Optional[int] = None):
25 - self.num_bits = num_bits
26 - self.pack_factor = 32 // self.num_bits
30 +
31 + self.pack_factor = 32 // num_bits
32 self.strategy = strategy
33
34 self.group_size: int
@@ -37,10 +42,16 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
42 else:
43 self.group_size = group_size
44
45 + if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
46 + raise ValueError(
47 + f"Unsupported num_bits = {num_bits}. "
48 + f"Supported num_bits = {WNA16_SUPPORTED_TYPES_MAP.keys()}")
49 +
50 + self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
51 +
52 # Verify supported on platform.
41 - verify_gptq_marlin_supported(num_bits=self.num_bits,
42 - group_size=self.group_size,
43 - is_sym=True)
53 + verify_marlin_supported(quant_type=self.quant_type,
54 + group_size=self.group_size)
55
56 @classmethod
57 def get_min_capability(cls) -> int:
@@ -150,7 +161,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
161 perm=layer.g_idx_sort_indices,
162 size_k=layer.input_size_per_partition,
163 size_n=layer.output_size_per_partition,
153 - num_bits=self.num_bits)
164 + num_bits=self.quant_type.size_bits)
165 replace_tensor(layer, "weight_packed", marlin_qweight)
166
167 # Permute scales from compressed-tensors format to marlin format.
@@ -172,7 +183,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
183 g_idx=layer.g_idx,
184 g_idx_sort_indices=layer.g_idx_sort_indices,
185 workspace=layer.workspace,
175 - num_bits=self.num_bits,
186 + wtype=self.quant_type,
187 output_size_per_partition=layer.output_size_per_partition,
188 input_size_per_partition=layer.input_size_per_partition,
189 is_k_full=True,
vllm/model_executor/layers/quantization/gptq_marlin.py
+27 -16
@@ -10,11 +10,12 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13 - apply_gptq_marlin_linear, check_gptq_marlin_supported, marlin_is_k_full,
13 + apply_gptq_marlin_linear, check_marlin_supported, marlin_is_k_full,
14 marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15 marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
16 - verify_gptq_marlin_supported, verify_marlin_supports_shape)
16 + verify_marlin_supported, verify_marlin_supports_shape)
17 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18 +from vllm.scalar_type import scalar_types
19
20 logger = init_logger(__name__)
21
@@ -22,6 +23,12 @@ logger = init_logger(__name__)
23 class GPTQMarlinConfig(QuantizationConfig):
24 """Config class for GPTQ Marlin"""
25
26 + # (num_bits, is_sym) -> quant_type
27 + TYPE_MAP = {
28 + (4, True): scalar_types.uint4b8,
29 + (8, True): scalar_types.uint8b128,
30 + }
31 +
32 def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
33 is_sym: bool, lm_head_quantized: bool) -> None:
34 if desc_act and group_size == -1:
@@ -29,20 +36,23 @@ class GPTQMarlinConfig(QuantizationConfig):
36 # (since we have only one group per output channel)
37 desc_act = False
38
32 - self.weight_bits = weight_bits
33 - self.pack_factor = 32 // self.weight_bits # packed into int32
39 + self.pack_factor = 32 // weight_bits # packed into int32
40 self.group_size = group_size
41 self.desc_act = desc_act
36 - self.is_sym = is_sym
42 self.lm_head_quantized = lm_head_quantized
43
44 + if (weight_bits, is_sym) not in self.TYPE_MAP:
45 + raise ValueError("Unsupported quantization config: "
46 + f"bits={weight_bits}, sym={is_sym}")
47 +
48 + self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
49 +
50 # Verify supported on platform.
40 - verify_gptq_marlin_supported(num_bits=self.weight_bits,
41 - group_size=self.group_size,
42 - is_sym=self.is_sym)
51 + verify_marlin_supported(quant_type=self.quant_type,
52 + group_size=self.group_size)
53
54 def __repr__(self) -> str:
45 - return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
55 + return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
56 f"group_size={self.group_size}, "
57 f"desc_act={self.desc_act}, "
58 f"lm_head_quantized={self.lm_head_quantized})")
@@ -122,11 +132,12 @@ class GPTQMarlinConfig(QuantizationConfig):
132 or desc_act is None):
133 return False
134
125 - return check_gptq_marlin_supported(
126 - num_bits=num_bits,
127 - group_size=group_size,
128 - is_sym=sym,
129 - min_capability=cls.get_min_capability())
135 + if (num_bits, sym) not in cls.TYPE_MAP:
136 + return False
137 +
138 + return check_marlin_supported(quant_type=cls.TYPE_MAP[(num_bits, sym)],
139 + group_size=group_size,
140 + min_capability=cls.get_min_capability())
141
142
143 class GPTQMarlinLinearMethod(LinearMethodBase):
@@ -293,7 +304,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
304 perm=layer.g_idx_sort_indices,
305 size_k=layer.input_size_per_partition,
306 size_n=layer.output_size_per_partition,
296 - num_bits=self.quant_config.weight_bits)
307 + num_bits=self.quant_config.quant_type.size_bits)
308 replace_tensor(layer, "qweight", marlin_qweight)
309
310 # Permute scales from autogptq format to marlin format.
@@ -319,7 +330,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
330 g_idx=layer.g_idx,
331 g_idx_sort_indices=layer.g_idx_sort_indices,
332 workspace=layer.workspace,
322 - num_bits=self.quant_config.weight_bits,
333 + wtype=self.quant_config.quant_type,
334 output_size_per_partition=layer.output_size_per_partition,
335 input_size_per_partition=layer.input_size_per_partition,
336 is_k_full=layer.is_k_full,
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+19 -10
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 from vllm.model_executor.utils import set_weight_attrs
12 +from vllm.scalar_type import scalar_types
13
14 logger = init_logger(__name__)
15
@@ -17,9 +18,10 @@ GPTQ_MARLIN_24_MIN_THREAD_N = 128
18 GPTQ_MARLIN_24_MIN_THREAD_K = 128
19 GPTQ_MARLIN_24_MAX_PARALLEL = 64
20
20 -GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
21 +GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES = [
22 + scalar_types.uint4b8, scalar_types.uint8b128
23 +]
24 GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
22 -GPTQ_MARLIN_24_SUPPORTED_SYM = [True]
25
26
27 class GPTQMarlin24Config(QuantizationConfig):
@@ -31,14 +33,19 @@ class GPTQMarlin24Config(QuantizationConfig):
33 weight_bits: int,
34 group_size: int,
35 ) -> None:
34 - self.weight_bits = weight_bits
36 + quant_type = {
37 + 4: scalar_types.uint4b8,
38 + 8: scalar_types.uint8b128,
39 + }.get(weight_bits)
40 +
41 self.group_size = group_size
42
43 # Verify
38 - if self.weight_bits not in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS:
44 + if quant_type is None or \
45 + quant_type not in GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES:
46 raise ValueError(
40 - f"Marlin_24 does not support weight_bits = {self.weight_bits}. "
41 - f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_NUM_BITS} "
47 + f"Marlin_24 does not support quant_type = {quant_type}. "
48 + f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_QUANT_TYPES} "
49 "are supported.")
50 if self.group_size not in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES:
51 raise ValueError(
@@ -46,8 +53,10 @@ class GPTQMarlin24Config(QuantizationConfig):
53 f"Only group_sizes = {GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES} "
54 "are supported.")
55
56 + self.quant_type = quant_type
57 +
58 # 4 Bits packed into 32 bit datatype.
50 - self.pack_factor = 32 // self.weight_bits
59 + self.pack_factor = 32 // self.quant_type.size_bits
60
61 # Tile size used by marlin kernels.
62 self.tile_size = 16
@@ -66,8 +75,8 @@ class GPTQMarlin24Config(QuantizationConfig):
75 self.perm_len = 1024
76
77 def __repr__(self) -> str:
69 - return "Marlin24Config(weight_bits={}, group_size={})".format(
70 - self.weight_bits, self.group_size)
78 + return "Marlin24Config(quant_type={}, group_size={})".format(
79 + self.quant_type, self.group_size)
80
81 @classmethod
82 def get_name(cls) -> str:
@@ -279,7 +288,7 @@ class GPTQMarlin24LinearMethod(LinearMethodBase):
288
289 output_2d = ops.gptq_marlin_24_gemm(x_2d, qweight, meta, scales,
290 workspace,
282 - self.quant_config.weight_bits,
291 + self.quant_config.quant_type,
292 size_m, size_n, size_k)
293
294 output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+57 -63
@@ -5,6 +5,7 @@ import torch
5
6 from vllm import _custom_ops as ops
7 from vllm.platforms import current_platform
8 +from vllm.scalar_type import ScalarType, scalar_types
9
10 from .quant_utils import pack_cols, unpack_cols
11
@@ -13,7 +14,6 @@ GPTQ_MARLIN_MIN_THREAD_N = 64
14 GPTQ_MARLIN_MIN_THREAD_K = 128
15 GPTQ_MARLIN_MAX_PARALLEL = 16
16
16 -MARLIN_SUPPORTED_NUM_BITS = [4, 8]
17 MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
18
19 # In case there is a performance issue with Marlin, the variable below can be
@@ -22,76 +22,70 @@ MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
22 USE_FP32_REDUCE_DEFAULT = True
23
24
25 -def _check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
26 - min_capability: Optional[int],
27 - has_zp: bool) -> Tuple[bool, Optional[str]]:
28 - if min_capability is not None:
25 +# For binary size and compile time, we don't support the same types for with and
26 +# without runtime zero-point. We support common cases, i.e. AWQ and GPTQ.
27 +# TODO: we may want to move this into the C++ so its closer to the actual impl
28 +def query_marlin_supported_quant_types(has_zp: bool,
29 + min_capability: Optional[int] = None):
30 + if min_capability is None:
31 major, minor = current_platform.get_device_capability()
30 - device_capability = major * 10 + minor
31 - if device_capability < min_capability:
32 - return (False, "Marlin does not support device_capability = {}"
33 - ", the min_capability required is {}".format(
34 - device_capability, min_capability))
35 -
36 - if num_bits not in MARLIN_SUPPORTED_NUM_BITS:
37 - return (False, "Marlin does not support weight_bits = {}. "
38 - "Only weight_bits = {} are supported.".format(
39 - num_bits, MARLIN_SUPPORTED_NUM_BITS))
40 -
41 - if group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
42 - return (False, "Marlin does not support group_size = {}. Only "
43 - "group_sizes = {} are supported.".format(
44 - group_size, MARLIN_SUPPORTED_GROUP_SIZES))
45 -
46 - if not has_zp and not is_sym:
47 - return (False,
48 - "Marlin without zero_points must have symmetric quantization")
32 + min_capability = major * 10 + minor
33
50 - return True, None
34 + if min_capability < 80:
35 + return []
36
37 + if has_zp:
38 + # AWQ style, unsigned + runtime zero-point
39 + return [scalar_types.uint4, scalar_types.uint8]
40 + else:
41 + # GPTQ style, unsigned + symmetric bias
42 + # TODO: once fp8_marlin is merged into "gptq_marlin" we should be able
43 + # to add `scalar_types.float8_e4m3fn` here
44 + return [scalar_types.uint4b8, scalar_types.uint8b128]
45
53 -def check_gptq_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
54 - min_capability: int) -> bool:
55 - cond, _ = _check_marlin_supported(num_bits,
56 - group_size,
57 - is_sym,
58 - min_capability,
59 - has_zp=False)
60 - return cond
46
47 +def _check_marlin_supported(
48 + quant_type: ScalarType,
49 + group_size: Optional[int],
50 + has_zp: bool,
51 + min_capability: Optional[int] = None) -> Tuple[bool, Optional[str]]:
52
63 -def check_awq_marlin_supported(num_bits: int, group_size: int, has_zp: bool,
64 - min_capability: int) -> bool:
65 - cond, _ = _check_marlin_supported(num_bits,
66 - group_size,
67 - False,
68 - min_capability,
69 - has_zp=has_zp)
70 - return cond
53 + if min_capability is None:
54 + major, minor = current_platform.get_device_capability()
55 + min_capability = major * 10 + minor
56
57 + supported_types = query_marlin_supported_quant_types(
58 + has_zp, min_capability)
59
73 -def verify_gptq_marlin_supported(num_bits: int, group_size: int,
74 - is_sym: bool) -> None:
75 - cond, err_msg = _check_marlin_supported(num_bits,
76 - group_size,
77 - is_sym,
78 - min_capability=None,
79 - has_zp=False)
80 - if not cond:
81 - assert err_msg is not None
82 - raise ValueError("GPTQ" + err_msg)
60 + if quant_type not in supported_types:
61 + return (False, f"Marlin does not support weight_bits = {quant_type}. "
62 + f"Only types = {supported_types} "
63 + f"are supported (for group_size = {group_size}, "
64 + f"min_capability = {min_capability}, zp = {has_zp}).")
65 + if (group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES):
66 + return (False, f"Marlin does not support group_size = {group_size}. "
67 + f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} "
68 + "are supported.")
69 +
70 + return True, None
71 +
72 +
73 +def check_marlin_supported(quant_type: ScalarType,
74 + group_size: int,
75 + has_zp: bool = False,
76 + min_capability: Optional[int] = None) -> bool:
77 + cond, _ = _check_marlin_supported(quant_type, group_size, has_zp,
78 + min_capability)
79 + return cond
80
81
85 -def verify_awq_marlin_supported(num_bits: int, group_size: int,
86 - has_zp: bool) -> None:
87 - cond, err_msg = _check_marlin_supported(num_bits,
88 - group_size,
89 - False,
90 - min_capability=None,
91 - has_zp=has_zp)
82 +def verify_marlin_supported(quant_type: ScalarType,
83 + group_size: int,
84 + has_zp: bool = False) -> None:
85 + cond, err_msg = _check_marlin_supported(quant_type, group_size, has_zp)
86 if not cond:
87 assert err_msg is not None
94 - raise ValueError("AWQ" + err_msg)
88 + raise ValueError(err_msg)
89
90
91 def verify_marlin_supports_shape(output_size_per_partition: int,
@@ -245,7 +239,7 @@ def apply_gptq_marlin_linear(
239 g_idx: torch.Tensor,
240 g_idx_sort_indices: torch.Tensor,
241 workspace: torch.Tensor,
248 - num_bits: int,
242 + wtype: ScalarType,
243 output_size_per_partition: int,
244 input_size_per_partition: int,
245 is_k_full: bool,
@@ -261,7 +255,7 @@ def apply_gptq_marlin_linear(
255 g_idx,
256 g_idx_sort_indices,
257 workspace,
264 - num_bits,
258 + wtype,
259 size_m=reshaped_x.shape[0],
260 size_n=output_size_per_partition,
261 size_k=input_size_per_partition,
@@ -283,7 +277,7 @@ def apply_awq_marlin_linear(
277 g_idx: torch.Tensor,
278 g_idx_sort_indices: torch.Tensor,
279 workspace: torch.Tensor,
286 - num_bits: int,
280 + quant_type: ScalarType,
281 output_size_per_partition: int,
282 input_size_per_partition: int,
283 bias: Optional[torch.Tensor] = None,
@@ -298,7 +292,7 @@ def apply_awq_marlin_linear(
292 g_idx,
293 g_idx_sort_indices,
294 workspace,
301 - num_bits,
295 + quant_type,
296 size_m=reshaped_x.shape[0],
297 size_n=output_size_per_partition,
298 size_k=input_size_per_partition,
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
+19 -10
@@ -5,10 +5,12 @@ from typing import List
5 import numpy as np
6 import torch
7
8 +from vllm.scalar_type import ScalarType
9 +
10 from .marlin_utils import (GPTQ_MARLIN_TILE, marlin_permute_scales,
11 marlin_zero_points)
10 -from .quant_utils import (get_pack_factor, quantize_weights,
11 - quantize_weights_with_zp, sort_weights)
12 +from .quant_utils import (get_pack_factor, gptq_quantize_weights,
13 + quantize_weights, sort_weights)
14
15
16 class MarlinWorkspace:
@@ -90,9 +92,10 @@ def get_weight_perm(num_bits: int):
92 return perm
93
94
93 -def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
95 +def marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int,
96 act_order: bool):
97 size_k, size_n = w.shape
98 + num_bits = quant_type.size_bits
99
100 # Normalize group_size
101 if group_size == -1:
@@ -100,8 +103,8 @@ def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
103 assert group_size <= size_k
104
105 # Quantize (and apply act_order if provided)
103 - w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
104 - act_order)
106 + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
107 + w, quant_type, group_size, act_order)
108
109 # For act_order, sort the "weights" and "g_idx" so that group ids are
110 # increasing
@@ -122,7 +125,8 @@ def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
125 return res_list
126
127
125 -def awq_marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int):
128 +def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType,
129 + group_size: int):
130 size_k, size_n = w.shape
131
132 # Normalize group_size
@@ -135,13 +139,18 @@ def awq_marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int):
139 num_groups = size_k // group_size
140
141 # Quantize with zp
138 - w_ref, q_w, s, zp = quantize_weights_with_zp(w, num_bits, group_size)
142 + w_ref, q_w, s, zp = quantize_weights(w,
143 + quant_type,
144 + group_size,
145 + zero_points=True)
146
147 # Reformat to marlin
141 - weight_perm = get_weight_perm(num_bits)
142 - marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
148 + weight_perm = get_weight_perm(quant_type.size_bits)
149 + marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
150 + weight_perm)
151 marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
144 - marlin_zp = marlin_zero_points(zp, num_groups, size_n, num_bits)
152 + marlin_zp = marlin_zero_points(zp, num_groups, size_n,
153 + quant_type.size_bits)
154
155 # Create result
156 res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py
+14 -16
@@ -6,8 +6,10 @@ from typing import List
6 import numpy
7 import torch
8
9 +from vllm.scalar_type import ScalarType
10 +
11 from .marlin_utils_test import marlin_weights
10 -from .quant_utils import quantize_weights
12 +from .quant_utils import gptq_quantize_weights
13
14
15 # This is PyTorch implementation of main part of reorder_meta()
@@ -348,13 +350,11 @@ def check_24(w, num_rows_to_sample=50, _verbose=False):
350 print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
351
352
351 -def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
353 +def compress_quantized_24_weight(q_24, size_k, size_n, wtype: ScalarType):
354 assert q_24.shape == (size_k, size_n)
355
354 - # Remove zp to normalize over 0
355 - max_q_val = (1 << num_bits) - 1
356 - zp = (max_q_val + 1) // 2
357 - q_24_no_zp = q_24 - zp
356 + # Remove bias to normalize over 0
357 + q_24_no_zp = q_24 - wtype.bias
358
359 # Compress
360 q_24_no_zp = q_24_no_zp.t().contiguous()
@@ -362,8 +362,8 @@ def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
362 q_24_no_zp)
363 q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
364
365 - # Restore zp
366 - q_24_comp = q_24_no_zp_comp + zp
365 + # Restore bias
366 + q_24_comp = q_24_no_zp_comp + wtype.bias
367
368 # Resize meta to its actual shape (without moving any data)
369 meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
@@ -427,7 +427,7 @@ def marlin_permute_scales_24(s: torch.Tensor, size_k: int, size_n: int,
427
428 def marlin_24_quantize(
429 w: torch.Tensor,
430 - num_bits: int,
430 + quant_type: ScalarType,
431 group_size: int,
432 ):
433 size_k, size_n = w.shape
@@ -441,20 +441,18 @@ def marlin_24_quantize(
441 w_24, mask_24 = inject_24(w, size_k, size_n)
442
443 # Quantize
444 - w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
445 - num_bits,
446 - group_size,
447 - act_order=False)
444 + w_24_ref, q_w_24, s, g_idx, rand_perm = gptq_quantize_weights(
445 + w_24, quant_type, group_size, act_order=False)
446
447 # Compress quantized weight
448 q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
451 - num_bits)
449 + quant_type)
450 size_k_comp = size_k // 2
451
452 # Reformat to marlin
455 - weight_perm = get_weight_perm_24(num_bits)
453 + weight_perm = get_weight_perm_24(quant_type.size_bits)
454 marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
457 - num_bits, weight_perm)
455 + quant_type.size_bits, weight_perm)
456 marlin_24_s = marlin_permute_scales_24(s, size_k, size_n, group_size)
457
458 # Create result
vllm/model_executor/layers/quantization/utils/quant_utils.py
+64 -84
@@ -4,7 +4,11 @@ from typing import List
4 import numpy
5 import torch
6
7 -SUPPORTED_NUM_BITS = [4, 8]
7 +from vllm.model_executor.layers.quantization.qqq import (
8 + MARLIN_QQQ_SUPPORTED_NUM_BITS)
9 +from vllm.scalar_type import ScalarType, scalar_types
10 +
11 +SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128]
12 SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
13
14 # Note: this is a hack. We should update each model to register the
@@ -45,7 +49,7 @@ def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
49
50
51 def get_pack_factor(num_bits):
48 - assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
52 + assert 32 % num_bits == 0, f"Unsupported num_bits = {num_bits}"
53 return 32 // num_bits
54
55
@@ -74,24 +78,23 @@ def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
78 )
79
80
77 -def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
78 - act_order: bool):
81 +def quantize_weights(w: torch.Tensor,
82 + quant_type: ScalarType,
83 + group_size: int,
84 + zero_points: bool = False):
85 + assert quant_type.is_integer(), \
86 + "Floating point quantization may work but has not been tested"
87 +
88 orig_device = w.device
89 + orig_type = w.dtype
90 size_k, size_n = w.shape
91
92 assert w.is_floating_point(), "w must be float"
83 - assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
84 - assert group_size in SUPPORTED_GROUP_SIZES + [
85 - size_k
86 - ], f"Unsupported groupsize = {group_size}"
93
94 if group_size == -1:
95 group_size = size_k
96 assert group_size <= size_k
97
92 - max_q_val = 2**num_bits - 1
93 - half_q_val = (max_q_val + 1) // 2
94 -
98 # Reshape to [groupsize, -1]
99 if group_size < size_k:
100 w = w.reshape((-1, group_size, size_n))
@@ -99,16 +102,34 @@ def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
102 w = w.reshape((group_size, -1))
103
104 # Compute scale for each group
102 - s = torch.max(torch.abs(w), 0, keepdim=True)[0]
103 - s *= 2 / max_q_val # 2 => symmetric
105 + max_val = torch.max(w, 0, keepdim=True).values
106 + min_val = torch.min(w, 0, keepdim=True).values
107 +
108 + max_q_val = quant_type.max()
109 + min_q_val = quant_type.min()
110 +
111 + if zero_points:
112 + assert not quant_type.is_signed() and quant_type.max() > 0
113 + w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
114 + maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
115 + .clamp(min_q_val, max_q_val).int()
116 + else:
117 + # If the bias is such that there are no possible negative/positive
118 + # values, set the max value to inf to avoid divide by 0
119 + w_s = torch.max(
120 + abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
121 + abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
122 + maybe_w_zp = None
123
124 # Quantize
106 - q_w = torch.round(w / s).int()
107 - q_w += half_q_val
108 - q_w = torch.clamp(q_w, 0, max_q_val)
125 + w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0)
126 + w_q = torch.clamp(w_q, min_q_val, max_q_val)
127
128 # Compute ref (dequantized)
111 - w_ref = (q_w - half_q_val).half() * s
129 + w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
130 +
131 + if quant_type.has_bias():
132 + w_q += quant_type.bias
133
134 # Restore original shapes
135 if group_size < size_k:
@@ -119,90 +140,48 @@ def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
140 w = w.reshape((size_k, size_n)).contiguous()
141 return w
142
122 - q_w = reshape_w(q_w)
143 + w_q = reshape_w(w_q)
144 w_ref = reshape_w(w_ref)
145
125 - s = s.reshape((-1, size_n)).contiguous()
146 + w_s = w_s.reshape((-1, size_n)).contiguous()
147
127 - # Apply act_order
128 - g_idx = torch.empty(0, dtype=torch.int, device=w.device)
129 - rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
130 - if act_order:
131 - assert (
132 - group_size < size_k
133 - ), "For act_order, groupsize = {} must be less than size_k = {}".format(
134 - group_size, size_k)
135 -
136 - w_ref, q_w, g_idx, rand_perm = permute_rows(q_w, w_ref, group_size)
148 + if zero_points:
149 + maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous()
150 + maybe_w_zp = maybe_w_zp.to(device=orig_device)
151
152 return (
153 w_ref.to(device=orig_device),
140 - q_w.to(device=orig_device),
141 - s.to(device=orig_device),
142 - g_idx.to(device=orig_device),
143 - rand_perm.to(device=orig_device),
154 + w_q.to(device=orig_device),
155 + w_s.to(device=orig_device),
156 + maybe_w_zp,
157 )
158
159
147 -def quantize_weights_with_zp(w: torch.Tensor, num_bits: int, group_size: int):
148 - orig_device = w.device
149 - size_k, size_n = w.shape
160 +def gptq_quantize_weights(w: torch.Tensor, quant_type: ScalarType,
161 + group_size: int, act_order: bool):
162 + size_k, _ = w.shape
163
164 assert w.is_floating_point(), "w must be float"
152 - assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
165 + assert quant_type in SUPPORTED_GPTQ_QUANT_TYPES, \
166 + f"Unsupported gptq type = {quant_type}"
167 assert group_size in SUPPORTED_GROUP_SIZES + [
168 size_k
169 ], f"Unsupported groupsize = {group_size}"
170
157 - if group_size == -1:
158 - group_size = size_k
159 - assert group_size <= size_k
160 -
161 - max_q_val = 2**num_bits - 1
162 - min_q_val = 0
171 + w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
172
164 - # Reshape to [groupsize, -1]
165 - if group_size < size_k:
166 - w = w.reshape((-1, group_size, size_n))
167 - w = w.permute(1, 0, 2)
168 - w = w.reshape((group_size, -1))
169 -
170 - # Compute scale for each group
171 - max = torch.max(w, 0, keepdim=True)[0]
172 - min = torch.min(w, 0, keepdim=True)[0]
173 - s = (max - min).clamp(min=1e-5) / max_q_val
174 -
175 - # Compute zero-point for each group
176 - zp = (-torch.round(min / s)).clamp(min_q_val, max_q_val).int()
177 -
178 - # Quantize
179 - q_w = torch.round(w / s).int() + zp
180 - q_w = torch.clamp(q_w, min_q_val, max_q_val)
181 -
182 - # Compute ref (dequantized)
183 - w_ref = (q_w - zp).half() * s
184 -
185 - # Restore original shapes
186 - if group_size < size_k:
187 -
188 - def reshape_w(w):
189 - w = w.reshape((group_size, -1, size_n))
190 - w = w.permute(1, 0, 2)
191 - w = w.reshape((size_k, size_n)).contiguous()
192 - return w
193 -
194 - q_w = reshape_w(q_w)
195 - w_ref = reshape_w(w_ref)
173 + # Apply act_order
174 + g_idx = torch.empty(0, dtype=torch.int, device=w.device)
175 + rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
176 + if act_order:
177 + assert (
178 + group_size < size_k
179 + ), "For act_order, groupsize = {} must be less than size_k = {}".format(
180 + group_size, size_k)
181
197 - s = s.reshape((-1, size_n)).contiguous()
198 - zp = zp.reshape((-1, size_n)).contiguous()
182 + w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size)
183
200 - return (
201 - w_ref.to(device=orig_device),
202 - q_w.to(device=orig_device),
203 - s.to(device=orig_device),
204 - zp.to(device=orig_device),
205 - )
184 + return w_ref, w_q, w_s, g_idx, rand_perm
185
186
187 # QQQ employs different quant schemes for per-group and
@@ -212,7 +191,8 @@ def qqq_quantize_weights(w: torch.Tensor, num_bits: int, group_size: int):
191 size_k, size_n = w.shape
192
193 assert w.is_floating_point(), "w must be float"
215 - assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
194 + assert num_bits in MARLIN_QQQ_SUPPORTED_NUM_BITS, \
195 + f"Unsupported num_bits = {num_bits}"
196 assert group_size in SUPPORTED_GROUP_SIZES + [
197 size_k
198 ], f"Unsupported groupsize = {group_size}"
vllm/scalar_type.py new
+35
@@ -0,0 +1,35 @@
1 +from ._core_ext import NanRepr, ScalarType
2 +
3 +# naming generally follows: https://github.com/jax-ml/ml_dtypes
4 +# for floating point types (leading f) the scheme is:
5 +# `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
6 +# flags:
7 +# - no-flags: means it follows IEEE 754 conventions
8 +# - f: means finite values only (no infinities)
9 +# - n: means nans are supported (non-standard encoding)
10 +# for integer types the scheme is:
11 +# `[u]int<size_bits>[b<bias>]`
12 +# - if bias is not present it means its zero
13 +
14 +
15 +class scalar_types:
16 + int4 = ScalarType.int_(4, None)
17 + uint4 = ScalarType.uint(4, None)
18 + int8 = ScalarType.int_(8, None)
19 + uint8 = ScalarType.uint(8, None)
20 + float8_e4m3fn = ScalarType.float_(4, 3, True,
21 + NanRepr.EXTD_RANGE_MAX_MIN.value)
22 + float8_e5m2 = ScalarType.float_IEEE754(5, 2)
23 + float16_e8m7 = ScalarType.float_IEEE754(8, 7)
24 + float16_e5m10 = ScalarType.float_IEEE754(5, 10)
25 +
26 + # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
27 + float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE.value)
28 +
29 + # "gptq" types
30 + uint4b8 = ScalarType.uint(4, 8)
31 + uint8b128 = ScalarType.uint(8, 128)
32 +
33 + # colloquial names
34 + bfloat16 = float16_e8m7
35 + float16 = float16_e5m10