@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27ce85476e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Add marlin_24 unit tests (#4901)

Alexander Matveev committed May 19, 2024 at 11:37 UTC 27ce85476e6b170c5c90c65ac5c3268911135766
6 files changed +649 -103
tests/kernels/test_marlin_gemm.py
+74 -13
@@ -7,23 +7,32 @@ import torch
7
8 from vllm import _custom_ops as ops
9 from vllm.model_executor.layers.quantization.gptq_marlin import (
10 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
11 GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
12 +from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
13 + GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
14 + GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
15 +from vllm.model_executor.layers.quantization.utils.marlin_perms import (
16 + marlin_perm)
17 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 - MarlinWorkspace, is_marlin_supported, marlin_quantize, marlin_weights)
18 + MarlinWorkspace, compute_max_diff, is_marlin_supported, marlin_24_quantize,
19 + marlin_quantize, marlin_weights)
20 from vllm.model_executor.layers.quantization.utils.quant_utils import (
21 gptq_pack, quantize_weights, sort_weights)
22
23 ACT_ORDER_OPTS = [False, True]
24 K_FULL_OPTS = [False, True]
25
19 -K_CHUNKS = [128, 256]
20 -N_CHUNKS = [64, 128, 256]
26 +MARLIN_K_CHUNKS = [128]
27 +MARLIN_N_CHUNKS = [64, 128, 256]
28 +
29 +MARLIN_24_K_CHUNKS = [128]
30 +MARLIN_24_N_CHUNKS = [256]
31
32 MNK_FACTORS = [
33 (1, 1, 1),
34 (1, 4, 8),
35 (1, 7, 5),
26 - (1, 7 * 4, 5 * 1),
36 (13, 17, 67),
37 (26, 37, 13),
38 (67, 13, 11),
@@ -31,14 +40,13 @@ MNK_FACTORS = [
40
41
42 def rand_data(shape):
34 - data = torch.rand(shape).to(torch.half).cuda()
35 - return data
43 + return torch.randn(shape, dtype=torch.half, device="cuda")
44
45
46 @pytest.mark.skipif(not is_marlin_supported(),
47 reason="Marlin is not supported on this GPU type.")
40 -@pytest.mark.parametrize("k_chunk", K_CHUNKS)
41 -@pytest.mark.parametrize("n_chunk", N_CHUNKS)
48 +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
49 +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
50 @pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
51 @pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
52 @pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
@@ -82,7 +90,8 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
90 q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
91
92 # Pack to Marlin format
85 - marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits)
93 + marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits,
94 + marlin_perm[num_bits])
95
96 # Run Marlin repack GPU kernel
97 marlin_q_w_2 = ops.gptq_marlin_repack(
@@ -99,8 +108,8 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
108
109 @pytest.mark.skipif(not is_marlin_supported(),
110 reason="Marlin is not supported on this GPU type.")
102 -@pytest.mark.parametrize("k_chunk", K_CHUNKS)
103 -@pytest.mark.parametrize("n_chunk", N_CHUNKS)
111 +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
112 +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
113 @pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
114 @pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
115 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
@@ -136,7 +145,8 @@ def test_marlin_gemm(
145 w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
146 b_weight, num_bits, group_size, act_order)
147
139 - workspace = MarlinWorkspace(size_n)
148 + workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
149 + GPTQ_MARLIN_MAX_PARALLEL)
150
151 output = ops.gptq_marlin_gemm(
152 a_input,
@@ -155,4 +165,55 @@ def test_marlin_gemm(
165
166 torch.cuda.synchronize()
167
158 - assert torch.allclose(output, output_ref, rtol=1e-2)
168 + max_diff = compute_max_diff(output, output_ref)
169 + print("max_diff = {}".format(max_diff))
170 +
171 + assert max_diff < 0.04
172 +
173 +
174 +@pytest.mark.skipif(not is_marlin_supported(),
175 + reason="Marlin is not supported on this GPU type.")
176 +@pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
177 +@pytest.mark.parametrize("n_chunk", MARLIN_24_N_CHUNKS)
178 +@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
179 +@pytest.mark.parametrize("group_size", GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES)
180 +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
181 +def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
182 + m_factor, n_factor, k_factor = mnk_factors
183 +
184 + size_m = m_factor
185 + size_k = k_chunk * k_factor
186 + size_n = n_chunk * n_factor
187 +
188 + print(f"MNK = {size_m} {size_n} {size_k}")
189 + print(f"groupsize = {group_size}")
190 +
191 + a_input = rand_data((size_m, size_k))
192 + b_weight = rand_data((size_k, size_n))
193 +
194 + (w_24_ref, marlin_24_q_w_comp, marlin_24_meta,
195 + marlin_24_s) = marlin_24_quantize(b_weight, num_bits, group_size)
196 +
197 + workspace_24 = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
198 + GPTQ_MARLIN_24_MAX_PARALLEL)
199 +
200 + output_ref = torch.matmul(a_input, w_24_ref)
201 +
202 + output = ops.gptq_marlin_24_gemm(
203 + a_input,
204 + marlin_24_q_w_comp,
205 + marlin_24_meta,
206 + marlin_24_s,
207 + workspace_24.scratch,
208 + num_bits,
209 + a_input.shape[0],
210 + b_weight.shape[1],
211 + a_input.shape[1],
212 + )
213 +
214 + torch.cuda.synchronize()
215 +
216 + max_diff = compute_max_diff(output, output_ref)
217 + print("max_diff = {}".format(max_diff))
218 +
219 + assert max_diff < 0.04
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+19 -8
@@ -12,6 +12,15 @@ from vllm.model_executor.utils import set_weight_attrs
12
13 logger = init_logger(__name__)
14
15 +GPTQ_MARLIN_24_TILE = 16
16 +GPTQ_MARLIN_24_MIN_THREAD_N = 128
17 +GPTQ_MARLIN_24_MIN_THREAD_K = 128
18 +GPTQ_MARLIN_24_MAX_PARALLEL = 16
19 +
20 +GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
21 +GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
22 +GPTQ_MARLIN_24_SUPPORTED_SYM = [True]
23 +
24
25 class GPTQMarlin24Config(QuantizationConfig):
26 """Config class for Marlin24.
@@ -25,15 +34,17 @@ class GPTQMarlin24Config(QuantizationConfig):
34 self.weight_bits = weight_bits
35 self.group_size = group_size
36
28 - if self.weight_bits != 4 and self.weight_bits != 8:
29 - raise ValueError("weight_bits must be 4 or 8. Got = {}".format(
30 - self.weight_bits))
31 -
32 - if self.group_size != 128 and self.group_size != -1:
37 + # Verify
38 + if self.weight_bits not in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS:
39 + raise ValueError(
40 + f"Marlin_24 does not support weight_bits = {self.weight_bits}. "
41 + f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_NUM_BITS} "
42 + "are supported.")
43 + if self.group_size not in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES:
44 raise ValueError(
34 - "Currently, only group size 128 and -1 (channelwise) "
35 - "is supported for Marlin24, but got group_size of "
36 - f"{self.group_size}")
45 + f"Marlin_24 does not support group_size = {self.group_size}. "
46 + f"Only group_sizes = {GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES} "
47 + "are supported.")
48
49 # 4 Bits packed into 32 bit datatype.
50 self.pack_factor = 32 // self.weight_bits
vllm/model_executor/layers/quantization/utils/format_24.py new
+308
@@ -0,0 +1,308 @@
1 +#
2 +# Modified by Roberto Lopez Castro (roberto.lopez.castro@udc.es).
3 +#
4 +
5 +import torch
6 +
7 +
8 +# This is PyTorch implementation of main part of reorder_meta()
9 +# function, from tools/util/include/cutlass/util/host_reorder.h file
10 +# of CUTLASS source tree. Furthermore, CUTLASS template for sparse
11 +# GEMM decides upon layout of this matrix, and at the moment for the
12 +# sparse GEMM executed on tensor cores, this is layout described by
13 +# ColumnMajorInterleaved<2> data structure, in
14 +# include/cutlass/layout/matrix.h of CUTLASS source tree. The
15 +# reordering of meta matrix into meta_reordered matrix calculated
16 +# according to these segments of CUTLASS code is re-implemented here.
17 +# Note that this calculation produces offsets for scattering metadata
18 +# matrix elements into reordered metadata matrix elements (or,
19 +# equivalently, for gathering reordered metadata matrix element back
20 +# into metadata matrix elements).
21 +def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype,
22 + device):
23 + dst_rows = torch.arange(0, m, device=device)[:, None].repeat(1, meta_ncols)
24 + dst_cols = torch.arange(0, meta_ncols, device=device).repeat(m, 1)
25 +
26 + # Reorder the rows, then swizzle the 2x2 blocks.
27 + group_x = 64
28 + group_y = 32 if meta_dtype.itemsize == 2 else 16
29 +
30 + dst_rows = (dst_rows // group_x * group_x + (dst_rows % 2) * 2 +
31 + (dst_rows % 8) // 4 + ((dst_rows % group_y) % 4) // 2 * 32 +
32 + ((dst_rows % group_x) // 8) * 4)
33 +
34 + topright = ((dst_rows % 2 == 0) & (dst_cols % 2 == 1)).to(torch.int8)
35 + bottomleft = ((dst_rows % 2 == 1) & (dst_cols % 2 == 0)).to(torch.int8)
36 + dst_rows += topright - bottomleft
37 + dst_cols -= topright - bottomleft
38 +
39 + # Assumed that meta tensor is to be stored in CUTLASS
40 + # InterleavedColumnMajor layout, and reverse engineered
41 + # corresponding code to store values into this tensor.
42 + interleave = 2
43 + cols_maj = dst_cols // interleave
44 + cols_min = dst_cols % interleave
45 + return (cols_maj * m * interleave + dst_rows * interleave +
46 + cols_min).view(-1)
47 +
48 +
49 +# This function converts dense matrix into sparse semi-structured
50 +# representation, producing "compressed" matrix, in the layout used by
51 +# CUTLASS backend, and corresponding metadata matrix.
52 +def sparse_semi_structured_from_dense_cutlass(dense):
53 + if dense.dim() != 2:
54 + raise RuntimeError(
55 + f"Expected 2-dimensional dense tensor, got {dense.dim()}-dimensional tensor" # noqa: E501
56 + )
57 +
58 + m, k = dense.shape
59 + device = dense.device
60 +
61 + meta_dtype = torch.int8
62 + if dense.dtype == torch.int8:
63 + meta_dtype = torch.int32
64 + elif dense.dtype in [torch.half, torch.bfloat16, torch.float, torch.int32]:
65 + meta_dtype = torch.int16
66 + else:
67 + raise RuntimeError(f"Invalid datatype {dense.dtype} of dense matrix")
68 + quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4
69 + if quadbits_per_meta_elem not in (4, 8):
70 + raise RuntimeError(
71 + "Invalid number of elements per meta element calculated")
72 +
73 + if meta_dtype == torch.int32:
74 + if m % 16 != 0:
75 + raise RuntimeError(
76 + f"Number of rows of dense matrix {m} must be divisible by 16")
77 + else:
78 + if m % 32 != 0:
79 + raise RuntimeError(
80 + f"Number of rows of dense matrix {m} must be divisible by 32")
81 + if k % (4 * quadbits_per_meta_elem) != 0:
82 + raise RuntimeError(
83 + f"Number of columns of dense matrix {k} must be divisible by {4 * quadbits_per_meta_elem}" # noqa: E501
84 + )
85 +
86 + if dense.dtype != torch.float:
87 + ksparse = 4
88 + dense_4 = dense.view(-1, k // ksparse, ksparse)
89 + m0, m1, m2, m3 = (dense_4 != 0).unbind(-1)
90 + else:
91 + ksparse = 2
92 + dense_2 = dense.view(-1, k // ksparse, ksparse)
93 + m0, m2 = m1, m3 = (dense_2 != 0).unbind(-1)
94 + meta_ncols = k // (ksparse * quadbits_per_meta_elem)
95 +
96 + # Encoding quadruples of True/False values as follows:
97 + # [True, True, False, False] -> 0b0100
98 + # [True, False, True, False] -> 0b1000
99 + # [False, True, True, False] -> 0b1001
100 + # [True, False, False, True ] -> 0b1100
101 + # [False, True, False, True ] -> 0b1101
102 + # [False, False, True, True ] -> 0b1110
103 + # Thus, lower two bits in the encoding are index of the True value
104 + # at the lowest index in the quadruple, and the higher two bits in
105 + # the encoding are index of the other True value in the quadruple.
106 + # In case there are less than two True values, than False value or
107 + # values at some index or indices are considered True for the
108 + # encoding. In case there are more than two True values, then the
109 + # excess True value(s) at some indices are considered False for
110 + # the encoding. The exact encodings used for these cases are as
111 + # follows:
112 + # [False, False, False, False] -> 0b1110
113 + # [False, False, False, True ] -> 0b1110
114 + # [False, False, True, False] -> 0b1110
115 + # [False, True, False, False] -> 0b1001
116 + # [False, True, True, True ] -> 0b1101
117 + # [True, False, False, False] -> 0b1000
118 + # [True, False, True, True ] -> 0b1100
119 + # [True, True, False, True ] -> 0b0100
120 + # [True, True, True, False] -> 0b0100
121 + # [True, True, True, True ] -> 0b0100
122 + # These particular encodings are chosen, with the help of Espresso
123 + # logic minimizer software, for the purpose of minimization of
124 + # corresponding Boolean functions, that translate non-zero flags
125 + # into encoding bits. Note also possible choices for the first
126 + # and last of these encodings were limited only to (0b0100,
127 + # 0b1110), in order to produce valid encodings for 1:2 sparsity
128 + # case.
129 +
130 + expr0 = m0 & m1
131 + expr1 = ~m0 & m1
132 + expr2 = ~m0 & ~m1
133 + bit0 = expr1
134 + bit1 = expr2
135 + bit2 = expr0 | expr2 | m3
136 + bit3 = expr1 | ~m1
137 + idxs0 = bit0 | (bit1.to(torch.int64) << 1)
138 + idxs1 = bit2 | (bit3.to(torch.int64) << 1)
139 +
140 + if dense.dtype != torch.float:
141 + sparse0 = dense_4.gather(
142 + -1, idxs0.unsqueeze(-1)) # type: ignore[possibly-undefined]
143 + sparse1 = dense_4.gather(-1, idxs1.unsqueeze(-1))
144 + sparse = torch.stack((sparse0, sparse1), dim=-1).view(m, k // 2)
145 + else:
146 + sparse = dense_2.gather(-1,
147 + idxs0.unsqueeze(-1) // 2).view(
148 + m,
149 + k // 2) # type: ignore[possibly-undefined]
150 +
151 + meta_4 = idxs0 | (idxs1 << 2)
152 + meta_n = meta_4.view(
153 + (-1, meta_ncols, quadbits_per_meta_elem)).to(meta_dtype)
154 +
155 + if quadbits_per_meta_elem == 4:
156 + meta = (meta_n[:, :, 0]
157 + | (meta_n[:, :, 1] << 4)
158 + | (meta_n[:, :, 2] << 8)
159 + | (meta_n[:, :, 3] << 12))
160 + elif quadbits_per_meta_elem == 8:
161 + meta = (meta_n[:, :, 0]
162 + | (meta_n[:, :, 1] << 4)
163 + | (meta_n[:, :, 2] << 8)
164 + | (meta_n[:, :, 3] << 12)
165 + | (meta_n[:, :, 4] << 16)
166 + | (meta_n[:, :, 5] << 20)
167 + | (meta_n[:, :, 6] << 24)
168 + | (meta_n[:, :, 7] << 28))
169 +
170 + # Reorder meta tensor elements.
171 + meta_reordered = meta.new_empty(
172 + (m * meta_ncols, )) # type: ignore[possibly-undefined]
173 + meta_offsets = _calculate_meta_reordering_scatter_offsets(
174 + m, meta_ncols, meta_dtype, device)
175 + meta_reordered.scatter_(0, meta_offsets, meta.view(-1))
176 +
177 + return (sparse, meta_reordered.view(m, meta_ncols))
178 +
179 +
180 +# This function performs reverse of the function above - it
181 +# reconstructs dense matrix from a pair of "compressed" matrix, given
182 +# in the layout used by CUTLASS backend, and accompanying metadata
183 +# matrix.
184 +def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered):
185 + if sparse.dim() != 2:
186 + raise RuntimeError(
187 + f"Expected 2-dimensional sparse tensor, got {sparse.dim()}-dimensional tensor" # noqa: E501
188 + )
189 +
190 + m, k = sparse.shape
191 + device = sparse.device
192 +
193 + if meta_reordered.dim() != 2:
194 + raise RuntimeError(
195 + f"Expected 2-dimensional meta tensor, got {meta_reordered.dim()}-dimensional tensor" # noqa: E501
196 + )
197 + if meta_reordered.device != device:
198 + raise RuntimeError(
199 + f"Expected meta matrix to be on {device} device, got matrix on {meta_reordered.device} device" # noqa: E501
200 + )
201 +
202 + meta_dtype = meta_reordered.dtype
203 + if meta_dtype not in (torch.int16, torch.int32):
204 + raise RuntimeError(f"Invalid datatype {meta_dtype} of meta matrix")
205 + quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4
206 +
207 + ksparse = 4 if sparse.dtype != torch.float else 2
208 +
209 + meta_nrows, meta_ncols = meta_reordered.shape
210 + if meta_nrows != m:
211 + raise RuntimeError(
212 + f"Number of rows of meta matrix {meta_nrows} must be equal to number of columns of spase matrix {m}" # noqa: E501
213 + )
214 + if meta_ncols * ksparse * quadbits_per_meta_elem != 2 * k:
215 + raise RuntimeError(
216 + f"Number of columns of sparse matrix {k} different from the {meta_ncols * ksparse * quadbits_per_meta_elem // 2}, " # noqa: E501
217 + "expected according to the number of columns of meta matrix")
218 +
219 + # Undo meta tensor elements reordering.
220 + meta_offsets = _calculate_meta_reordering_scatter_offsets(
221 + m, meta_ncols, meta_dtype, device)
222 + meta = torch.gather(meta_reordered.view(-1), 0,
223 + meta_offsets).view(m, meta_ncols)
224 +
225 + # Unpack sparse tensor back to original dense tensor, using
226 + # information provided by meta tensor. Note that torch.float
227 + # datatype is handled pretty much the same as
228 + # torch.half/torch.bfloat16, as metadata for a pair of torch.float
229 + # value is encoded as if underlying 8 bytes contain four
230 + # torch.half/torch.bfloat16 values, where either first two or last
231 + # two are zeros.
232 + meta_2 = torch.empty(
233 + (m, meta_ncols, 2 * quadbits_per_meta_elem),
234 + dtype=meta_dtype,
235 + device=device,
236 + )
237 + if quadbits_per_meta_elem == 4:
238 + meta_2[:, :, 0] = meta & 0b11
239 + meta_2[:, :, 1] = (meta >> 2) & 0b11
240 + meta_2[:, :, 2] = (meta >> 4) & 0b11
241 + meta_2[:, :, 3] = (meta >> 6) & 0b11
242 + meta_2[:, :, 4] = (meta >> 8) & 0b11
243 + meta_2[:, :, 5] = (meta >> 10) & 0b11
244 + meta_2[:, :, 6] = (meta >> 12) & 0b11
245 + meta_2[:, :, 7] = (meta >> 14) & 0b11
246 + elif quadbits_per_meta_elem == 8:
247 + meta_2[:, :, 0] = meta & 0b11
248 + meta_2[:, :, 1] = (meta >> 2) & 0b11
249 + meta_2[:, :, 2] = (meta >> 4) & 0b11
250 + meta_2[:, :, 3] = (meta >> 6) & 0b11
251 + meta_2[:, :, 4] = (meta >> 8) & 0b11
252 + meta_2[:, :, 5] = (meta >> 10) & 0b11
253 + meta_2[:, :, 6] = (meta >> 12) & 0b11
254 + meta_2[:, :, 7] = (meta >> 14) & 0b11
255 + meta_2[:, :, 8] = (meta >> 16) & 0b11
256 + meta_2[:, :, 9] = (meta >> 18) & 0b11
257 + meta_2[:, :, 10] = (meta >> 20) & 0b11
258 + meta_2[:, :, 11] = (meta >> 22) & 0b11
259 + meta_2[:, :, 12] = (meta >> 24) & 0b11
260 + meta_2[:, :, 13] = (meta >> 26) & 0b11
261 + meta_2[:, :, 14] = (meta >> 28) & 0b11
262 + meta_2[:, :, 15] = (meta >> 30) & 0b11
263 +
264 + dense_offsets = meta_2.view(-1) + (
265 + torch.arange(0, 2 * m * k // ksparse, device=device) * 4).view(
266 + -1, 1).repeat(1, 2).view(-1)
267 +
268 + dense = torch.zeros((m * 2 * k, ), dtype=sparse.dtype, device=device)
269 + if sparse.dtype != torch.float:
270 + # dense.scatter_(0, dense_offsets, sparse.view(-1))
271 + dense.scatter_(0, dense_offsets, sparse.reshape(-1))
272 + else:
273 + dense.view(torch.half).scatter_(0, dense_offsets,
274 + sparse.view(torch.half).view(-1))
275 +
276 + return dense.view(m, 2 * k)
277 +
278 +
279 +def mask_creator(tensor):
280 + """
281 + Class for creating N:M sparsity masks.
282 + Masks will be created using the N:M ratio, where for every block of
283 + M weights, N will be pruned based on ranked weight value. Each mask
284 + will correspond to the given tensor.
285 +
286 + :param N: The number of weights in a group to keep
287 + :param M: The size of a weight group
288 + """
289 + N = 2
290 + M = 4
291 +
292 + mask = None
293 + # for i, tensor in enumerate(tensors):
294 + if tensor.numel() % M != 0:
295 + raise ValueError(
296 + f"Tensor of size {tensor.shape} can't be evenly divided into "
297 + f"{M} groups")
298 +
299 + num_groups = tensor.numel() // M
300 +
301 + # N:M sparsity for linear layers
302 + tensor_temp = tensor.detach().abs().reshape(num_groups, M)
303 + index = torch.argsort(tensor_temp, dim=1)[:, :int(M - N)]
304 +
305 + w_b = torch.ones(tensor_temp.shape, device=tensor_temp.device)
306 + mask = w_b.scatter_(dim=1, index=index, value=0).reshape(tensor.shape)
307 +
308 + return mask
vllm/model_executor/layers/quantization/utils/marlin_24_perms.py new
+58
@@ -0,0 +1,58 @@
1 +"""This file is used for /tests and /benchmarks"""
2 +import numpy
3 +import torch
4 +
5 +
6 +# Precompute permutations for Marlin24 weight and scale shuffling # noqa: E501
7 +#
8 +# Marlin works on [16*2,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
9 +# with the tensor-core format that is described here:
10 +# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
11 +#
12 +# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
13 +# (without the need to use ldmatrix instructions) # noqa: E501
14 +def get_perms_24(num_bits):
15 + perm_list = []
16 + for i in range(32):
17 + perm1 = []
18 + col = i // 4
19 + col_o = col // 2
20 + for block in [0, 1]:
21 + for row in [
22 + 2 * (i % 4),
23 + 2 * (i % 4) + 1,
24 + 2 * (i % 4 + 4),
25 + 2 * (i % 4 + 4) + 1,
26 + ]:
27 + perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
28 + 4 * block)
29 + for j in range(4):
30 + perm_list.extend([p + 1 * j for p in perm1])
31 + perm = numpy.array(perm_list)
32 +
33 + if num_bits == 4:
34 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
35 + elif num_bits == 8:
36 + interleave = numpy.array([0, 2, 1, 3])
37 + else:
38 + raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits))
39 +
40 + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
41 + perm = torch.from_numpy(perm)
42 + scale_perm = []
43 + for i in range(8):
44 + scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
45 + scale_perm_single = []
46 + for i in range(8):
47 + scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
48 + return perm, scale_perm, scale_perm_single
49 +
50 +
51 +marlin_24_perm = {}
52 +marlin_24_scale_perm = {}
53 +marlin_24_scale_perm_single = {}
54 +for num_bits in [4, 8]:
55 + perm_24, scale_perm_24, scale_perm_single_24 = get_perms_24(num_bits)
56 + marlin_24_perm[num_bits] = perm_24
57 + marlin_24_scale_perm[num_bits] = scale_perm_24
58 + marlin_24_scale_perm_single[num_bits] = scale_perm_single_24
vllm/model_executor/layers/quantization/utils/marlin_perms.py new
+58
@@ -0,0 +1,58 @@
1 +"""This file is used for /tests and /benchmarks"""
2 +import numpy
3 +import torch
4 +
5 +
6 +# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
7 +#
8 +# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
9 +# with the tensor-core format that is described here:
10 +# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
11 +#
12 +# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
13 +# (without the need to use ldmatrix instructions) # noqa: E501
14 +def get_perms(num_bits):
15 + perm_list = []
16 + for i in range(32):
17 + perm1 = []
18 + col = i // 4
19 + for block in [0, 1]:
20 + for row in [
21 + 2 * (i % 4),
22 + 2 * (i % 4) + 1,
23 + 2 * (i % 4 + 4),
24 + 2 * (i % 4 + 4) + 1,
25 + ]:
26 + perm1.append(16 * row + col + 8 * block)
27 + for j in range(4):
28 + perm_list.extend([p + 256 * j for p in perm1])
29 +
30 + perm = numpy.array(perm_list)
31 +
32 + if num_bits == 4:
33 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
34 + elif num_bits == 8:
35 + interleave = numpy.array([0, 2, 1, 3])
36 + else:
37 + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
38 +
39 + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
40 + perm = torch.from_numpy(perm)
41 + scale_perm = []
42 + for i in range(8):
43 + scale_perm.extend([i + 8 * j for j in range(8)])
44 + scale_perm_single = []
45 + for i in range(4):
46 + scale_perm_single.extend(
47 + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
48 + return perm, scale_perm, scale_perm_single
49 +
50 +
51 +marlin_perm = {}
52 +marlin_scale_perm = {}
53 +marlin_scale_perm_single = {}
54 +for num_bits in [4, 8]:
55 + perm, scale_perm, scale_perm_single = get_perms(num_bits)
56 + marlin_perm[num_bits] = perm
57 + marlin_scale_perm[num_bits] = scale_perm
58 + marlin_scale_perm_single[num_bits] = scale_perm_single
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+132 -82
@@ -1,79 +1,28 @@
1 """This file is used for /tests and /benchmarks"""
2 +import random
3 +
4 import numpy
5 import torch
6
5 -from vllm.model_executor.layers.quantization.gptq_marlin import (
6 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_TILE)
7 +from vllm.model_executor.layers.quantization.utils.format_24 import (
8 + mask_creator, sparse_semi_structured_from_dense_cutlass)
9 +from vllm.model_executor.layers.quantization.utils.marlin_24_perms import (
10 + marlin_24_perm, marlin_24_scale_perm, marlin_24_scale_perm_single)
11 +from vllm.model_executor.layers.quantization.utils.marlin_perms import (
12 + marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
13 from vllm.model_executor.layers.quantization.utils.quant_utils import (
14 get_pack_factor, quantize_weights, sort_weights)
15
16 __cuda_arch = torch.cuda.get_device_capability()
17
18 +MARLIN_TILE = 16
19 +
20
21 def is_marlin_supported():
22 return __cuda_arch[0] >= 8
23
24
17 -# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
18 -#
19 -# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
20 -# with the tensor-core format that is described here:
21 -# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
22 -#
23 -# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
24 -# (without the need to use ldmatrix instructions) # noqa: E501
25 -def _get_perms(num_bits):
26 - perm_list = []
27 - for i in range(32):
28 - perm1 = []
29 - col = i // 4
30 - for block in [0, 1]:
31 - for row in [
32 - 2 * (i % 4),
33 - 2 * (i % 4) + 1,
34 - 2 * (i % 4 + 4),
35 - 2 * (i % 4 + 4) + 1,
36 - ]:
37 - perm1.append(16 * row + col + 8 * block)
38 - for j in range(4):
39 - perm_list.extend([p + 256 * j for p in perm1])
40 -
41 - perm = numpy.array(perm_list)
42 -
43 - if num_bits == 4:
44 - interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
45 - elif num_bits == 8:
46 - interleave = numpy.array([0, 2, 1, 3])
47 - else:
48 - raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
49 -
50 - perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
51 - perm = torch.from_numpy(perm)
52 - scale_perm = []
53 - for i in range(8):
54 - scale_perm.extend([i + 8 * j for j in range(8)])
55 - scale_perm_single = []
56 - for i in range(4):
57 - scale_perm_single.extend(
58 - [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
59 - return perm, scale_perm, scale_perm_single
60 -
61 -
62 -_perm = {}
63 -_scale_perm = {}
64 -_scale_perm_single = {}
65 -for num_bits in [4, 8]:
66 - perm, scale_perm, scale_perm_single = _get_perms(num_bits)
67 - _perm[num_bits] = perm
68 - _scale_perm[num_bits] = scale_perm
69 - _scale_perm_single[num_bits] = scale_perm_single
70 -
71 -
72 -def marlin_permute_weights(q_w,
73 - size_k,
74 - size_n,
75 - num_bits,
76 - tile=GPTQ_MARLIN_TILE):
25 +def marlin_permute_weights(q_w, size_k, size_n, perm, tile=MARLIN_TILE):
26 assert q_w.shape == (size_k, size_n)
27 assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
28 assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
@@ -83,15 +32,14 @@ def marlin_permute_weights(q_w,
32 q_w = q_w.permute((0, 2, 1, 3))
33 q_w = q_w.reshape((size_k // tile, size_n * tile))
34
86 - q_w = q_w.reshape(
87 - (-1, _perm[num_bits].numel()))[:, _perm[num_bits]].reshape(q_w.shape)
35 + q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
36
37 return q_w
38
39
92 -def marlin_weights(q_w, size_k, size_n, num_bits):
40 +def marlin_weights(q_w, size_k, size_n, num_bits, perm):
41 # Permute
94 - q_w = marlin_permute_weights(q_w, size_k, size_n, num_bits)
42 + q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
43
44 # Pack
45 pack_factor = get_pack_factor(num_bits)
@@ -101,7 +49,6 @@ def marlin_weights(q_w, size_k, size_n, num_bits):
49
50 q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
51 dtype=numpy.uint32)
104 -
52 for i in range(pack_factor):
53 q_packed |= q_w[:, i::pack_factor] << num_bits * i
54
@@ -110,15 +57,12 @@ def marlin_weights(q_w, size_k, size_n, num_bits):
57 return q_packed
58
59
113 -def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
60 +def marlin_permute_scales(s, size_k, size_n, group_size, scale_perm,
61 + scale_perm_single):
62 if group_size < size_k and group_size != -1:
115 - s = s.reshape((-1, len(_scale_perm[num_bits])))[:,
116 - _scale_perm[num_bits]]
63 + s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
64 else:
118 - s = s.reshape(
119 - (-1,
120 - len(_scale_perm_single[num_bits])))[:,
121 - _scale_perm_single[num_bits]]
65 + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
66 s = s.reshape((-1, size_n)).contiguous()
67
68 return s
@@ -148,8 +92,11 @@ def marlin_quantize(
92 q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
93
94 # Reformat to marlin
151 - marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits)
152 - marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, num_bits)
95 + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits,
96 + marlin_perm[num_bits])
97 + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size,
98 + marlin_scale_perm[num_bits],
99 + marlin_scale_perm_single[num_bits])
100
101 # Create result
102 res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
@@ -159,15 +106,118 @@ def marlin_quantize(
106 return res_list
107
108
109 +def inject_24(w, size_k, size_n):
110 + assert w.shape == (size_k, size_n)
111 +
112 + mask = mask_creator(w.t()).t().cuda().bool()
113 +
114 + return (mask * w).contiguous(), mask.contiguous()
115 +
116 +
117 +def check_24(w, num_rows_to_sample=50, _verbose=False):
118 + BLOCK_SIZE = 4
119 + MAX_NON_ZEROS = 2
120 +
121 + w = w.t().contiguous()
122 +
123 + print("check_24: w.shape = {}".format(w.shape))
124 +
125 + num_rows, num_cols = w.shape
126 + sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample)
127 + if _verbose:
128 + print(f"Sampled row idxs = {sampled_row_idxs}")
129 +
130 + total_segments = 0
131 + non_24_segments = 0
132 + for i in sampled_row_idxs:
133 + for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
134 + total_segments += 1
135 + block = w[i, j:j + BLOCK_SIZE]
136 + num_nonzero = torch.count_nonzero(block)
137 + if num_nonzero > MAX_NON_ZEROS:
138 + print("i = {} j = {} block = {}".format(i, j, block))
139 + non_24_segments += 1
140 +
141 + print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
142 +
143 +
144 +def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
145 + assert q_24.shape == (size_k, size_n)
146 +
147 + # Remove zp to normalize over 0
148 + max_q_val = (1 << num_bits) - 1
149 + zp = (max_q_val + 1) // 2
150 + q_24_no_zp = q_24 - zp
151 +
152 + # Compress
153 + q_24_no_zp = q_24_no_zp.t().contiguous()
154 + q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
155 + q_24_no_zp)
156 + q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
157 +
158 + # Restore zp
159 + q_24_comp = q_24_no_zp_comp + zp
160 +
161 + # Resize meta to its actual shape (without moving any data)
162 + meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
163 +
164 + return q_24_comp, meta
165 +
166 +
167 +def marlin_24_quantize(
168 + w: torch.Tensor,
169 + num_bits: int,
170 + group_size: int,
171 +):
172 + size_k, size_n = w.shape
173 +
174 + # Normalize group_size
175 + if group_size == -1:
176 + group_size = size_k
177 + assert group_size <= size_k
178 +
179 + # Inject 2:4 sparsity
180 + w_24, mask_24 = inject_24(w, size_k, size_n)
181 +
182 + # Quantize
183 + w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
184 + num_bits,
185 + group_size,
186 + act_order=False)
187 +
188 + # Compress quantized weight
189 + q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
190 + num_bits)
191 + size_k_comp = size_k // 2
192 +
193 + # Reformat to marlin
194 + marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
195 + num_bits, marlin_24_perm[num_bits])
196 + marlin_24_s = marlin_permute_scales(s, size_k, size_n, group_size,
197 + marlin_24_scale_perm[num_bits],
198 + marlin_24_scale_perm_single[num_bits])
199 +
200 + # Create result
201 + res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s]
202 + for i in range(len(res_list)):
203 + res_list[i] = res_list[i].to(w.device)
204 +
205 + return res_list
206 +
207 +
208 +def compute_max_diff(output, output_ref):
209 + return torch.mean(torch.abs(output - output_ref)) / torch.mean(
210 + torch.abs(output_ref))
211 +
212 +
213 class MarlinWorkspace:
214
164 - def __init__(self, out_features):
165 - assert (out_features % GPTQ_MARLIN_MIN_THREAD_N == 0), (
166 - "out_features = {} is undivisible by GPTQ_MARLIN_MIN_THREAD_N = {}"
167 - .format(out_features, GPTQ_MARLIN_MIN_THREAD_N))
215 + def __init__(self, out_features, min_thread_n, max_parallel):
216 + assert (out_features % min_thread_n == 0), (
217 + "out_features = {} is undivisible by min_thread_n = {}".format(
218 + out_features, min_thread_n))
219
169 - max_workspace_size = ((out_features // GPTQ_MARLIN_MIN_THREAD_N) *
170 - GPTQ_MARLIN_MAX_PARALLEL)
220 + max_workspace_size = ((out_features // min_thread_n) * max_parallel)
221
222 self.scratch = torch.zeros(max_workspace_size,
223 dtype=torch.int,