A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Add marlin_24 unit tests (#4901)
Alexander Matveev committed
May 19, 2024 at 11:37 UTC
27ce85476e6b170c5c90c65ac5c3268911135766
6 files changed
+649
-103
tests/kernels/test_marlin_gemm.py
+74
-13
@@ -7,23 +7,32 @@ import torch
7
8
from vllm import _custom_ops as ops
9
from vllm.model_executor.layers.quantization.gptq_marlin import (
10
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
11
GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
12
+from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
13
+ GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
14
+ GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
15
+from vllm.model_executor.layers.quantization.utils.marlin_perms import (
16
+ marlin_perm)
17
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
- MarlinWorkspace, is_marlin_supported, marlin_quantize, marlin_weights)
18
+ MarlinWorkspace, compute_max_diff, is_marlin_supported, marlin_24_quantize,
19
+ marlin_quantize, marlin_weights)
20
from vllm.model_executor.layers.quantization.utils.quant_utils import (
21
gptq_pack, quantize_weights, sort_weights)
22
23
ACT_ORDER_OPTS = [False, True]
24
K_FULL_OPTS = [False, True]
25
19
-K_CHUNKS = [128, 256]
20
-N_CHUNKS = [64, 128, 256]
26
+MARLIN_K_CHUNKS = [128]
27
+MARLIN_N_CHUNKS = [64, 128, 256]
28
+
29
+MARLIN_24_K_CHUNKS = [128]
30
+MARLIN_24_N_CHUNKS = [256]
31
32
MNK_FACTORS = [
33
(1, 1, 1),
34
(1, 4, 8),
35
(1, 7, 5),
26
- (1, 7 * 4, 5 * 1),
36
(13, 17, 67),
37
(26, 37, 13),
38
(67, 13, 11),
@@ -31,14 +40,13 @@ MNK_FACTORS = [
40
41
42
def rand_data(shape):
34
- data = torch.rand(shape).to(torch.half).cuda()
35
- return data
43
+ return torch.randn(shape, dtype=torch.half, device="cuda")
44
45
46
@pytest.mark.skipif(not is_marlin_supported(),
47
reason="Marlin is not supported on this GPU type.")
40
-@pytest.mark.parametrize("k_chunk", K_CHUNKS)
41
-@pytest.mark.parametrize("n_chunk", N_CHUNKS)
48
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
49
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
50
@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
51
@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
52
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
@@ -82,7 +90,8 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
90
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
91
92
# Pack to Marlin format
85
- marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits)
93
+ marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits,
94
+ marlin_perm[num_bits])
95
96
# Run Marlin repack GPU kernel
97
marlin_q_w_2 = ops.gptq_marlin_repack(
@@ -99,8 +108,8 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
108
109
@pytest.mark.skipif(not is_marlin_supported(),
110
reason="Marlin is not supported on this GPU type.")
102
-@pytest.mark.parametrize("k_chunk", K_CHUNKS)
103
-@pytest.mark.parametrize("n_chunk", N_CHUNKS)
111
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
112
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
113
@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
114
@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
115
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
@@ -136,7 +145,8 @@ def test_marlin_gemm(
145
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
146
b_weight, num_bits, group_size, act_order)
147
139
- workspace = MarlinWorkspace(size_n)
148
+ workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
149
+ GPTQ_MARLIN_MAX_PARALLEL)
150
151
output = ops.gptq_marlin_gemm(
152
a_input,
@@ -155,4 +165,55 @@ def test_marlin_gemm(
165
166
torch.cuda.synchronize()
167
158
- assert torch.allclose(output, output_ref, rtol=1e-2)
168
+ max_diff = compute_max_diff(output, output_ref)
169
+ print("max_diff = {}".format(max_diff))
170
+
171
+ assert max_diff < 0.04
172
+
173
+
174
+@pytest.mark.skipif(not is_marlin_supported(),
175
+ reason="Marlin is not supported on this GPU type.")
176
+@pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
177
+@pytest.mark.parametrize("n_chunk", MARLIN_24_N_CHUNKS)
178
+@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
179
+@pytest.mark.parametrize("group_size", GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES)
180
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
181
+def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
182
+ m_factor, n_factor, k_factor = mnk_factors
183
+
184
+ size_m = m_factor
185
+ size_k = k_chunk * k_factor
186
+ size_n = n_chunk * n_factor
187
+
188
+ print(f"MNK = {size_m} {size_n} {size_k}")
189
+ print(f"groupsize = {group_size}")
190
+
191
+ a_input = rand_data((size_m, size_k))
192
+ b_weight = rand_data((size_k, size_n))
193
+
194
+ (w_24_ref, marlin_24_q_w_comp, marlin_24_meta,
195
+ marlin_24_s) = marlin_24_quantize(b_weight, num_bits, group_size)
196
+
197
+ workspace_24 = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
198
+ GPTQ_MARLIN_24_MAX_PARALLEL)
199
+
200
+ output_ref = torch.matmul(a_input, w_24_ref)
201
+
202
+ output = ops.gptq_marlin_24_gemm(
203
+ a_input,
204
+ marlin_24_q_w_comp,
205
+ marlin_24_meta,
206
+ marlin_24_s,
207
+ workspace_24.scratch,
208
+ num_bits,
209
+ a_input.shape[0],
210
+ b_weight.shape[1],
211
+ a_input.shape[1],
212
+ )
213
+
214
+ torch.cuda.synchronize()
215
+
216
+ max_diff = compute_max_diff(output, output_ref)
217
+ print("max_diff = {}".format(max_diff))
218
+
219
+ assert max_diff < 0.04
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+19
-8
@@ -12,6 +12,15 @@ from vllm.model_executor.utils import set_weight_attrs
12
13
logger = init_logger(__name__)
14
15
+GPTQ_MARLIN_24_TILE = 16
16
+GPTQ_MARLIN_24_MIN_THREAD_N = 128
17
+GPTQ_MARLIN_24_MIN_THREAD_K = 128
18
+GPTQ_MARLIN_24_MAX_PARALLEL = 16
19
+
20
+GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
21
+GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
22
+GPTQ_MARLIN_24_SUPPORTED_SYM = [True]
23
+
24
25
class GPTQMarlin24Config(QuantizationConfig):
26
"""Config class for Marlin24.
@@ -25,15 +34,17 @@ class GPTQMarlin24Config(QuantizationConfig):
34
self.weight_bits = weight_bits
35
self.group_size = group_size
36
28
- if self.weight_bits != 4 and self.weight_bits != 8:
29
- raise ValueError("weight_bits must be 4 or 8. Got = {}".format(
30
- self.weight_bits))
31
-
32
- if self.group_size != 128 and self.group_size != -1:
37
+ # Verify
38
+ if self.weight_bits not in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS:
39
+ raise ValueError(
40
+ f"Marlin_24 does not support weight_bits = {self.weight_bits}. "
41
+ f"Only weight_bits = {GPTQ_MARLIN_24_SUPPORTED_NUM_BITS} "
42
+ "are supported.")
43
+ if self.group_size not in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES:
44
raise ValueError(
34
- "Currently, only group size 128 and -1 (channelwise) "
35
- "is supported for Marlin24, but got group_size of "
36
- f"{self.group_size}")
45
+ f"Marlin_24 does not support group_size = {self.group_size}. "
46
+ f"Only group_sizes = {GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES} "
47
+ "are supported.")
48
49
# 4 Bits packed into 32 bit datatype.
50
self.pack_factor = 32 // self.weight_bits
vllm/model_executor/layers/quantization/utils/format_24.py
new
+308
@@ -0,0 +1,308 @@
1
+#
2
+# Modified by Roberto Lopez Castro (roberto.lopez.castro@udc.es).
3
+#
4
+
5
+import torch
6
+
7
+
8
+# This is PyTorch implementation of main part of reorder_meta()
9
+# function, from tools/util/include/cutlass/util/host_reorder.h file
10
+# of CUTLASS source tree. Furthermore, CUTLASS template for sparse
11
+# GEMM decides upon layout of this matrix, and at the moment for the
12
+# sparse GEMM executed on tensor cores, this is layout described by
13
+# ColumnMajorInterleaved<2> data structure, in
14
+# include/cutlass/layout/matrix.h of CUTLASS source tree. The
15
+# reordering of meta matrix into meta_reordered matrix calculated
16
+# according to these segments of CUTLASS code is re-implemented here.
17
+# Note that this calculation produces offsets for scattering metadata
18
+# matrix elements into reordered metadata matrix elements (or,
19
+# equivalently, for gathering reordered metadata matrix element back
20
+# into metadata matrix elements).
21
+def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype,
22
+ device):
23
+ dst_rows = torch.arange(0, m, device=device)[:, None].repeat(1, meta_ncols)
24
+ dst_cols = torch.arange(0, meta_ncols, device=device).repeat(m, 1)
25
+
26
+ # Reorder the rows, then swizzle the 2x2 blocks.
27
+ group_x = 64
28
+ group_y = 32 if meta_dtype.itemsize == 2 else 16
29
+
30
+ dst_rows = (dst_rows // group_x * group_x + (dst_rows % 2) * 2 +
31
+ (dst_rows % 8) // 4 + ((dst_rows % group_y) % 4) // 2 * 32 +
32
+ ((dst_rows % group_x) // 8) * 4)
33
+
34
+ topright = ((dst_rows % 2 == 0) & (dst_cols % 2 == 1)).to(torch.int8)
35
+ bottomleft = ((dst_rows % 2 == 1) & (dst_cols % 2 == 0)).to(torch.int8)
36
+ dst_rows += topright - bottomleft
37
+ dst_cols -= topright - bottomleft
38
+
39
+ # Assumed that meta tensor is to be stored in CUTLASS
40
+ # InterleavedColumnMajor layout, and reverse engineered
41
+ # corresponding code to store values into this tensor.
42
+ interleave = 2
43
+ cols_maj = dst_cols // interleave
44
+ cols_min = dst_cols % interleave
45
+ return (cols_maj * m * interleave + dst_rows * interleave +
46
+ cols_min).view(-1)
47
+
48
+
49
+# This function converts dense matrix into sparse semi-structured
50
+# representation, producing "compressed" matrix, in the layout used by
51
+# CUTLASS backend, and corresponding metadata matrix.
52
+def sparse_semi_structured_from_dense_cutlass(dense):
53
+ if dense.dim() != 2:
54
+ raise RuntimeError(
55
+ f"Expected 2-dimensional dense tensor, got {dense.dim()}-dimensional tensor" # noqa: E501
56
+ )
57
+
58
+ m, k = dense.shape
59
+ device = dense.device
60
+
61
+ meta_dtype = torch.int8
62
+ if dense.dtype == torch.int8:
63
+ meta_dtype = torch.int32
64
+ elif dense.dtype in [torch.half, torch.bfloat16, torch.float, torch.int32]:
65
+ meta_dtype = torch.int16
66
+ else:
67
+ raise RuntimeError(f"Invalid datatype {dense.dtype} of dense matrix")
68
+ quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4
69
+ if quadbits_per_meta_elem not in (4, 8):
70
+ raise RuntimeError(
71
+ "Invalid number of elements per meta element calculated")
72
+
73
+ if meta_dtype == torch.int32:
74
+ if m % 16 != 0:
75
+ raise RuntimeError(
76
+ f"Number of rows of dense matrix {m} must be divisible by 16")
77
+ else:
78
+ if m % 32 != 0:
79
+ raise RuntimeError(
80
+ f"Number of rows of dense matrix {m} must be divisible by 32")
81
+ if k % (4 * quadbits_per_meta_elem) != 0:
82
+ raise RuntimeError(
83
+ f"Number of columns of dense matrix {k} must be divisible by {4 * quadbits_per_meta_elem}" # noqa: E501
84
+ )
85
+
86
+ if dense.dtype != torch.float:
87
+ ksparse = 4
88
+ dense_4 = dense.view(-1, k // ksparse, ksparse)
89
+ m0, m1, m2, m3 = (dense_4 != 0).unbind(-1)
90
+ else:
91
+ ksparse = 2
92
+ dense_2 = dense.view(-1, k // ksparse, ksparse)
93
+ m0, m2 = m1, m3 = (dense_2 != 0).unbind(-1)
94
+ meta_ncols = k // (ksparse * quadbits_per_meta_elem)
95
+
96
+ # Encoding quadruples of True/False values as follows:
97
+ # [True, True, False, False] -> 0b0100
98
+ # [True, False, True, False] -> 0b1000
99
+ # [False, True, True, False] -> 0b1001
100
+ # [True, False, False, True ] -> 0b1100
101
+ # [False, True, False, True ] -> 0b1101
102
+ # [False, False, True, True ] -> 0b1110
103
+ # Thus, lower two bits in the encoding are index of the True value
104
+ # at the lowest index in the quadruple, and the higher two bits in
105
+ # the encoding are index of the other True value in the quadruple.
106
+ # In case there are less than two True values, than False value or
107
+ # values at some index or indices are considered True for the
108
+ # encoding. In case there are more than two True values, then the
109
+ # excess True value(s) at some indices are considered False for
110
+ # the encoding. The exact encodings used for these cases are as
111
+ # follows:
112
+ # [False, False, False, False] -> 0b1110
113
+ # [False, False, False, True ] -> 0b1110
114
+ # [False, False, True, False] -> 0b1110
115
+ # [False, True, False, False] -> 0b1001
116
+ # [False, True, True, True ] -> 0b1101
117
+ # [True, False, False, False] -> 0b1000
118
+ # [True, False, True, True ] -> 0b1100
119
+ # [True, True, False, True ] -> 0b0100
120
+ # [True, True, True, False] -> 0b0100
121
+ # [True, True, True, True ] -> 0b0100
122
+ # These particular encodings are chosen, with the help of Espresso
123
+ # logic minimizer software, for the purpose of minimization of
124
+ # corresponding Boolean functions, that translate non-zero flags
125
+ # into encoding bits. Note also possible choices for the first
126
+ # and last of these encodings were limited only to (0b0100,
127
+ # 0b1110), in order to produce valid encodings for 1:2 sparsity
128
+ # case.
129
+
130
+ expr0 = m0 & m1
131
+ expr1 = ~m0 & m1
132
+ expr2 = ~m0 & ~m1
133
+ bit0 = expr1
134
+ bit1 = expr2
135
+ bit2 = expr0 | expr2 | m3
136
+ bit3 = expr1 | ~m1
137
+ idxs0 = bit0 | (bit1.to(torch.int64) << 1)
138
+ idxs1 = bit2 | (bit3.to(torch.int64) << 1)
139
+
140
+ if dense.dtype != torch.float:
141
+ sparse0 = dense_4.gather(
142
+ -1, idxs0.unsqueeze(-1)) # type: ignore[possibly-undefined]
143
+ sparse1 = dense_4.gather(-1, idxs1.unsqueeze(-1))
144
+ sparse = torch.stack((sparse0, sparse1), dim=-1).view(m, k // 2)
145
+ else:
146
+ sparse = dense_2.gather(-1,
147
+ idxs0.unsqueeze(-1) // 2).view(
148
+ m,
149
+ k // 2) # type: ignore[possibly-undefined]
150
+
151
+ meta_4 = idxs0 | (idxs1 << 2)
152
+ meta_n = meta_4.view(
153
+ (-1, meta_ncols, quadbits_per_meta_elem)).to(meta_dtype)
154
+
155
+ if quadbits_per_meta_elem == 4:
156
+ meta = (meta_n[:, :, 0]
157
+ | (meta_n[:, :, 1] << 4)
158
+ | (meta_n[:, :, 2] << 8)
159
+ | (meta_n[:, :, 3] << 12))
160
+ elif quadbits_per_meta_elem == 8:
161
+ meta = (meta_n[:, :, 0]
162
+ | (meta_n[:, :, 1] << 4)
163
+ | (meta_n[:, :, 2] << 8)
164
+ | (meta_n[:, :, 3] << 12)
165
+ | (meta_n[:, :, 4] << 16)
166
+ | (meta_n[:, :, 5] << 20)
167
+ | (meta_n[:, :, 6] << 24)
168
+ | (meta_n[:, :, 7] << 28))
169
+
170
+ # Reorder meta tensor elements.
171
+ meta_reordered = meta.new_empty(
172
+ (m * meta_ncols, )) # type: ignore[possibly-undefined]
173
+ meta_offsets = _calculate_meta_reordering_scatter_offsets(
174
+ m, meta_ncols, meta_dtype, device)
175
+ meta_reordered.scatter_(0, meta_offsets, meta.view(-1))
176
+
177
+ return (sparse, meta_reordered.view(m, meta_ncols))
178
+
179
+
180
+# This function performs reverse of the function above - it
181
+# reconstructs dense matrix from a pair of "compressed" matrix, given
182
+# in the layout used by CUTLASS backend, and accompanying metadata
183
+# matrix.
184
+def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered):
185
+ if sparse.dim() != 2:
186
+ raise RuntimeError(
187
+ f"Expected 2-dimensional sparse tensor, got {sparse.dim()}-dimensional tensor" # noqa: E501
188
+ )
189
+
190
+ m, k = sparse.shape
191
+ device = sparse.device
192
+
193
+ if meta_reordered.dim() != 2:
194
+ raise RuntimeError(
195
+ f"Expected 2-dimensional meta tensor, got {meta_reordered.dim()}-dimensional tensor" # noqa: E501
196
+ )
197
+ if meta_reordered.device != device:
198
+ raise RuntimeError(
199
+ f"Expected meta matrix to be on {device} device, got matrix on {meta_reordered.device} device" # noqa: E501
200
+ )
201
+
202
+ meta_dtype = meta_reordered.dtype
203
+ if meta_dtype not in (torch.int16, torch.int32):
204
+ raise RuntimeError(f"Invalid datatype {meta_dtype} of meta matrix")
205
+ quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4
206
+
207
+ ksparse = 4 if sparse.dtype != torch.float else 2
208
+
209
+ meta_nrows, meta_ncols = meta_reordered.shape
210
+ if meta_nrows != m:
211
+ raise RuntimeError(
212
+ f"Number of rows of meta matrix {meta_nrows} must be equal to number of columns of spase matrix {m}" # noqa: E501
213
+ )
214
+ if meta_ncols * ksparse * quadbits_per_meta_elem != 2 * k:
215
+ raise RuntimeError(
216
+ f"Number of columns of sparse matrix {k} different from the {meta_ncols * ksparse * quadbits_per_meta_elem // 2}, " # noqa: E501
217
+ "expected according to the number of columns of meta matrix")
218
+
219
+ # Undo meta tensor elements reordering.
220
+ meta_offsets = _calculate_meta_reordering_scatter_offsets(
221
+ m, meta_ncols, meta_dtype, device)
222
+ meta = torch.gather(meta_reordered.view(-1), 0,
223
+ meta_offsets).view(m, meta_ncols)
224
+
225
+ # Unpack sparse tensor back to original dense tensor, using
226
+ # information provided by meta tensor. Note that torch.float
227
+ # datatype is handled pretty much the same as
228
+ # torch.half/torch.bfloat16, as metadata for a pair of torch.float
229
+ # value is encoded as if underlying 8 bytes contain four
230
+ # torch.half/torch.bfloat16 values, where either first two or last
231
+ # two are zeros.
232
+ meta_2 = torch.empty(
233
+ (m, meta_ncols, 2 * quadbits_per_meta_elem),
234
+ dtype=meta_dtype,
235
+ device=device,
236
+ )
237
+ if quadbits_per_meta_elem == 4:
238
+ meta_2[:, :, 0] = meta & 0b11
239
+ meta_2[:, :, 1] = (meta >> 2) & 0b11
240
+ meta_2[:, :, 2] = (meta >> 4) & 0b11
241
+ meta_2[:, :, 3] = (meta >> 6) & 0b11
242
+ meta_2[:, :, 4] = (meta >> 8) & 0b11
243
+ meta_2[:, :, 5] = (meta >> 10) & 0b11
244
+ meta_2[:, :, 6] = (meta >> 12) & 0b11
245
+ meta_2[:, :, 7] = (meta >> 14) & 0b11
246
+ elif quadbits_per_meta_elem == 8:
247
+ meta_2[:, :, 0] = meta & 0b11
248
+ meta_2[:, :, 1] = (meta >> 2) & 0b11
249
+ meta_2[:, :, 2] = (meta >> 4) & 0b11
250
+ meta_2[:, :, 3] = (meta >> 6) & 0b11
251
+ meta_2[:, :, 4] = (meta >> 8) & 0b11
252
+ meta_2[:, :, 5] = (meta >> 10) & 0b11
253
+ meta_2[:, :, 6] = (meta >> 12) & 0b11
254
+ meta_2[:, :, 7] = (meta >> 14) & 0b11
255
+ meta_2[:, :, 8] = (meta >> 16) & 0b11
256
+ meta_2[:, :, 9] = (meta >> 18) & 0b11
257
+ meta_2[:, :, 10] = (meta >> 20) & 0b11
258
+ meta_2[:, :, 11] = (meta >> 22) & 0b11
259
+ meta_2[:, :, 12] = (meta >> 24) & 0b11
260
+ meta_2[:, :, 13] = (meta >> 26) & 0b11
261
+ meta_2[:, :, 14] = (meta >> 28) & 0b11
262
+ meta_2[:, :, 15] = (meta >> 30) & 0b11
263
+
264
+ dense_offsets = meta_2.view(-1) + (
265
+ torch.arange(0, 2 * m * k // ksparse, device=device) * 4).view(
266
+ -1, 1).repeat(1, 2).view(-1)
267
+
268
+ dense = torch.zeros((m * 2 * k, ), dtype=sparse.dtype, device=device)
269
+ if sparse.dtype != torch.float:
270
+ # dense.scatter_(0, dense_offsets, sparse.view(-1))
271
+ dense.scatter_(0, dense_offsets, sparse.reshape(-1))
272
+ else:
273
+ dense.view(torch.half).scatter_(0, dense_offsets,
274
+ sparse.view(torch.half).view(-1))
275
+
276
+ return dense.view(m, 2 * k)
277
+
278
+
279
+def mask_creator(tensor):
280
+ """
281
+ Class for creating N:M sparsity masks.
282
+ Masks will be created using the N:M ratio, where for every block of
283
+ M weights, N will be pruned based on ranked weight value. Each mask
284
+ will correspond to the given tensor.
285
+
286
+ :param N: The number of weights in a group to keep
287
+ :param M: The size of a weight group
288
+ """
289
+ N = 2
290
+ M = 4
291
+
292
+ mask = None
293
+ # for i, tensor in enumerate(tensors):
294
+ if tensor.numel() % M != 0:
295
+ raise ValueError(
296
+ f"Tensor of size {tensor.shape} can't be evenly divided into "
297
+ f"{M} groups")
298
+
299
+ num_groups = tensor.numel() // M
300
+
301
+ # N:M sparsity for linear layers
302
+ tensor_temp = tensor.detach().abs().reshape(num_groups, M)
303
+ index = torch.argsort(tensor_temp, dim=1)[:, :int(M - N)]
304
+
305
+ w_b = torch.ones(tensor_temp.shape, device=tensor_temp.device)
306
+ mask = w_b.scatter_(dim=1, index=index, value=0).reshape(tensor.shape)
307
+
308
+ return mask
vllm/model_executor/layers/quantization/utils/marlin_24_perms.py
new
+58
@@ -0,0 +1,58 @@
1
+"""This file is used for /tests and /benchmarks"""
2
+import numpy
3
+import torch
4
+
5
+
6
+# Precompute permutations for Marlin24 weight and scale shuffling # noqa: E501
7
+#
8
+# Marlin works on [16*2,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
9
+# with the tensor-core format that is described here:
10
+# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
11
+#
12
+# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
13
+# (without the need to use ldmatrix instructions) # noqa: E501
14
+def get_perms_24(num_bits):
15
+ perm_list = []
16
+ for i in range(32):
17
+ perm1 = []
18
+ col = i // 4
19
+ col_o = col // 2
20
+ for block in [0, 1]:
21
+ for row in [
22
+ 2 * (i % 4),
23
+ 2 * (i % 4) + 1,
24
+ 2 * (i % 4 + 4),
25
+ 2 * (i % 4 + 4) + 1,
26
+ ]:
27
+ perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
28
+ 4 * block)
29
+ for j in range(4):
30
+ perm_list.extend([p + 1 * j for p in perm1])
31
+ perm = numpy.array(perm_list)
32
+
33
+ if num_bits == 4:
34
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
35
+ elif num_bits == 8:
36
+ interleave = numpy.array([0, 2, 1, 3])
37
+ else:
38
+ raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits))
39
+
40
+ perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
41
+ perm = torch.from_numpy(perm)
42
+ scale_perm = []
43
+ for i in range(8):
44
+ scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
45
+ scale_perm_single = []
46
+ for i in range(8):
47
+ scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
48
+ return perm, scale_perm, scale_perm_single
49
+
50
+
51
+marlin_24_perm = {}
52
+marlin_24_scale_perm = {}
53
+marlin_24_scale_perm_single = {}
54
+for num_bits in [4, 8]:
55
+ perm_24, scale_perm_24, scale_perm_single_24 = get_perms_24(num_bits)
56
+ marlin_24_perm[num_bits] = perm_24
57
+ marlin_24_scale_perm[num_bits] = scale_perm_24
58
+ marlin_24_scale_perm_single[num_bits] = scale_perm_single_24
vllm/model_executor/layers/quantization/utils/marlin_perms.py
new
+58
@@ -0,0 +1,58 @@
1
+"""This file is used for /tests and /benchmarks"""
2
+import numpy
3
+import torch
4
+
5
+
6
+# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
7
+#
8
+# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
9
+# with the tensor-core format that is described here:
10
+# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
11
+#
12
+# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
13
+# (without the need to use ldmatrix instructions) # noqa: E501
14
+def get_perms(num_bits):
15
+ perm_list = []
16
+ for i in range(32):
17
+ perm1 = []
18
+ col = i // 4
19
+ for block in [0, 1]:
20
+ for row in [
21
+ 2 * (i % 4),
22
+ 2 * (i % 4) + 1,
23
+ 2 * (i % 4 + 4),
24
+ 2 * (i % 4 + 4) + 1,
25
+ ]:
26
+ perm1.append(16 * row + col + 8 * block)
27
+ for j in range(4):
28
+ perm_list.extend([p + 256 * j for p in perm1])
29
+
30
+ perm = numpy.array(perm_list)
31
+
32
+ if num_bits == 4:
33
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
34
+ elif num_bits == 8:
35
+ interleave = numpy.array([0, 2, 1, 3])
36
+ else:
37
+ raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
38
+
39
+ perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
40
+ perm = torch.from_numpy(perm)
41
+ scale_perm = []
42
+ for i in range(8):
43
+ scale_perm.extend([i + 8 * j for j in range(8)])
44
+ scale_perm_single = []
45
+ for i in range(4):
46
+ scale_perm_single.extend(
47
+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
48
+ return perm, scale_perm, scale_perm_single
49
+
50
+
51
+marlin_perm = {}
52
+marlin_scale_perm = {}
53
+marlin_scale_perm_single = {}
54
+for num_bits in [4, 8]:
55
+ perm, scale_perm, scale_perm_single = get_perms(num_bits)
56
+ marlin_perm[num_bits] = perm
57
+ marlin_scale_perm[num_bits] = scale_perm
58
+ marlin_scale_perm_single[num_bits] = scale_perm_single
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+132
-82
@@ -1,79 +1,28 @@
1
"""This file is used for /tests and /benchmarks"""
2
+import random
3
+
4
import numpy
5
import torch
6
5
-from vllm.model_executor.layers.quantization.gptq_marlin import (
6
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_TILE)
7
+from vllm.model_executor.layers.quantization.utils.format_24 import (
8
+ mask_creator, sparse_semi_structured_from_dense_cutlass)
9
+from vllm.model_executor.layers.quantization.utils.marlin_24_perms import (
10
+ marlin_24_perm, marlin_24_scale_perm, marlin_24_scale_perm_single)
11
+from vllm.model_executor.layers.quantization.utils.marlin_perms import (
12
+ marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
13
from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
get_pack_factor, quantize_weights, sort_weights)
15
16
__cuda_arch = torch.cuda.get_device_capability()
17
18
+MARLIN_TILE = 16
19
+
20
21
def is_marlin_supported():
22
return __cuda_arch[0] >= 8
23
24
17
-# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
18
-#
19
-# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
20
-# with the tensor-core format that is described here:
21
-# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
22
-#
23
-# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
24
-# (without the need to use ldmatrix instructions) # noqa: E501
25
-def _get_perms(num_bits):
26
- perm_list = []
27
- for i in range(32):
28
- perm1 = []
29
- col = i // 4
30
- for block in [0, 1]:
31
- for row in [
32
- 2 * (i % 4),
33
- 2 * (i % 4) + 1,
34
- 2 * (i % 4 + 4),
35
- 2 * (i % 4 + 4) + 1,
36
- ]:
37
- perm1.append(16 * row + col + 8 * block)
38
- for j in range(4):
39
- perm_list.extend([p + 256 * j for p in perm1])
40
-
41
- perm = numpy.array(perm_list)
42
-
43
- if num_bits == 4:
44
- interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
45
- elif num_bits == 8:
46
- interleave = numpy.array([0, 2, 1, 3])
47
- else:
48
- raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
49
-
50
- perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
51
- perm = torch.from_numpy(perm)
52
- scale_perm = []
53
- for i in range(8):
54
- scale_perm.extend([i + 8 * j for j in range(8)])
55
- scale_perm_single = []
56
- for i in range(4):
57
- scale_perm_single.extend(
58
- [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
59
- return perm, scale_perm, scale_perm_single
60
-
61
-
62
-_perm = {}
63
-_scale_perm = {}
64
-_scale_perm_single = {}
65
-for num_bits in [4, 8]:
66
- perm, scale_perm, scale_perm_single = _get_perms(num_bits)
67
- _perm[num_bits] = perm
68
- _scale_perm[num_bits] = scale_perm
69
- _scale_perm_single[num_bits] = scale_perm_single
70
-
71
-
72
-def marlin_permute_weights(q_w,
73
- size_k,
74
- size_n,
75
- num_bits,
76
- tile=GPTQ_MARLIN_TILE):
25
+def marlin_permute_weights(q_w, size_k, size_n, perm, tile=MARLIN_TILE):
26
assert q_w.shape == (size_k, size_n)
27
assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
28
assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
@@ -83,15 +32,14 @@ def marlin_permute_weights(q_w,
32
q_w = q_w.permute((0, 2, 1, 3))
33
q_w = q_w.reshape((size_k // tile, size_n * tile))
34
86
- q_w = q_w.reshape(
87
- (-1, _perm[num_bits].numel()))[:, _perm[num_bits]].reshape(q_w.shape)
35
+ q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
36
37
return q_w
38
39
92
-def marlin_weights(q_w, size_k, size_n, num_bits):
40
+def marlin_weights(q_w, size_k, size_n, num_bits, perm):
41
# Permute
94
- q_w = marlin_permute_weights(q_w, size_k, size_n, num_bits)
42
+ q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
43
44
# Pack
45
pack_factor = get_pack_factor(num_bits)
@@ -101,7 +49,6 @@ def marlin_weights(q_w, size_k, size_n, num_bits):
49
50
q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
51
dtype=numpy.uint32)
104
-
52
for i in range(pack_factor):
53
q_packed |= q_w[:, i::pack_factor] << num_bits * i
54
@@ -110,15 +57,12 @@ def marlin_weights(q_w, size_k, size_n, num_bits):
57
return q_packed
58
59
113
-def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
60
+def marlin_permute_scales(s, size_k, size_n, group_size, scale_perm,
61
+ scale_perm_single):
62
if group_size < size_k and group_size != -1:
115
- s = s.reshape((-1, len(_scale_perm[num_bits])))[:,
116
- _scale_perm[num_bits]]
63
+ s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
64
else:
118
- s = s.reshape(
119
- (-1,
120
- len(_scale_perm_single[num_bits])))[:,
121
- _scale_perm_single[num_bits]]
65
+ s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
66
s = s.reshape((-1, size_n)).contiguous()
67
68
return s
@@ -148,8 +92,11 @@ def marlin_quantize(
92
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
93
94
# Reformat to marlin
151
- marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits)
152
- marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, num_bits)
95
+ marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits,
96
+ marlin_perm[num_bits])
97
+ marlin_s = marlin_permute_scales(s, size_k, size_n, group_size,
98
+ marlin_scale_perm[num_bits],
99
+ marlin_scale_perm_single[num_bits])
100
101
# Create result
102
res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
@@ -159,15 +106,118 @@ def marlin_quantize(
106
return res_list
107
108
109
+def inject_24(w, size_k, size_n):
110
+ assert w.shape == (size_k, size_n)
111
+
112
+ mask = mask_creator(w.t()).t().cuda().bool()
113
+
114
+ return (mask * w).contiguous(), mask.contiguous()
115
+
116
+
117
+def check_24(w, num_rows_to_sample=50, _verbose=False):
118
+ BLOCK_SIZE = 4
119
+ MAX_NON_ZEROS = 2
120
+
121
+ w = w.t().contiguous()
122
+
123
+ print("check_24: w.shape = {}".format(w.shape))
124
+
125
+ num_rows, num_cols = w.shape
126
+ sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample)
127
+ if _verbose:
128
+ print(f"Sampled row idxs = {sampled_row_idxs}")
129
+
130
+ total_segments = 0
131
+ non_24_segments = 0
132
+ for i in sampled_row_idxs:
133
+ for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
134
+ total_segments += 1
135
+ block = w[i, j:j + BLOCK_SIZE]
136
+ num_nonzero = torch.count_nonzero(block)
137
+ if num_nonzero > MAX_NON_ZEROS:
138
+ print("i = {} j = {} block = {}".format(i, j, block))
139
+ non_24_segments += 1
140
+
141
+ print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
142
+
143
+
144
+def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
145
+ assert q_24.shape == (size_k, size_n)
146
+
147
+ # Remove zp to normalize over 0
148
+ max_q_val = (1 << num_bits) - 1
149
+ zp = (max_q_val + 1) // 2
150
+ q_24_no_zp = q_24 - zp
151
+
152
+ # Compress
153
+ q_24_no_zp = q_24_no_zp.t().contiguous()
154
+ q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
155
+ q_24_no_zp)
156
+ q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
157
+
158
+ # Restore zp
159
+ q_24_comp = q_24_no_zp_comp + zp
160
+
161
+ # Resize meta to its actual shape (without moving any data)
162
+ meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
163
+
164
+ return q_24_comp, meta
165
+
166
+
167
+def marlin_24_quantize(
168
+ w: torch.Tensor,
169
+ num_bits: int,
170
+ group_size: int,
171
+):
172
+ size_k, size_n = w.shape
173
+
174
+ # Normalize group_size
175
+ if group_size == -1:
176
+ group_size = size_k
177
+ assert group_size <= size_k
178
+
179
+ # Inject 2:4 sparsity
180
+ w_24, mask_24 = inject_24(w, size_k, size_n)
181
+
182
+ # Quantize
183
+ w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
184
+ num_bits,
185
+ group_size,
186
+ act_order=False)
187
+
188
+ # Compress quantized weight
189
+ q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
190
+ num_bits)
191
+ size_k_comp = size_k // 2
192
+
193
+ # Reformat to marlin
194
+ marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
195
+ num_bits, marlin_24_perm[num_bits])
196
+ marlin_24_s = marlin_permute_scales(s, size_k, size_n, group_size,
197
+ marlin_24_scale_perm[num_bits],
198
+ marlin_24_scale_perm_single[num_bits])
199
+
200
+ # Create result
201
+ res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s]
202
+ for i in range(len(res_list)):
203
+ res_list[i] = res_list[i].to(w.device)
204
+
205
+ return res_list
206
+
207
+
208
+def compute_max_diff(output, output_ref):
209
+ return torch.mean(torch.abs(output - output_ref)) / torch.mean(
210
+ torch.abs(output_ref))
211
+
212
+
213
class MarlinWorkspace:
214
164
- def __init__(self, out_features):
165
- assert (out_features % GPTQ_MARLIN_MIN_THREAD_N == 0), (
166
- "out_features = {} is undivisible by GPTQ_MARLIN_MIN_THREAD_N = {}"
167
- .format(out_features, GPTQ_MARLIN_MIN_THREAD_N))
215
+ def __init__(self, out_features, min_thread_n, max_parallel):
216
+ assert (out_features % min_thread_n == 0), (
217
+ "out_features = {} is undivisible by min_thread_n = {}".format(
218
+ out_features, min_thread_n))
219
169
- max_workspace_size = ((out_features // GPTQ_MARLIN_MIN_THREAD_N) *
170
- GPTQ_MARLIN_MAX_PARALLEL)
220
+ max_workspace_size = ((out_features // min_thread_n) * max_parallel)
221
222
self.scratch = torch.zeros(max_workspace_size,
223
dtype=torch.int,