A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Add marlin unit tests and marlin benchmark script (#4815)
alexm-nm committed
May 16, 2024 at 09:36 UTC
5c342570d7e4c73bfaa4c4057174b92117d322bb
6 files changed
+736
benchmarks/kernels/benchmark_marlin.py
new
+183
@@ -0,0 +1,183 @@
1
+import argparse
2
+
3
+import torch
4
+import torch.utils.benchmark as benchmark
5
+from benchmark_shapes import WEIGHT_SHAPES
6
+
7
+from vllm import _custom_ops as ops
8
+from vllm.model_executor.layers.quantization.gptq_marlin import (
9
+ GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
10
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11
+ MarlinWorkspace, marlin_quantize)
12
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
13
+ gptq_pack, quantize_weights, sort_weights)
14
+
15
+DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
16
+DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
17
+
18
+ACT_ORDER_OPTS = [False, True]
19
+K_FULL_OPTS = [False, True]
20
+
21
+
22
+def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
23
+ size_m, size_k, size_n):
24
+ label = "Quant Matmul"
25
+
26
+ sub_label = ("{}, act={} k_full={}, b={}, g={}, "
27
+ "MKN=({}x{}x{})".format(model, act_order, is_k_full, num_bits,
28
+ group_size, size_m, size_k, size_n))
29
+
30
+ print(f"Testing: {sub_label}")
31
+
32
+ a = torch.randn(size_m, size_k).to(torch.half).cuda()
33
+ b = torch.rand(size_k, size_n).to(torch.half).cuda()
34
+
35
+ a_tmp = (torch.zeros(size_m, size_k).to(torch.half).cuda())
36
+
37
+ # Marlin quant
38
+ (
39
+ marlin_w_ref,
40
+ marlin_q_w,
41
+ marlin_s,
42
+ marlin_g_idx,
43
+ marlin_sort_indices,
44
+ marlin_rand_perm,
45
+ ) = marlin_quantize(b, num_bits, group_size, act_order)
46
+
47
+ # GPTQ quant
48
+ (w_ref, q_w, s, g_idx,
49
+ rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
50
+ q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
51
+
52
+ # For act_order, sort the "weights" and "g_idx"
53
+ # so that group ids are increasing
54
+ repack_sort_indices = torch.empty(0, dtype=torch.int, device=b.device)
55
+ if act_order:
56
+ (q_w, g_idx, repack_sort_indices) = sort_weights(q_w, g_idx)
57
+
58
+ # Prepare
59
+ marlin_workspace = MarlinWorkspace(size_n)
60
+
61
+ globals = {
62
+ "marlin_w_ref": marlin_w_ref,
63
+ "marlin_q_w": marlin_q_w,
64
+ "marlin_s": marlin_s,
65
+ "marlin_g_idx": marlin_g_idx,
66
+ "marlin_sort_indices": marlin_sort_indices,
67
+ "marlin_rand_perm": marlin_rand_perm,
68
+ "q_w_gptq": q_w_gptq,
69
+ "repack_sort_indices": repack_sort_indices,
70
+ "num_bits": num_bits,
71
+ "group_size": group_size,
72
+ "size_m": size_m,
73
+ "size_n": size_n,
74
+ "size_k": size_k,
75
+ "is_k_full": is_k_full,
76
+ "a": a,
77
+ "a_tmp": a_tmp,
78
+ "gptq_marlin_gemm": ops.gptq_marlin_gemm,
79
+ "gptq_marlin_repack": ops.gptq_marlin_repack,
80
+ "marlin_workspace": marlin_workspace,
81
+ }
82
+
83
+ min_run_time = 1
84
+
85
+ # Warmup pytorch
86
+ for i in range(5):
87
+ torch.matmul(a, marlin_w_ref)
88
+
89
+ results.append(
90
+ benchmark.Timer(
91
+ stmt="torch.matmul(a, marlin_w_ref)",
92
+ globals=globals,
93
+ label=label,
94
+ sub_label=sub_label,
95
+ description="pytorch_gemm",
96
+ ).blocked_autorange(min_run_time=min_run_time))
97
+
98
+ results.append(
99
+ benchmark.Timer(
100
+ stmt=
101
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full)", # noqa: E501
102
+ globals=globals,
103
+ label=label,
104
+ sub_label=sub_label,
105
+ description="gptq_marlin_gemm",
106
+ ).blocked_autorange(min_run_time=min_run_time))
107
+
108
+ results.append(
109
+ benchmark.Timer(
110
+ stmt=
111
+ "q_res = gptq_marlin_repack(q_w_gptq, repack_sort_indices, size_k, size_n, num_bits)", # noqa: E501
112
+ globals=globals,
113
+ label=label,
114
+ sub_label=sub_label,
115
+ description="gptq_marlin_repack",
116
+ ).blocked_autorange(min_run_time=min_run_time))
117
+
118
+
119
+def main(args):
120
+ print("Benchmarking models:")
121
+ for i, model in enumerate(args.models):
122
+ print(f"[{i}] {model}")
123
+
124
+ results = []
125
+
126
+ for model in args.models:
127
+ for layer in WEIGHT_SHAPES[model]:
128
+ size_k = layer[0]
129
+ size_n = layer[1]
130
+
131
+ if len(args.limit_k) > 0 and size_k not in args.limit_k:
132
+ continue
133
+
134
+ if len(args.limit_n) > 0 and size_n not in args.limit_n:
135
+ continue
136
+
137
+ for act_order in ACT_ORDER_OPTS:
138
+ for is_k_full in K_FULL_OPTS:
139
+ for num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
140
+ for group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
141
+ if len(
142
+ args.limit_group_size
143
+ ) > 0 and group_size not in args.limit_group_size:
144
+ continue
145
+
146
+ # For act_order, the group_size must be less than
147
+ # size_k
148
+ if act_order and (group_size == size_k
149
+ or group_size == -1):
150
+ continue
151
+
152
+ for size_m in args.batch_sizes:
153
+ bench_run(results, model, act_order, is_k_full,
154
+ num_bits, group_size, size_m, size_k,
155
+ size_n)
156
+
157
+ compare = benchmark.Compare(results)
158
+ compare.print()
159
+
160
+
161
+# For quick benchmarking use:
162
+# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 # noqa E501
163
+#
164
+if __name__ == "__main__":
165
+ parser = argparse.ArgumentParser(
166
+ description="Benchmark Marlin across specified models/shapes/batches")
167
+ parser.add_argument(
168
+ "--models",
169
+ nargs="+",
170
+ type=str,
171
+ default=DEFAULT_MODELS,
172
+ choices=WEIGHT_SHAPES.keys(),
173
+ )
174
+ parser.add_argument("--batch-sizes",
175
+ nargs="+",
176
+ type=int,
177
+ default=DEFAULT_BATCH_SIZES)
178
+ parser.add_argument("--limit-k", nargs="+", type=int, default=[])
179
+ parser.add_argument("--limit-n", nargs="+", type=int, default=[])
180
+ parser.add_argument("--limit-group-size", nargs="+", type=int, default=[])
181
+
182
+ args = parser.parse_args()
183
+ main(args)
benchmarks/kernels/benchmark_shapes.py
new
+75
@@ -0,0 +1,75 @@
1
+WEIGHT_SHAPES = {
2
+ "ideal": [[4 * 256 * 32, 256 * 32]],
3
+ "mistralai/Mistral-7B-v0.1/TP1": [
4
+ [4096, 6144],
5
+ [4096, 4096],
6
+ [4096, 28672],
7
+ [14336, 4096],
8
+ ],
9
+ "mistralai/Mistral-7B-v0.1/TP2": [
10
+ [4096, 3072],
11
+ [2048, 4096],
12
+ [4096, 14336],
13
+ [7168, 4096],
14
+ ],
15
+ "mistralai/Mistral-7B-v0.1/TP4": [
16
+ [4096, 1536],
17
+ [1024, 4096],
18
+ [4096, 7168],
19
+ [3584, 4096],
20
+ ],
21
+ "meta-llama/Llama-2-7b-hf/TP1": [
22
+ [4096, 12288],
23
+ [4096, 4096],
24
+ [4096, 22016],
25
+ [11008, 4096],
26
+ ],
27
+ "meta-llama/Llama-2-7b-hf/TP2": [
28
+ [4096, 6144],
29
+ [2048, 4096],
30
+ [4096, 11008],
31
+ [5504, 4096],
32
+ ],
33
+ "meta-llama/Llama-2-7b-hf/TP4": [
34
+ [4096, 3072],
35
+ [1024, 4096],
36
+ [4096, 5504],
37
+ [2752, 4096],
38
+ ],
39
+ "meta-llama/Llama-2-13b-hf/TP1": [
40
+ [5120, 15360],
41
+ [5120, 5120],
42
+ [5120, 27648],
43
+ [13824, 5120],
44
+ ],
45
+ "meta-llama/Llama-2-13b-hf/TP2": [
46
+ [5120, 7680],
47
+ [2560, 5120],
48
+ [5120, 13824],
49
+ [6912, 5120],
50
+ ],
51
+ "meta-llama/Llama-2-13b-hf/TP4": [
52
+ [5120, 3840],
53
+ [1280, 5120],
54
+ [5120, 6912],
55
+ [3456, 5120],
56
+ ],
57
+ "meta-llama/Llama-2-70b-hf/TP1": [
58
+ [8192, 10240],
59
+ [8192, 8192],
60
+ [8192, 57344],
61
+ [28672, 8192],
62
+ ],
63
+ "meta-llama/Llama-2-70b-hf/TP2": [
64
+ [8192, 5120],
65
+ [4096, 8192],
66
+ [8192, 28672],
67
+ [14336, 8192],
68
+ ],
69
+ "meta-llama/Llama-2-70b-hf/TP4": [
70
+ [8192, 2560],
71
+ [2048, 8192],
72
+ [8192, 14336],
73
+ [7168, 8192],
74
+ ],
75
+}
tests/kernels/test_marlin_gemm.py
new
+158
@@ -0,0 +1,158 @@
1
+"""Tests for the marlin kernel.
2
+
3
+Run `pytest tests/kernels/marlin/test_marlin_gemm.py`.
4
+"""
5
+import pytest
6
+import torch
7
+
8
+from vllm import _custom_ops as ops
9
+from vllm.model_executor.layers.quantization.gptq_marlin import (
10
+ GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
11
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
+ MarlinWorkspace, is_marlin_supported, marlin_quantize, marlin_weights)
13
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
+ gptq_pack, quantize_weights, sort_weights)
15
+
16
+ACT_ORDER_OPTS = [False, True]
17
+K_FULL_OPTS = [False, True]
18
+
19
+K_CHUNKS = [128, 256]
20
+N_CHUNKS = [64, 128, 256]
21
+
22
+MNK_FACTORS = [
23
+ (1, 1, 1),
24
+ (1, 4, 8),
25
+ (1, 7, 5),
26
+ (1, 7 * 4, 5 * 1),
27
+ (13, 17, 67),
28
+ (26, 37, 13),
29
+ (67, 13, 11),
30
+]
31
+
32
+
33
+def rand_data(shape):
34
+ data = torch.rand(shape).to(torch.half).cuda()
35
+ return data
36
+
37
+
38
+@pytest.mark.skipif(not is_marlin_supported(),
39
+ reason="Marlin is not supported on this GPU type.")
40
+@pytest.mark.parametrize("k_chunk", K_CHUNKS)
41
+@pytest.mark.parametrize("n_chunk", N_CHUNKS)
42
+@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
43
+@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
44
+@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
45
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
46
+def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
47
+ mnk_factors):
48
+ m_factor, n_factor, k_factor = mnk_factors
49
+
50
+ size_m = m_factor
51
+ size_k = k_chunk * k_factor
52
+ size_n = n_chunk * n_factor
53
+
54
+ print(f"MNK = {size_m} {size_n} {size_k}")
55
+
56
+ # Filter act_order
57
+ if act_order:
58
+ if group_size == -1:
59
+ return
60
+ if group_size == size_k:
61
+ return
62
+
63
+ # Normalize group_size
64
+ if group_size == -1:
65
+ group_size = size_k
66
+ assert group_size <= size_k
67
+
68
+ # Create input
69
+ b_weight = rand_data((size_k, size_n))
70
+
71
+ # Quantize (and apply act_order if provided)
72
+ w_ref, q_w, s, g_idx, rand_perm = quantize_weights(b_weight, num_bits,
73
+ group_size, act_order)
74
+
75
+ # Pack to GPTQ format
76
+ q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
77
+
78
+ # For act_order, sort the "weights" and "g_idx" so that group ids are
79
+ # increasing
80
+ sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
81
+ if act_order:
82
+ q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
83
+
84
+ # Pack to Marlin format
85
+ marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits)
86
+
87
+ # Run Marlin repack GPU kernel
88
+ marlin_q_w_2 = ops.gptq_marlin_repack(
89
+ q_w_gptq,
90
+ sort_indices,
91
+ size_k,
92
+ size_n,
93
+ num_bits,
94
+ )
95
+ torch.cuda.synchronize()
96
+
97
+ assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
98
+
99
+
100
+@pytest.mark.skipif(not is_marlin_supported(),
101
+ reason="Marlin is not supported on this GPU type.")
102
+@pytest.mark.parametrize("k_chunk", K_CHUNKS)
103
+@pytest.mark.parametrize("n_chunk", N_CHUNKS)
104
+@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
105
+@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
106
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
107
+@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
108
+@pytest.mark.parametrize("is_k_full", K_FULL_OPTS)
109
+def test_marlin_gemm(
110
+ k_chunk,
111
+ n_chunk,
112
+ num_bits,
113
+ group_size,
114
+ mnk_factors,
115
+ act_order,
116
+ is_k_full,
117
+):
118
+ m_factor, n_factor, k_factor = mnk_factors
119
+
120
+ size_m = m_factor
121
+ size_k = k_chunk * k_factor
122
+ size_n = n_chunk * n_factor
123
+
124
+ print(f"MNK = {size_m} {size_n} {size_k}")
125
+ print(f"groupsize = {group_size}")
126
+
127
+ if act_order:
128
+ if group_size == -1:
129
+ return
130
+ if group_size == size_k:
131
+ return
132
+
133
+ a_input = rand_data((size_m, size_k))
134
+ b_weight = rand_data((size_k, size_n))
135
+
136
+ w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
137
+ b_weight, num_bits, group_size, act_order)
138
+
139
+ workspace = MarlinWorkspace(size_n)
140
+
141
+ output = ops.gptq_marlin_gemm(
142
+ a_input,
143
+ marlin_q_w,
144
+ marlin_s,
145
+ g_idx,
146
+ sort_indices,
147
+ workspace.scratch,
148
+ num_bits,
149
+ a_input.shape[0],
150
+ b_weight.shape[1],
151
+ a_input.shape[1],
152
+ is_k_full,
153
+ )
154
+ output_ref = torch.matmul(a_input, w_ref)
155
+
156
+ torch.cuda.synchronize()
157
+
158
+ assert torch.allclose(output, output_ref, rtol=1e-2)
vllm/model_executor/layers/quantization/utils/__init__.py
vllm/model_executor/layers/quantization/utils/marlin_utils.py
new
+174
@@ -0,0 +1,174 @@
1
+"""This file is used for /tests and /benchmarks"""
2
+import numpy
3
+import torch
4
+
5
+from vllm.model_executor.layers.quantization.gptq_marlin import (
6
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_TILE)
7
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
8
+ get_pack_factor, quantize_weights, sort_weights)
9
+
10
+__cuda_arch = torch.cuda.get_device_capability()
11
+
12
+
13
+def is_marlin_supported():
14
+ return __cuda_arch[0] >= 8
15
+
16
+
17
+# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
18
+#
19
+# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
20
+# with the tensor-core format that is described here:
21
+# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
22
+#
23
+# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
24
+# (without the need to use ldmatrix instructions) # noqa: E501
25
+def _get_perms(num_bits):
26
+ perm_list = []
27
+ for i in range(32):
28
+ perm1 = []
29
+ col = i // 4
30
+ for block in [0, 1]:
31
+ for row in [
32
+ 2 * (i % 4),
33
+ 2 * (i % 4) + 1,
34
+ 2 * (i % 4 + 4),
35
+ 2 * (i % 4 + 4) + 1,
36
+ ]:
37
+ perm1.append(16 * row + col + 8 * block)
38
+ for j in range(4):
39
+ perm_list.extend([p + 256 * j for p in perm1])
40
+
41
+ perm = numpy.array(perm_list)
42
+
43
+ if num_bits == 4:
44
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
45
+ elif num_bits == 8:
46
+ interleave = numpy.array([0, 2, 1, 3])
47
+ else:
48
+ raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
49
+
50
+ perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
51
+ perm = torch.from_numpy(perm)
52
+ scale_perm = []
53
+ for i in range(8):
54
+ scale_perm.extend([i + 8 * j for j in range(8)])
55
+ scale_perm_single = []
56
+ for i in range(4):
57
+ scale_perm_single.extend(
58
+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
59
+ return perm, scale_perm, scale_perm_single
60
+
61
+
62
+_perm = {}
63
+_scale_perm = {}
64
+_scale_perm_single = {}
65
+for num_bits in [4, 8]:
66
+ perm, scale_perm, scale_perm_single = _get_perms(num_bits)
67
+ _perm[num_bits] = perm
68
+ _scale_perm[num_bits] = scale_perm
69
+ _scale_perm_single[num_bits] = scale_perm_single
70
+
71
+
72
+def marlin_permute_weights(q_w,
73
+ size_k,
74
+ size_n,
75
+ num_bits,
76
+ tile=GPTQ_MARLIN_TILE):
77
+ assert q_w.shape == (size_k, size_n)
78
+ assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
79
+ assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
80
+
81
+ # Permute weights to 16x64 marlin tiles
82
+ q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
83
+ q_w = q_w.permute((0, 2, 1, 3))
84
+ q_w = q_w.reshape((size_k // tile, size_n * tile))
85
+
86
+ q_w = q_w.reshape(
87
+ (-1, _perm[num_bits].numel()))[:, _perm[num_bits]].reshape(q_w.shape)
88
+
89
+ return q_w
90
+
91
+
92
+def marlin_weights(q_w, size_k, size_n, num_bits):
93
+ # Permute
94
+ q_w = marlin_permute_weights(q_w, size_k, size_n, num_bits)
95
+
96
+ # Pack
97
+ pack_factor = get_pack_factor(num_bits)
98
+ orig_device = q_w.device
99
+
100
+ q_w = q_w.cpu().numpy().astype(numpy.uint32)
101
+
102
+ q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
103
+ dtype=numpy.uint32)
104
+
105
+ for i in range(pack_factor):
106
+ q_packed |= q_w[:, i::pack_factor] << num_bits * i
107
+
108
+ q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
109
+
110
+ return q_packed
111
+
112
+
113
+def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
114
+ if group_size < size_k and group_size != -1:
115
+ s = s.reshape((-1, len(_scale_perm[num_bits])))[:,
116
+ _scale_perm[num_bits]]
117
+ else:
118
+ s = s.reshape(
119
+ (-1,
120
+ len(_scale_perm_single[num_bits])))[:,
121
+ _scale_perm_single[num_bits]]
122
+ s = s.reshape((-1, size_n)).contiguous()
123
+
124
+ return s
125
+
126
+
127
+def marlin_quantize(
128
+ w: torch.Tensor,
129
+ num_bits: int,
130
+ group_size: int,
131
+ act_order: bool,
132
+):
133
+ size_k, size_n = w.shape
134
+
135
+ # Normalize group_size
136
+ if group_size == -1:
137
+ group_size = size_k
138
+ assert group_size <= size_k
139
+
140
+ # Quantize (and apply act_order if provided)
141
+ w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
142
+ act_order)
143
+
144
+ # For act_order, sort the "weights" and "g_idx" so that group ids are
145
+ # increasing
146
+ sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
147
+ if act_order:
148
+ q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
149
+
150
+ # Reformat to marlin
151
+ marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits)
152
+ marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, num_bits)
153
+
154
+ # Create result
155
+ res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
156
+ for i in range(len(res_list)):
157
+ res_list[i] = res_list[i].to(w.device)
158
+
159
+ return res_list
160
+
161
+
162
+class MarlinWorkspace:
163
+
164
+ def __init__(self, out_features):
165
+ assert (out_features % GPTQ_MARLIN_MIN_THREAD_N == 0), (
166
+ "out_features = {} is undivisible by GPTQ_MARLIN_MIN_THREAD_N = {}"
167
+ .format(out_features, GPTQ_MARLIN_MIN_THREAD_N))
168
+
169
+ max_workspace_size = ((out_features // GPTQ_MARLIN_MIN_THREAD_N) *
170
+ GPTQ_MARLIN_MAX_PARALLEL)
171
+
172
+ self.scratch = torch.zeros(max_workspace_size,
173
+ dtype=torch.int,
174
+ device="cuda")
vllm/model_executor/layers/quantization/utils/quant_utils.py
new
+146
@@ -0,0 +1,146 @@
1
+"""This file is used for /tests and /benchmarks"""
2
+import numpy
3
+import torch
4
+
5
+SUPPORTED_NUM_BITS = [4, 8]
6
+SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
7
+
8
+
9
+def get_pack_factor(num_bits):
10
+ assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
11
+ return 32 // num_bits
12
+
13
+
14
+def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
15
+ assert q_w.shape == w_ref.shape
16
+
17
+ orig_device = q_w.device
18
+ k_size, _ = q_w.shape
19
+
20
+ g_idx = torch.zeros((k_size, ), dtype=torch.int32)
21
+ for i in range(k_size):
22
+ g_idx[i] = i // group_size
23
+
24
+ # Simulate act_order by doing a random permutation on K
25
+ rand_perm = torch.randperm(k_size)
26
+
27
+ g_idx = g_idx[rand_perm].contiguous()
28
+ q_w = q_w[rand_perm, :].contiguous()
29
+ w_ref = w_ref[rand_perm, :].contiguous()
30
+
31
+ return (
32
+ w_ref.to(device=orig_device),
33
+ q_w.to(device=orig_device),
34
+ g_idx.to(device=orig_device),
35
+ rand_perm.to(device=orig_device),
36
+ )
37
+
38
+
39
+def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
40
+ act_order: bool):
41
+ orig_device = w.device
42
+ size_k, size_n = w.shape
43
+
44
+ assert w.is_floating_point(), "w must be float"
45
+ assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
46
+ assert group_size in SUPPORTED_GROUP_SIZES + [
47
+ size_k
48
+ ], f"Unsupported groupsize = {group_size}"
49
+
50
+ if group_size == -1:
51
+ group_size = size_k
52
+ assert group_size <= size_k
53
+
54
+ max_q_val = 2**num_bits - 1
55
+ half_q_val = (max_q_val + 1) // 2
56
+
57
+ # Reshape to [groupsize, -1]
58
+ if group_size < size_k:
59
+ w = w.reshape((-1, group_size, size_n))
60
+ w = w.permute(1, 0, 2)
61
+ w = w.reshape((group_size, -1))
62
+
63
+ # Compute scale for each group
64
+ s = torch.max(torch.abs(w), 0, keepdim=True)[0]
65
+ s *= 2 / max_q_val # 2 => symmetric
66
+
67
+ # Quantize
68
+ q_w = torch.round(w / s).int()
69
+ q_w += half_q_val
70
+ q_w = torch.clamp(q_w, 0, max_q_val)
71
+
72
+ # Compute ref (dequantized)
73
+ w_ref = (q_w - half_q_val).half() * s
74
+
75
+ # Restore original shapes
76
+ if group_size < size_k:
77
+
78
+ def reshape_w(w):
79
+ w = w.reshape((group_size, -1, size_n))
80
+ w = w.permute(1, 0, 2)
81
+ w = w.reshape((size_k, size_n)).contiguous()
82
+ return w
83
+
84
+ q_w = reshape_w(q_w)
85
+ w_ref = reshape_w(w_ref)
86
+
87
+ s = s.reshape((-1, size_n)).contiguous()
88
+
89
+ # Apply act_order
90
+ g_idx = torch.empty(0, dtype=torch.int, device=w.device)
91
+ rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
92
+ if act_order:
93
+ assert (
94
+ group_size < size_k
95
+ ), "For act_order, groupsize = {} must be less than size_k = {}".format(
96
+ group_size, size_k)
97
+
98
+ w_ref, q_w, g_idx, rand_perm = permute_rows(q_w, w_ref, group_size)
99
+
100
+ return (
101
+ w_ref.to(device=orig_device),
102
+ q_w.to(device=orig_device),
103
+ s.to(device=orig_device),
104
+ g_idx.to(device=orig_device),
105
+ rand_perm.to(device=orig_device),
106
+ )
107
+
108
+
109
+def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
110
+ orig_device = q_w.device
111
+
112
+ sort_indices = torch.argsort(g_idx).to(
113
+ dtype=torch.int32) # Sort based on g_idx
114
+
115
+ g_idx = g_idx[sort_indices].contiguous()
116
+ q_w = q_w[sort_indices, :].contiguous()
117
+
118
+ return (
119
+ q_w.to(device=orig_device),
120
+ g_idx.to(device=orig_device),
121
+ sort_indices.to(device=orig_device),
122
+ )
123
+
124
+
125
+def gptq_pack(
126
+ q_w: torch.Tensor,
127
+ num_bits: int,
128
+ size_k: int,
129
+ size_n: int,
130
+):
131
+ assert q_w.shape == (size_k, size_n)
132
+
133
+ pack_factor = get_pack_factor(num_bits)
134
+ assert size_k % pack_factor == 0
135
+
136
+ orig_device = q_w.device
137
+
138
+ q_w = q_w.cpu().numpy().astype(numpy.uint32)
139
+
140
+ q_res = numpy.zeros((size_k // pack_factor, size_n), dtype=numpy.uint32)
141
+
142
+ for i in range(pack_factor):
143
+ q_res |= q_w[i::pack_factor, :] << num_bits * i
144
+
145
+ q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
146
+ return q_res